merged with stable

.hgtags

@@ -1,80 +1,81 b''
 1bd3e92b7e2e2d2024152b34bb88dff1db544a71 v4.0.0
 170c5398320ea6cddd50955e88d408794c21d43a v4.0.1
 c3fe200198f5aa34cf2e4066df2881a9cefe3704 v4.1.0
 7fd5c850745e2ea821fb4406af5f4bff9b0a7526 v4.1.1
 41c87da28a179953df86061d817bc35533c66dd2 v4.1.2
 baaf9f5bcea3bae0ef12ae20c8b270482e62abb6 v4.2.0
 32a70c7e56844a825f61df496ee5eaf8c3c4e189 v4.2.1
 fa695cdb411d294679ac081d595ac654e5613b03 v4.3.0
 0e4dc11b58cad833c513fe17bac39e6850edf959 v4.3.1
 8a876f48f5cb1d018b837db28ff928500cb32cfb v4.4.0
 8dd86b410b1aac086ffdfc524ef300f896af5047 v4.4.1
 d2514226abc8d3b4f6fb57765f47d1b6fb360a05 v4.4.2
 27d783325930af6dad2741476c0d0b1b7c8415c2 v4.5.0
 7f2016f352abcbdba4a19d4039c386e9629449da v4.5.1
 416fec799314c70a5c780fb28b3357b08869333a v4.5.2
 27c3b85fafc83143e6678fbc3da69e1615bcac55 v4.6.0
 5ad13deb9118c2a5243d4032d4d9cc174e5872db v4.6.1
 2be921e01fa24bb102696ada596f87464c3666f6 v4.7.0
 7198bdec29c2872c974431d55200d0398354cdb1 v4.7.1
 bd1c8d230fe741c2dfd7100a0ef39fd0774fd581 v4.7.2
 9731914f89765d9628dc4dddc84bc9402aa124c8 v4.8.0
 c5a2b7d0e4bbdebc4a62d7b624befe375207b659 v4.9.0
 d9aa3b27ac9f7e78359775c75fedf7bfece232f1 v4.9.1
 4ba4d74981cec5d6b28b158f875a2540952c2f74 v4.10.0
 0a6821cbd6b0b3c21503002f88800679fa35ab63 v4.10.1
 434ad90ec8d621f4416074b84f6e9ce03964defb v4.10.2
 68baee10e698da2724c6e0f698c03a6abb993bf2 v4.10.3
 00821d3afd1dce3f4767cc353f84a17f7d5218a1 v4.10.4
 22f6744ad8cc274311825f63f953e4dee2ea5cb9 v4.10.5
 96eb24bea2f5f9258775245e3f09f6fa0a4dda01 v4.10.6
 3121217a812c956d7dd5a5875821bd73e8002a32 v4.11.0
 fa98b454715ac5b912f39e84af54345909a2a805 v4.11.1
 3982abcfdcc229a723cebe52d3a9bcff10bba08e v4.11.2
 33195f145db9172f0a8f1487e09207178a6ab065 v4.11.3
 194c74f33e32bbae6fc4d71ec5a999cff3c13605 v4.11.4
 8fbd8b0c3ddc2fa4ac9e4ca16942a03eb593df2d v4.11.5
 f0609aa5d5d05a1ca2f97c3995542236131c9d8a v4.11.6
 b5b30547d90d2e088472a70c84878f429ffbf40d v4.12.0
 9072253aa8894d20c00b4a43dc61c2168c1eff94 v4.12.1
 6a517543ea9ef9987d74371bd2a315eb0b232dc9 v4.12.2
 7fc0731b024c3114be87865eda7ab621cc957e32 v4.12.3
 6d531c0b068c6eda62dddceedc9f845ecb6feb6f v4.12.4
 3d6bf2d81b1564830eb5e83396110d2a9a93eb1e v4.13.0
 5468fc89e708bd90e413cd0d54350017abbdbc0e v4.13.1
 610d621550521c314ee97b3d43473ac0bcf06fb8 v4.13.2
 7dc62c090881fb5d03268141e71e0940d7c3295d v4.13.3
 9151328c1c46b72ba6f00d7640d9141e75aa1ca2 v4.14.0
 a47eeac5dfa41fa6779d90452affba4091c3ade8 v4.14.1
 4b34ce0d2c3c10510626b3b65044939bb7a2cddf v4.15.0
 14502561d22e6b70613674cd675ae9a604b7989f v4.15.1
 4aaa40b605b01af78a9f6882eca561c54b525ef0 v4.15.2
 797744642eca86640ed20bef2cd77445780abaec v4.16.0
 6c3452c7c25ed35ff269690929e11960ed6ad7d3 v4.16.1
 5d8057df561c4b6b81b6401aed7d2f911e6e77f7 v4.16.2
 13acfc008896ef4c62546bab5074e8f6f89b4fa7 v4.17.0
 45b9b610976f483877142fe75321808ce9ebac59 v4.17.1
 ad5bd0c4bd322fdbd04bb825a3d027e08f7a3901 v4.17.2
 037f5794b55a6236d68f6485a485372dde6566e0 v4.17.3
 83bc3100cfd6094c1d04f475ddb299b7dc3d0b33 v4.17.4
 e3de8c95baf8cc9109ca56aee8193a2cb6a54c8a v4.17.4
 f37a3126570477543507f0bc9d245ce75546181a v4.18.0
 71d8791463e87b64c1a18475de330ee600d37561 v4.18.1
 4bd6b75dac1d25c64885d4d49385e5533f21c525 v4.18.2
 12ed92fe57f2e9fc7b71dc0b65e26c2da5c7085f v4.18.3
 ddef396a6567117de531d67d44c739cbbfc3eebb v4.19.0
 c0c65acd73914bf4368222d510afe1161ab8c07c v4.19.1
 7ac623a4a2405917e2af660d645ded662011e40d v4.19.2
 ef7ffda65eeb90c3ba88590a6cb816ef9b0bc232 v4.19.3
 3e635489bb7961df93b01e42454ad1a8730ae968 v4.20.0
 7e2eb896a02ca7cd2cd9f0f853ef3dac3f0039e3 v4.20.1
 8bb5fece08ab65986225b184e46f53d2a71729cb v4.21.0
 90734aac31ee4563bbe665a43ff73190cc762275 v4.22.0
 a9655707f7cf4146affc51c12fe5ed8e02898a57 v4.23.0
 56310d93b33b97535908ef9c7b0985b89bb7fad2 v4.23.1
 7637c38528fa38c1eabc1fde6a869c20995a0da7 v4.23.2
 6aeb4ac3ef7f0ac699c914740dad3688c9495e83 v4.24.0
 6eaf953da06e468a4c4e5239d3d0e700bda6b163 v4.24.1
 f8161cbc2d94a935d3c395a0e758d9a094287169 v4.25.0
 77fe47b5b39338e71b2c040de2c0359b529b6251 v4.25.1
 27475bd8a718b9a00a37a8563c4927120865ad85 v4.25.2
+b4ba10dcb4ab67d02b8c5cff32a3827f6c4fdedb v4.26.0

.release.cfg

@@ -1,33 +1,34 b''
 [DEFAULT]
 done = false
 
 [task:bump_version]
 done = true
 
 [task:rc_tools_pinned]
 done = true
 
 [task:fixes_on_stable]
 done = true
 
 [task:pip2nix_generated]
 done = true
 
 [task:changelog_updated]
 done = true
 
 [task:generate_api_docs]
 done = true
 
+[task:updated_translation]
+done = true
+
 [release]
 state = prepared
-version = 4.25.2
-
-[task:updated_translation]
+version = 4.26.0
 
 [task:generate_js_routes]
 
 [task:updated_trial_license]
 
 [task:generate_oss_licenses]
 

docs/admin/repo_admin/repo-admin-tasks.rst

@@ -1,104 +1,118 b''
 .. _repo-admin-tasks:
 
 Common Admin Tasks for Repositories
 -----------------------------------
 
 
 Manually Force Delete Repository
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 In case of attached forks or pull-requests repositories should be archived.
 Here is how to force delete a repository and remove all dependent objects
 
 
 .. code-block:: bash
    :dedent: 1
 
     # starts the ishell interactive prompt
     $ rccontrol ishell enterprise-1
 
 .. code-block:: python
    :dedent: 1
 
     In [4]: from rhodecode.model.repo import RepoModel
     In [3]: repo = Repository.get_by_repo_name('test_repos/repo_with_prs')
     In [5]: RepoModel().delete(repo, forks='detach', pull_requests='delete')
     In [6]: Session().commit()
 
 
 Below is a fully automated example to force delete repositories reading from a
 file where each line is a repository name. This can be executed via simple CLI command
 without entering the interactive shell.
 
 Save the below content as a file named `repo_delete_task.py`
 
 
 .. code-block:: python
    :dedent: 1
 
     from rhodecode.model.db import *
     from rhodecode.model.repo import RepoModel
     with open('delete_repos.txt', 'rb') as f:
         # read all lines from file
         repos = f.readlines()
         for repo_name in repos:
             repo_name = repo_name.strip()  # cleanup the name just in case
             repo = Repository.get_by_repo_name(repo_name)
             if not repo:
                 raise Exception('Repo with name {} not found'.format(repo_name))
             RepoModel().delete(repo, forks='detach', pull_requests='delete')
             Session().commit()
             print('Removed repository {}'.format(repo_name))
 
 
 The code above will read the names of repositories from a file called `delete_repos.txt`
 Each lines should represent a single name e.g `repo_name_1` or `repo_group/repo_name_2`
 
 Run this line from CLI to execute the code from the `repo_delete_task.py` file and
 exit the ishell after the execution::
 
     echo "%run repo_delete_task.py" | rccontrol ishell enterprise-1
 
 
 
 
 Bulk edit permissions for all repositories or groups
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 In case when a permissions should be applied in bulk here are two ways to apply
 the permissions onto *all* repositories and/or repository groups.
 
 1) Start by running the interactive ishell interface
 
 .. code-block:: bash
    :dedent: 1
 
     # starts the ishell interactive prompt
     $ rccontrol ishell enterprise-1
 
 
 2a) Add user called 'admin' into all repositories with write permission.
     Permissions can be also `repository.read`, `repository.admin`, `repository.none`
 
 .. code-block:: python
    :dedent: 1
 
    In [1]: from rhodecode.model.repo import RepoModel
    In [2]: user = User.get_by_username('admin')
    In [3]: permission_name = 'repository.write'
    In [4]: for repo in Repository.get_all():
       ...:     RepoModel().grant_user_permission(repo, user, permission_name)
       ...:     Session().commit()
 
 2b) Add user called 'admin' into all repository groups with write permission.
     Permissions can be also can be `group.read`, `group.admin`, `group.none`
 
 .. code-block:: python
    :dedent: 1
 
    In [1]: from rhodecode.model.repo import RepoModel
    In [2]: user = User.get_by_username('admin')
    In [3]: permission_name = 'group.write'
    In [4]: for repo_group in RepoGroup.get_all():
        ...:     RepoGroupModel().grant_user_permission(repo_group, user, permission_name)
-       ...:     Session().commit()
  No newline at end of file
+       ...:     Session().commit()
+
+
+Delete a problematic pull request
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. code-block:: python
+   :dedent: 1
+
+   In [1]: from rhodecode.model.pull_request import PullRequestModel
+   In [2]: pullrequest_id = 123
+   In [3]: pr = PullRequest.get(pullrequest_id)
+   In [4]: super_admin = User.get_first_super_admin()
+   In [5]: PullRequestModel().delete(pr, super_admin)
+   In [6]: Session().commit()

rhodecode/apps/ops/__init__.py

@@ -1,56 +1,64 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 from rhodecode.apps._base import ADMIN_PREFIX
 
 
 def admin_routes(config):
     from rhodecode.apps.ops.views import OpsView
 
     config.add_route(
         name='ops_ping',
         pattern='/ping')
     config.add_view(
         OpsView,
         attr='ops_ping',
         route_name='ops_ping', request_method='GET',
         renderer='json_ext')
 
     config.add_route(
         name='ops_error_test',
         pattern='/error')
     config.add_view(
         OpsView,
         attr='ops_error_test',
         route_name='ops_error_test', request_method='GET',
         renderer='json_ext')
 
     config.add_route(
         name='ops_redirect_test',
         pattern='/redirect')
     config.add_view(
         OpsView,
         attr='ops_redirect_test',
         route_name='ops_redirect_test', request_method='GET',
         renderer='json_ext')
 
+    config.add_route(
+        name='ops_healthcheck',
+        pattern='/status')
+    config.add_view(
+        OpsView,
+        attr='ops_healthcheck',
+        route_name='ops_healthcheck', request_method='GET',
+        renderer='json_ext')
 
 def includeme(config):
     config.include(admin_routes, route_prefix=ADMIN_PREFIX + '/ops')

rhodecode/apps/ops/views.py

@@ -1,74 +1,97 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import time
 import logging
 
 
 from pyramid.httpexceptions import HTTPFound
 
 from rhodecode.apps._base import BaseAppView
 from rhodecode.lib import helpers as h
+from rhodecode.lib.auth import LoginRequired
+from rhodecode.model.db import UserApiKeys
 
 log = logging.getLogger(__name__)
 
 
 class OpsView(BaseAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         c.user = c.auth_user.get_instance()
 
         return c
 
     def ops_ping(self):
         data = {
             'instance': self.request.registry.settings.get('instance_id'),
         }
         if getattr(self.request, 'user'):
             caller_name = 'anonymous'
             if self.request.user.user_id:
                 caller_name = self.request.user.username
 
             data.update({
                 'caller_ip': self.request.user.ip_addr,
                 'caller_name': caller_name,
             })
         return {'ok': data}
 
     def ops_error_test(self):
         """
         Test exception handling and emails on errors
         """
 
         class TestException(Exception):
             pass
         # add timeout so we add some sort of rate limiter
         time.sleep(2)
         msg = ('RhodeCode Enterprise test exception. '
                'Client:{}. Generation time: {}.'.format(self.request.user, time.time()))
         raise TestException(msg)
 
     def ops_redirect_test(self):
         """
         Test redirect handling
         """
         redirect_to = self.request.GET.get('to') or h.route_path('home')
         raise HTTPFound(redirect_to)
+
+    @LoginRequired(auth_token_access=[UserApiKeys.ROLE_HTTP])
+    def ops_healthcheck(self):
+        from rhodecode.lib.system_info import load_system_info
+
+        vcsserver_info = load_system_info('vcs_server')
+        if vcsserver_info:
+            vcsserver_info = vcsserver_info['human_value']
+
+        db_info = load_system_info('database_info')
+        if db_info:
+            db_info = db_info['human_value']
+
+        health_spec = {
+            'caller_ip': self.request.user.ip_addr,
+            'vcsserver': vcsserver_info,
+            'db': db_info,
+        }
+
+        return {'healthcheck': health_spec}
+

rhodecode/lib/celerylib/__init__.py

@@ -1,69 +1,72 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import socket
 import logging
 
 import rhodecode
 from zope.cachedescriptors.property import Lazy as LazyProperty
 from rhodecode.lib.celerylib.loader import (
     celery_app, RequestContextTask, get_logger)
 
 async_task = celery_app.task
 
 
 log = logging.getLogger(__name__)
 
 
 class ResultWrapper(object):
     def __init__(self, task):
         self.task = task
 
     @LazyProperty
     def result(self):
         return self.task
 
 
 def run_task(task, *args, **kwargs):
     log.debug('Got task `%s` for execution', task)
+    if task is None:
+        raise ValueError('Got non-existing task for execution')
+
     if rhodecode.CELERY_ENABLED:
         celery_is_up = False
         try:
             t = task.apply_async(args=args, kwargs=kwargs)
             celery_is_up = True
             log.debug('executing task %s:%s in async mode', t.task_id, task)
             return t
 
         except socket.error as e:
             if isinstance(e, IOError) and e.errno == 111:
                 log.error('Unable to connect to celeryd `%s`. Sync execution', e)
             else:
                 log.exception("Exception while connecting to celeryd.")
         except KeyError as e:
             log.error('Unable to connect to celeryd `%s`. Sync execution', e)
         except Exception as e:
             log.exception(
                 "Exception while trying to run task asynchronous. "
                 "Fallback to sync execution.")
 
     else:
         log.debug('executing task %s:%s in sync mode', 'TASK', task)
 
     return ResultWrapper(task(*args, **kwargs))

rhodecode/lib/celerylib/tasks.py

@@ -1,407 +1,410 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 RhodeCode task modules, containing all task that suppose to be run
 by celery daemon
 """
 
 import os
 import time
 
 from pyramid import compat
 from pyramid_mailer.mailer import Mailer
 from pyramid_mailer.message import Message
 from email.utils import formatdate
 
 import rhodecode
 from rhodecode.lib import audit_logger
 from rhodecode.lib.celerylib import get_logger, async_task, RequestContextTask, run_task
 from rhodecode.lib import hooks_base
 from rhodecode.lib.utils2 import safe_int, str2bool, aslist
 from rhodecode.model.db import (
     Session, IntegrityError, true, Repository, RepoGroup, User)
 from rhodecode.model.permission import PermissionModel
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def send_email(recipients, subject, body='', html_body='', email_config=None,
                extra_headers=None):
     """
     Sends an email with defined parameters from the .ini files.
 
     :param recipients: list of recipients, it this is empty the defined email
         address from field 'email_to' is used instead
     :param subject: subject of the mail
     :param body: body of the mail
     :param html_body: html version of body
     :param email_config: specify custom configuration for mailer
     :param extra_headers: specify custom headers
     """
     log = get_logger(send_email)
 
     email_config = email_config or rhodecode.CONFIG
 
     mail_server = email_config.get('smtp_server') or None
     if mail_server is None:
         log.error("SMTP server information missing. Sending email failed. "
                   "Make sure that `smtp_server` variable is configured "
                   "inside the .ini file")
         return False
 
     subject = "%s %s" % (email_config.get('email_prefix', ''), subject)
 
     if recipients:
         if isinstance(recipients, compat.string_types):
             recipients = recipients.split(',')
     else:
         # if recipients are not defined we send to email_config + all admins
         admins = []
         for u in User.query().filter(User.admin == true()).all():
             if u.email:
                 admins.append(u.email)
         recipients = []
         config_email = email_config.get('email_to')
         if config_email:
             recipients += [config_email]
         recipients += admins
 
     # translate our LEGACY config into the one that pyramid_mailer supports
     email_conf = dict(
         host=mail_server,
         port=email_config.get('smtp_port', 25),
         username=email_config.get('smtp_username'),
         password=email_config.get('smtp_password'),
 
         tls=str2bool(email_config.get('smtp_use_tls')),
         ssl=str2bool(email_config.get('smtp_use_ssl')),
 
         # SSL key file
         # keyfile='',
 
         # SSL certificate file
         # certfile='',
 
         # Location of maildir
         # queue_path='',
 
         default_sender=email_config.get('app_email_from', '[e-mail]'),
 
         debug=str2bool(email_config.get('smtp_debug')),
         # /usr/sbin/sendmail	Sendmail executable
         # sendmail_app='',
 
         # {sendmail_app} -t -i -f {sender}	Template for sendmail execution
         # sendmail_template='',
     )
 
     if extra_headers is None:
         extra_headers = {}
 
     extra_headers.setdefault('Date', formatdate(time.time()))
 
     if 'thread_ids' in extra_headers:
         thread_ids = extra_headers.pop('thread_ids')
         extra_headers['References'] = ' '.join('<{}>'.format(t) for t in thread_ids)
 
     try:
         mailer = Mailer(**email_conf)
 
         message = Message(subject=subject,
                           sender=email_conf['default_sender'],
                           recipients=recipients,
                           body=body, html=html_body,
                           extra_headers=extra_headers)
         mailer.send_immediately(message)
 
     except Exception:
         log.exception('Mail sending failed')
         return False
     return True
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def create_repo(form_data, cur_user):
     from rhodecode.model.repo import RepoModel
     from rhodecode.model.user import UserModel
     from rhodecode.model.scm import ScmModel
     from rhodecode.model.settings import SettingsModel
 
     log = get_logger(create_repo)
 
     cur_user = UserModel()._get_user(cur_user)
     owner = cur_user
 
     repo_name = form_data['repo_name']
     repo_name_full = form_data['repo_name_full']
     repo_type = form_data['repo_type']
     description = form_data['repo_description']
     private = form_data['repo_private']
     clone_uri = form_data.get('clone_uri')
     repo_group = safe_int(form_data['repo_group'])
     copy_fork_permissions = form_data.get('copy_permissions')
     copy_group_permissions = form_data.get('repo_copy_permissions')
     fork_of = form_data.get('fork_parent_id')
     state = form_data.get('repo_state', Repository.STATE_PENDING)
 
     # repo creation defaults, private and repo_type are filled in form
     defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
     enable_statistics = form_data.get(
         'enable_statistics', defs.get('repo_enable_statistics'))
     enable_locking = form_data.get(
         'enable_locking', defs.get('repo_enable_locking'))
     enable_downloads = form_data.get(
         'enable_downloads', defs.get('repo_enable_downloads'))
 
     # set landing rev based on default branches for SCM
     landing_ref, _label = ScmModel.backend_landing_ref(repo_type)
 
     try:
         RepoModel()._create_repo(
             repo_name=repo_name_full,
             repo_type=repo_type,
             description=description,
             owner=owner,
             private=private,
             clone_uri=clone_uri,
             repo_group=repo_group,
             landing_rev=landing_ref,
             fork_of=fork_of,
             copy_fork_permissions=copy_fork_permissions,
             copy_group_permissions=copy_group_permissions,
             enable_statistics=enable_statistics,
             enable_locking=enable_locking,
             enable_downloads=enable_downloads,
             state=state
         )
         Session().commit()
 
         # now create this repo on Filesystem
         RepoModel()._create_filesystem_repo(
             repo_name=repo_name,
             repo_type=repo_type,
             repo_group=RepoModel()._get_repo_group(repo_group),
             clone_uri=clone_uri,
         )
         repo = Repository.get_by_repo_name(repo_name_full)
         hooks_base.create_repository(created_by=owner.username, **repo.get_dict())
 
         # update repo commit caches initially
         repo.update_commit_cache()
 
         # set new created state
         repo.set_state(Repository.STATE_CREATED)
         repo_id = repo.repo_id
         repo_data = repo.get_api_data()
 
         audit_logger.store(
             'repo.create', action_data={'data': repo_data},
             user=cur_user,
             repo=audit_logger.RepoWrap(repo_name=repo_name, repo_id=repo_id))
 
         Session().commit()
 
         PermissionModel().trigger_permission_flush()
 
     except Exception as e:
         log.warning('Exception occurred when creating repository, '
                     'doing cleanup...', exc_info=True)
         if isinstance(e, IntegrityError):
             Session().rollback()
 
         # rollback things manually !
         repo = Repository.get_by_repo_name(repo_name_full)
         if repo:
             Repository.delete(repo.repo_id)
             Session().commit()
             RepoModel()._delete_filesystem_repo(repo)
         log.info('Cleanup of repo %s finished', repo_name_full)
         raise
 
     return True
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def create_repo_fork(form_data, cur_user):
     """
     Creates a fork of repository using internal VCS methods
     """
     from rhodecode.model.repo import RepoModel
     from rhodecode.model.user import UserModel
 
     log = get_logger(create_repo_fork)
 
     cur_user = UserModel()._get_user(cur_user)
     owner = cur_user
 
     repo_name = form_data['repo_name']  # fork in this case
     repo_name_full = form_data['repo_name_full']
     repo_type = form_data['repo_type']
     description = form_data['description']
     private = form_data['private']
     clone_uri = form_data.get('clone_uri')
     repo_group = safe_int(form_data['repo_group'])
     landing_ref = form_data['landing_rev']
     copy_fork_permissions = form_data.get('copy_permissions')
     fork_id = safe_int(form_data.get('fork_parent_id'))
 
     try:
         fork_of = RepoModel()._get_repo(fork_id)
         RepoModel()._create_repo(
             repo_name=repo_name_full,
             repo_type=repo_type,
             description=description,
             owner=owner,
             private=private,
             clone_uri=clone_uri,
             repo_group=repo_group,
             landing_rev=landing_ref,
             fork_of=fork_of,
             copy_fork_permissions=copy_fork_permissions
         )
 
         Session().commit()
 
         base_path = Repository.base_path()
         source_repo_path = os.path.join(base_path, fork_of.repo_name)
 
         # now create this repo on Filesystem
         RepoModel()._create_filesystem_repo(
             repo_name=repo_name,
             repo_type=repo_type,
             repo_group=RepoModel()._get_repo_group(repo_group),
             clone_uri=source_repo_path,
         )
         repo = Repository.get_by_repo_name(repo_name_full)
         hooks_base.create_repository(created_by=owner.username, **repo.get_dict())
 
         # update repo commit caches initially
         config = repo._config
         config.set('extensions', 'largefiles', '')
         repo.update_commit_cache(config=config)
 
         # set new created state
         repo.set_state(Repository.STATE_CREATED)
 
         repo_id = repo.repo_id
         repo_data = repo.get_api_data()
         audit_logger.store(
             'repo.fork', action_data={'data': repo_data},
             user=cur_user,
             repo=audit_logger.RepoWrap(repo_name=repo_name, repo_id=repo_id))
 
         Session().commit()
     except Exception as e:
         log.warning('Exception occurred when forking repository, '
                     'doing cleanup...', exc_info=True)
         if isinstance(e, IntegrityError):
             Session().rollback()
 
         # rollback things manually !
         repo = Repository.get_by_repo_name(repo_name_full)
         if repo:
             Repository.delete(repo.repo_id)
             Session().commit()
             RepoModel()._delete_filesystem_repo(repo)
         log.info('Cleanup of repo %s finished', repo_name_full)
         raise
 
     return True
 
 
 @async_task(ignore_result=True)
 def repo_maintenance(repoid):
     from rhodecode.lib import repo_maintenance as repo_maintenance_lib
     log = get_logger(repo_maintenance)
     repo = Repository.get_by_id_or_repo_name(repoid)
     if repo:
         maintenance = repo_maintenance_lib.RepoMaintenance()
         tasks = maintenance.get_tasks_for_repo(repo)
         log.debug('Executing %s tasks on repo `%s`', tasks, repoid)
         executed_types = maintenance.execute(repo)
         log.debug('Got execution results %s', executed_types)
     else:
         log.debug('Repo `%s` not found or without a clone_url', repoid)
 
 
 @async_task(ignore_result=True)
 def check_for_update(send_email_notification=True, email_recipients=None):
     from rhodecode.model.update import UpdateModel
     from rhodecode.model.notification import EmailNotificationModel
 
     log = get_logger(check_for_update)
     update_url = UpdateModel().get_update_url()
     cur_ver = rhodecode.__version__
 
     try:
         data = UpdateModel().get_update_data(update_url)
 
         current_ver = UpdateModel().get_stored_version(fallback=cur_ver)
         latest_ver = data['versions'][0]['version']
         UpdateModel().store_version(latest_ver)
 
         if send_email_notification:
             log.debug('Send email notification is enabled. '
                       'Current RhodeCode version: %s, latest known: %s', current_ver, latest_ver)
             if UpdateModel().is_outdated(current_ver, latest_ver):
 
                 email_kwargs = {
                     'current_ver': current_ver,
                     'latest_ver': latest_ver,
                 }
 
                 (subject, email_body, email_body_plaintext) = EmailNotificationModel().render_email(
                     EmailNotificationModel.TYPE_UPDATE_AVAILABLE, **email_kwargs)
 
                 email_recipients = aslist(email_recipients, sep=',') or \
                                    [user.email for user in User.get_all_super_admins()]
                 run_task(send_email, email_recipients, subject,
                          email_body_plaintext, email_body)
 
     except Exception:
         pass
 
 
 @async_task(ignore_result=False)
 def beat_check(*args, **kwargs):
     log = get_logger(beat_check)
     log.info('%r: Got args: %r and kwargs %r', beat_check, args, kwargs)
     return time.time()
 
 
-@async_task(ignore_result=True)
-def sync_last_update(*args, **kwargs):
-
+def sync_last_update_for_objects(*args, **kwargs):
     skip_repos = kwargs.get('skip_repos')
     if not skip_repos:
         repos = Repository.query() \
             .order_by(Repository.group_id.asc())
 
         for repo in repos:
             repo.update_commit_cache()
 
     skip_groups = kwargs.get('skip_groups')
     if not skip_groups:
         repo_groups = RepoGroup.query() \
             .filter(RepoGroup.group_parent_id == None)
 
         for root_gr in repo_groups:
             for repo_gr in reversed(root_gr.recursive_groups()):
                 repo_gr.update_commit_cache()
+
+
+@async_task(ignore_result=True)
+def sync_last_update(*args, **kwargs):
+    sync_last_update_for_objects(*args, **kwargs)

rhodecode/lib/middleware/vcs.py

@@ -1,283 +1,284 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import gzip
 import shutil
 import logging
 import tempfile
 import urlparse
 
 from webob.exc import HTTPNotFound
 
 import rhodecode
 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
 from rhodecode.lib.middleware.simplegit import SimpleGit, GIT_PROTO_PAT
 from rhodecode.lib.middleware.simplehg import SimpleHg
 from rhodecode.lib.middleware.simplesvn import SimpleSvn
 from rhodecode.model.settings import VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
 VCS_TYPE_KEY = '_rc_vcs_type'
 VCS_TYPE_SKIP = '_rc_vcs_skip'
 
 
 def is_git(environ):
     """
     Returns True if requests should be handled by GIT wsgi middleware
     """
     is_git_path = GIT_PROTO_PAT.match(environ['PATH_INFO'])
     log.debug(
         'request path: `%s` detected as GIT PROTOCOL %s', environ['PATH_INFO'],
         is_git_path is not None)
 
     return is_git_path
 
 
 def is_hg(environ):
     """
     Returns True if requests target is mercurial server - header
     ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
     """
     is_hg_path = False
 
     http_accept = environ.get('HTTP_ACCEPT')
 
     if http_accept and http_accept.startswith('application/mercurial'):
         query = urlparse.parse_qs(environ['QUERY_STRING'])
         if 'cmd' in query:
             is_hg_path = True
 
     log.debug(
         'request path: `%s` detected as HG PROTOCOL %s', environ['PATH_INFO'],
         is_hg_path)
 
     return is_hg_path
 
 
 def is_svn(environ):
     """
     Returns True if requests target is Subversion server
     """
 
     http_dav = environ.get('HTTP_DAV', '')
     magic_path_segment = rhodecode.CONFIG.get(
         'rhodecode_subversion_magic_path', '/!svn')
     is_svn_path = (
         'subversion' in http_dav or
         magic_path_segment in environ['PATH_INFO']
         or environ['REQUEST_METHOD'] in ['PROPFIND', 'PROPPATCH']
     )
     log.debug(
         'request path: `%s` detected as SVN PROTOCOL %s', environ['PATH_INFO'],
         is_svn_path)
 
     return is_svn_path
 
 
 class GunzipMiddleware(object):
     """
     WSGI middleware that unzips gzip-encoded requests before
     passing on to the underlying application.
     """
 
     def __init__(self, application):
         self.app = application
 
     def __call__(self, environ, start_response):
         accepts_encoding_header = environ.get('HTTP_CONTENT_ENCODING', b'')
 
         if b'gzip' in accepts_encoding_header:
             log.debug('gzip detected, now running gunzip wrapper')
             wsgi_input = environ['wsgi.input']
 
             if not hasattr(environ['wsgi.input'], 'seek'):
                 # The gzip implementation in the standard library of Python 2.x
                 # requires the '.seek()' and '.tell()' methods to be available
                 # on the input stream.  Read the data into a temporary file to
                 # work around this limitation.
 
                 wsgi_input = tempfile.SpooledTemporaryFile(64 * 1024 * 1024)
                 shutil.copyfileobj(environ['wsgi.input'], wsgi_input)
                 wsgi_input.seek(0)
 
             environ['wsgi.input'] = gzip.GzipFile(fileobj=wsgi_input, mode='r')
             # since we "Ungzipped" the content we say now it's no longer gzip
             # content encoding
             del environ['HTTP_CONTENT_ENCODING']
 
             # content length has changes ? or i'm not sure
             if 'CONTENT_LENGTH' in environ:
                 del environ['CONTENT_LENGTH']
         else:
             log.debug('content not gzipped, gzipMiddleware passing '
                       'request further')
         return self.app(environ, start_response)
 
 
 def is_vcs_call(environ):
     if VCS_TYPE_KEY in environ:
         raw_type = environ[VCS_TYPE_KEY]
         return raw_type and raw_type != VCS_TYPE_SKIP
     return False
 
 
 def get_path_elem(route_path):
     if not route_path:
         return None
 
     cleaned_route_path = route_path.lstrip('/')
     if cleaned_route_path:
         cleaned_route_path_elems = cleaned_route_path.split('/')
         if cleaned_route_path_elems:
             return cleaned_route_path_elems[0]
     return None
 
 
 def detect_vcs_request(environ, backends):
     checks = {
         'hg': (is_hg, SimpleHg),
         'git': (is_git, SimpleGit),
         'svn': (is_svn, SimpleSvn),
     }
     handler = None
     # List of path views first chunk we don't do any checks
     white_list = [
         # e.g /_file_store/download
         '_file_store',
 
         # static files no detection
         '_static',
 
-        # skip ops ping
+        # skip ops ping, status
         '_admin/ops/ping',
+        '_admin/ops/status',
 
         # full channelstream connect should be VCS skipped
         '_admin/channelstream/connect',
     ]
 
     path_info = environ['PATH_INFO']
 
     path_elem = get_path_elem(path_info)
 
     if path_elem in white_list:
         log.debug('path `%s` in whitelist, skipping...', path_info)
         return handler
 
     path_url = path_info.lstrip('/')
     if path_url in white_list:
         log.debug('full url path `%s` in whitelist, skipping...', path_url)
         return handler
 
     if VCS_TYPE_KEY in environ:
         raw_type = environ[VCS_TYPE_KEY]
         if raw_type == VCS_TYPE_SKIP:
             log.debug('got `skip` marker for vcs detection, skipping...')
             return handler
 
         _check, handler = checks.get(raw_type) or [None, None]
         if handler:
             log.debug('got handler:%s from environ', handler)
 
     if not handler:
         log.debug('request start: checking if request for `%s` is of VCS type in order: %s', path_elem, backends)
         for vcs_type in backends:
             vcs_check, _handler = checks[vcs_type]
             if vcs_check(environ):
                 log.debug('vcs handler found %s', _handler)
                 handler = _handler
                 break
 
     return handler
 
 
 class VCSMiddleware(object):
 
     def __init__(self, app, registry, config, appenlight_client):
         self.application = app
         self.registry = registry
         self.config = config
         self.appenlight_client = appenlight_client
         self.use_gzip = True
         # order in which we check the middlewares, based on vcs.backends config
         self.check_middlewares = config['vcs.backends']
 
     def vcs_config(self, repo_name=None):
         """
         returns serialized VcsSettings
         """
         try:
             return VcsSettingsModel(
                 repo=repo_name).get_ui_settings_as_config_obj()
         except Exception:
             pass
 
     def wrap_in_gzip_if_enabled(self, app, config):
         if self.use_gzip:
             app = GunzipMiddleware(app)
         return app
 
     def _get_handler_app(self, environ):
         app = None
         log.debug('VCSMiddleware: detecting vcs type.')
         handler = detect_vcs_request(environ, self.check_middlewares)
         if handler:
             app = handler(self.config, self.registry)
 
         return app
 
     def __call__(self, environ, start_response):
         # check if we handle one of interesting protocols, optionally extract
         # specific vcsSettings and allow changes of how things are wrapped
         vcs_handler = self._get_handler_app(environ)
         if vcs_handler:
             # translate the _REPO_ID into real repo NAME for usage
             # in middleware
             environ['PATH_INFO'] = vcs_handler._get_by_id(environ['PATH_INFO'])
 
             # Set acl, url and vcs repo names.
             vcs_handler.set_repo_names(environ)
 
             # register repo config back to the handler
             vcs_conf = self.vcs_config(vcs_handler.acl_repo_name)
             # maybe damaged/non existent settings. We still want to
             # pass that point to validate on is_valid_and_existing_repo
             # and return proper HTTP Code back to client
             if vcs_conf:
                 vcs_handler.repo_vcs_config = vcs_conf
 
             # check for type, presence in database and on filesystem
             if not vcs_handler.is_valid_and_existing_repo(
                     vcs_handler.acl_repo_name,
                     vcs_handler.base_path,
                     vcs_handler.SCM):
                 return HTTPNotFound()(environ, start_response)
 
             environ['REPO_NAME'] = vcs_handler.url_repo_name
 
             # Wrap handler in middlewares if they are enabled.
             vcs_handler = self.wrap_in_gzip_if_enabled(
                 vcs_handler, self.config)
             vcs_handler, _ = wrap_in_appenlight_if_enabled(
                 vcs_handler, self.config, self.appenlight_client)
 
             return vcs_handler(environ, start_response)
 
         return self.application(environ, start_response)

rhodecode/lib/rc_cache/backends.py

@@ -1,354 +1,363 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2015-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import time
 import errno
 import logging
 
 import msgpack
 import gevent
 import redis
 
 from dogpile.cache.api import CachedValue
 from dogpile.cache.backends import memory as memory_backend
 from dogpile.cache.backends import file as file_backend
 from dogpile.cache.backends import redis as redis_backend
 from dogpile.cache.backends.file import NO_VALUE, compat, FileLock
 from dogpile.cache.util import memoized_property
 
 from pyramid.settings import asbool
 
 from rhodecode.lib.memory_lru_dict import LRUDict, LRUDictDebug
 
 
 _default_max_size = 1024
 
 log = logging.getLogger(__name__)
 
 
 class LRUMemoryBackend(memory_backend.MemoryBackend):
     key_prefix = 'lru_mem_backend'
     pickle_values = False
 
     def __init__(self, arguments):
         max_size = arguments.pop('max_size', _default_max_size)
 
         LRUDictClass = LRUDict
         if arguments.pop('log_key_count', None):
             LRUDictClass = LRUDictDebug
 
         arguments['cache_dict'] = LRUDictClass(max_size)
         super(LRUMemoryBackend, self).__init__(arguments)
 
     def delete(self, key):
         try:
             del self._cache[key]
         except KeyError:
             # we don't care if key isn't there at deletion
             pass
 
     def delete_multi(self, keys):
         for key in keys:
             self.delete(key)
 
 
 class PickleSerializer(object):
 
     def _dumps(self, value, safe=False):
         try:
             return compat.pickle.dumps(value)
         except Exception:
             if safe:
                 return NO_VALUE
             else:
                 raise
 
     def _loads(self, value, safe=True):
         try:
             return compat.pickle.loads(value)
         except Exception:
             if safe:
                 return NO_VALUE
             else:
                 raise
 
 
 class MsgPackSerializer(object):
 
     def _dumps(self, value, safe=False):
         try:
             return msgpack.packb(value)
         except Exception:
             if safe:
                 return NO_VALUE
             else:
                 raise
 
     def _loads(self, value, safe=True):
         """
         pickle maintained the `CachedValue` wrapper of the tuple
         msgpack does not, so it must be added back in.
        """
         try:
             value = msgpack.unpackb(value, use_list=False)
             return CachedValue(*value)
         except Exception:
             if safe:
                 return NO_VALUE
             else:
                 raise
 
 
 import fcntl
 flock_org = fcntl.flock
 
 
 class CustomLockFactory(FileLock):
 
     @memoized_property
     def _module(self):
 
         def gevent_flock(fd, operation):
             """
             Gevent compatible flock
             """
             # set non-blocking, this will cause an exception if we cannot acquire a lock
             operation |= fcntl.LOCK_NB
             start_lock_time = time.time()
             timeout = 60 * 15  # 15min
             while True:
                 try:
                     flock_org(fd, operation)
                     # lock has been acquired
                     break
                 except (OSError, IOError) as e:
                     # raise on other errors than Resource temporarily unavailable
                     if e.errno != errno.EAGAIN:
                         raise
                     elif (time.time() - start_lock_time) > timeout:
                         # waited to much time on a lock, better fail than loop for ever
                         log.error('Failed to acquire lock on `%s` after waiting %ss',
                                   self.filename, timeout)
                         raise
                     wait_timeout = 0.03
                     log.debug('Failed to acquire lock on `%s`, retry in %ss',
                               self.filename, wait_timeout)
                     gevent.sleep(wait_timeout)
 
         fcntl.flock = gevent_flock
         return fcntl
 
 
 class FileNamespaceBackend(PickleSerializer, file_backend.DBMBackend):
     key_prefix = 'file_backend'
 
     def __init__(self, arguments):
         arguments['lock_factory'] = CustomLockFactory
         db_file = arguments.get('filename')
 
         log.debug('initialing %s DB in %s', self.__class__.__name__, db_file)
         try:
             super(FileNamespaceBackend, self).__init__(arguments)
         except Exception:
             log.error('Failed to initialize db at: %s', db_file)
             raise
 
     def __repr__(self):
         return '{} `{}`'.format(self.__class__, self.filename)
 
     def list_keys(self, prefix=''):
         prefix = '{}:{}'.format(self.key_prefix, prefix)
 
         def cond(v):
             if not prefix:
                 return True
 
             if v.startswith(prefix):
                 return True
             return False
 
         with self._dbm_file(True) as dbm:
             try:
                 return filter(cond, dbm.keys())
             except Exception:
                 log.error('Failed to fetch DBM keys from DB: %s', self.get_store())
                 raise
 
     def get_store(self):
         return self.filename
 
     def _dbm_get(self, key):
         with self._dbm_file(False) as dbm:
             if hasattr(dbm, 'get'):
                 value = dbm.get(key, NO_VALUE)
             else:
                 # gdbm objects lack a .get method
                 try:
                     value = dbm[key]
                 except KeyError:
                     value = NO_VALUE
             if value is not NO_VALUE:
                 value = self._loads(value)
             return value
 
     def get(self, key):
         try:
             return self._dbm_get(key)
         except Exception:
             log.error('Failed to fetch DBM key %s from DB: %s', key, self.get_store())
             raise
 
     def set(self, key, value):
         with self._dbm_file(True) as dbm:
             dbm[key] = self._dumps(value)
 
     def set_multi(self, mapping):
         with self._dbm_file(True) as dbm:
             for key, value in mapping.items():
                 dbm[key] = self._dumps(value)
 
 
 class BaseRedisBackend(redis_backend.RedisBackend):
     key_prefix = ''
 
     def __init__(self, arguments):
         super(BaseRedisBackend, self).__init__(arguments)
         self._lock_timeout = self.lock_timeout
         self._lock_auto_renewal = asbool(arguments.pop("lock_auto_renewal", True))
 
         if self._lock_auto_renewal and not self._lock_timeout:
             # set default timeout for auto_renewal
             self._lock_timeout = 30
 
     def _create_client(self):
         args = {}
 
         if self.url is not None:
             args.update(url=self.url)
 
         else:
             args.update(
                 host=self.host, password=self.password,
                 port=self.port, db=self.db
             )
 
         connection_pool = redis.ConnectionPool(**args)
 
         return redis.StrictRedis(connection_pool=connection_pool)
 
     def list_keys(self, prefix=''):
         prefix = '{}:{}*'.format(self.key_prefix, prefix)
         return self.client.keys(prefix)
 
     def get_store(self):
         return self.client.connection_pool
 
     def get(self, key):
         value = self.client.get(key)
         if value is None:
             return NO_VALUE
         return self._loads(value)
 
     def get_multi(self, keys):
         if not keys:
             return []
         values = self.client.mget(keys)
         loads = self._loads
         return [
             loads(v) if v is not None else NO_VALUE
             for v in values]
 
     def set(self, key, value):
         if self.redis_expiration_time:
             self.client.setex(key, self.redis_expiration_time,
                               self._dumps(value))
         else:
             self.client.set(key, self._dumps(value))
 
     def set_multi(self, mapping):
         dumps = self._dumps
         mapping = dict(
             (k, dumps(v))
             for k, v in mapping.items()
         )
 
         if not self.redis_expiration_time:
             self.client.mset(mapping)
         else:
             pipe = self.client.pipeline()
             for key, value in mapping.items():
                 pipe.setex(key, self.redis_expiration_time, value)
             pipe.execute()
 
     def get_mutex(self, key):
         if self.distributed_lock:
             lock_key = redis_backend.u('_lock_{0}').format(key)
-            log.debug('Trying to acquire Redis lock for key %s', lock_key)
             return get_mutex_lock(self.client, lock_key, self._lock_timeout,
                                   auto_renewal=self._lock_auto_renewal)
         else:
             return None
 
 
 class RedisPickleBackend(PickleSerializer, BaseRedisBackend):
     key_prefix = 'redis_pickle_backend'
     pass
 
 
 class RedisMsgPackBackend(MsgPackSerializer, BaseRedisBackend):
     key_prefix = 'redis_msgpack_backend'
     pass
 
 
 def get_mutex_lock(client, lock_key, lock_timeout, auto_renewal=False):
     import redis_lock
 
     class _RedisLockWrapper(object):
         """LockWrapper for redis_lock"""
 
         @classmethod
         def get_lock(cls):
             return redis_lock.Lock(
                 redis_client=client,
                 name=lock_key,
                 expire=lock_timeout,
                 auto_renewal=auto_renewal,
                 strict=True,
             )
 
+        def __repr__(self):
+            return "{}:{}".format(self.__class__.__name__, lock_key)
+
+        def __str__(self):
+            return "{}:{}".format(self.__class__.__name__, lock_key)
+
         def __init__(self):
             self.lock = self.get_lock()
+            self.lock_key = lock_key
 
         def acquire(self, wait=True):
+            log.debug('Trying to acquire Redis lock for key %s', self.lock_key)
             try:
-                return self.lock.acquire(wait)
+                acquired = self.lock.acquire(wait)
+                log.debug('Got lock for key %s, %s', self.lock_key, acquired)
+                return acquired
             except redis_lock.AlreadyAcquired:
                 return False
             except redis_lock.AlreadyStarted:
                 # refresh thread exists, but it also means we acquired the lock
                 return True
 
         def release(self):
             try:
                 self.lock.release()
             except redis_lock.NotAcquired:
                 pass
 
     return _RedisLockWrapper()

rhodecode/lib/rc_cache/utils.py

@@ -1,418 +1,422 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2015-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import os
 import time
 import logging
 import functools
 import threading
 
 from dogpile.cache import CacheRegion
 from dogpile.cache.util import compat
 
 import rhodecode
 from rhodecode.lib.utils import safe_str, sha1
 from rhodecode.lib.utils2 import safe_unicode, str2bool
 from rhodecode.model.db import Session, CacheKey, IntegrityError
 
 from rhodecode.lib.rc_cache import cache_key_meta
 from rhodecode.lib.rc_cache import region_meta
 
 log = logging.getLogger(__name__)
 
 
 def isCython(func):
     """
     Private helper that checks if a function is a cython function.
     """
     return func.__class__.__name__ == 'cython_function_or_method'
 
 
 class RhodeCodeCacheRegion(CacheRegion):
 
     def conditional_cache_on_arguments(
             self, namespace=None,
             expiration_time=None,
             should_cache_fn=None,
             to_str=compat.string_type,
             function_key_generator=None,
             condition=True):
         """
         Custom conditional decorator, that will not touch any dogpile internals if
         condition isn't meet. This works a bit different than should_cache_fn
         And it's faster in cases we don't ever want to compute cached values
         """
         expiration_time_is_callable = compat.callable(expiration_time)
 
         if function_key_generator is None:
             function_key_generator = self.function_key_generator
 
         # workaround for py2 and cython problems, this block should be removed
         # once we've migrated to py3
         if 'cython' == 'cython':
             def decorator(fn):
                 if to_str is compat.string_type:
                     # backwards compatible
                     key_generator = function_key_generator(namespace, fn)
                 else:
                     key_generator = function_key_generator(namespace, fn, to_str=to_str)
 
                 @functools.wraps(fn)
                 def decorate(*arg, **kw):
                     key = key_generator(*arg, **kw)
 
                     @functools.wraps(fn)
                     def creator():
                         return fn(*arg, **kw)
 
                     if not condition:
                         return creator()
 
                     timeout = expiration_time() if expiration_time_is_callable \
                         else expiration_time
 
                     return self.get_or_create(key, creator, timeout, should_cache_fn)
 
                 def invalidate(*arg, **kw):
                     key = key_generator(*arg, **kw)
                     self.delete(key)
 
                 def set_(value, *arg, **kw):
                     key = key_generator(*arg, **kw)
                     self.set(key, value)
 
                 def get(*arg, **kw):
                     key = key_generator(*arg, **kw)
                     return self.get(key)
 
                 def refresh(*arg, **kw):
                     key = key_generator(*arg, **kw)
                     value = fn(*arg, **kw)
                     self.set(key, value)
                     return value
 
                 decorate.set = set_
                 decorate.invalidate = invalidate
                 decorate.refresh = refresh
                 decorate.get = get
                 decorate.original = fn
                 decorate.key_generator = key_generator
                 decorate.__wrapped__ = fn
 
                 return decorate
             return decorator
 
         def get_or_create_for_user_func(key_generator, user_func, *arg, **kw):
 
             if not condition:
                 log.debug('Calling un-cached func:%s', user_func.func_name)
-                return user_func(*arg, **kw)
+                start = time.time()
+                result = user_func(*arg, **kw)
+                total = time.time() - start
+                log.debug('un-cached func:%s took %.4fs', user_func.func_name, total)
+                return result
 
             key = key_generator(*arg, **kw)
 
             timeout = expiration_time() if expiration_time_is_callable \
                 else expiration_time
 
             log.debug('Calling cached fn:%s', user_func.func_name)
             return self.get_or_create(key, user_func, timeout, should_cache_fn, (arg, kw))
 
         def cache_decorator(user_func):
             if to_str is compat.string_type:
                 # backwards compatible
                 key_generator = function_key_generator(namespace, user_func)
             else:
                 key_generator = function_key_generator(namespace, user_func, to_str=to_str)
 
             def refresh(*arg, **kw):
                 """
                 Like invalidate, but regenerates the value instead
                 """
                 key = key_generator(*arg, **kw)
                 value = user_func(*arg, **kw)
                 self.set(key, value)
                 return value
 
             def invalidate(*arg, **kw):
                 key = key_generator(*arg, **kw)
                 self.delete(key)
 
             def set_(value, *arg, **kw):
                 key = key_generator(*arg, **kw)
                 self.set(key, value)
 
             def get(*arg, **kw):
                 key = key_generator(*arg, **kw)
                 return self.get(key)
 
             user_func.set = set_
             user_func.invalidate = invalidate
             user_func.get = get
             user_func.refresh = refresh
             user_func.key_generator = key_generator
             user_func.original = user_func
 
             # Use `decorate` to preserve the signature of :param:`user_func`.
             return decorator.decorate(user_func, functools.partial(
                 get_or_create_for_user_func, key_generator))
 
         return cache_decorator
 
 
 def make_region(*arg, **kw):
     return RhodeCodeCacheRegion(*arg, **kw)
 
 
 def get_default_cache_settings(settings, prefixes=None):
     prefixes = prefixes or []
     cache_settings = {}
     for key in settings.keys():
         for prefix in prefixes:
             if key.startswith(prefix):
                 name = key.split(prefix)[1].strip()
                 val = settings[key]
                 if isinstance(val, compat.string_types):
                     val = val.strip()
                 cache_settings[name] = val
     return cache_settings
 
 
 def compute_key_from_params(*args):
     """
     Helper to compute key from given params to be used in cache manager
     """
     return sha1("_".join(map(safe_str, args)))
 
 
 def backend_key_generator(backend):
     """
     Special wrapper that also sends over the backend to the key generator
     """
     def wrapper(namespace, fn):
         return key_generator(backend, namespace, fn)
     return wrapper
 
 
 def key_generator(backend, namespace, fn):
     fname = fn.__name__
 
     def generate_key(*args):
         backend_prefix = getattr(backend, 'key_prefix', None) or 'backend_prefix'
         namespace_pref = namespace or 'default_namespace'
         arg_key = compute_key_from_params(*args)
         final_key = "{}:{}:{}_{}".format(backend_prefix, namespace_pref, fname, arg_key)
 
         return final_key
 
     return generate_key
 
 
 def get_or_create_region(region_name, region_namespace=None):
     from rhodecode.lib.rc_cache.backends import FileNamespaceBackend
     region_obj = region_meta.dogpile_cache_regions.get(region_name)
     if not region_obj:
         raise EnvironmentError(
             'Region `{}` not in configured: {}.'.format(
                 region_name, region_meta.dogpile_cache_regions.keys()))
 
     region_uid_name = '{}:{}'.format(region_name, region_namespace)
     if isinstance(region_obj.actual_backend, FileNamespaceBackend):
         region_exist = region_meta.dogpile_cache_regions.get(region_namespace)
         if region_exist:
             log.debug('Using already configured region: %s', region_namespace)
             return region_exist
         cache_dir = region_meta.dogpile_config_defaults['cache_dir']
         expiration_time = region_obj.expiration_time
 
         if not os.path.isdir(cache_dir):
             os.makedirs(cache_dir)
         new_region = make_region(
             name=region_uid_name,
             function_key_generator=backend_key_generator(region_obj.actual_backend)
         )
         namespace_filename = os.path.join(
             cache_dir, "{}.cache.dbm".format(region_namespace))
         # special type that allows 1db per namespace
         new_region.configure(
             backend='dogpile.cache.rc.file_namespace',
             expiration_time=expiration_time,
             arguments={"filename": namespace_filename}
         )
 
         # create and save in region caches
         log.debug('configuring new region: %s', region_uid_name)
         region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region
 
     return region_obj
 
 
 def clear_cache_namespace(cache_region, cache_namespace_uid, invalidate=False):
     region = get_or_create_region(cache_region, cache_namespace_uid)
     cache_keys = region.backend.list_keys(prefix=cache_namespace_uid)
     num_delete_keys = len(cache_keys)
     if invalidate:
         region.invalidate(hard=False)
     else:
         if num_delete_keys:
             region.delete_multi(cache_keys)
     return num_delete_keys
 
 
 class ActiveRegionCache(object):
     def __init__(self, context, cache_data):
         self.context = context
         self.cache_data = cache_data
 
     def should_invalidate(self):
         return False
 
 
 class FreshRegionCache(object):
     def __init__(self, context, cache_data):
         self.context = context
         self.cache_data = cache_data
 
     def should_invalidate(self):
         return True
 
 
 class InvalidationContext(object):
     """
     usage::
 
         from rhodecode.lib import rc_cache
 
         cache_namespace_uid = CacheKey.SOME_NAMESPACE.format(1)
         region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=True)
         def heavy_compute(cache_name, param1, param2):
             print('COMPUTE {}, {}, {}'.format(cache_name, param1, param2))
 
         # invalidation namespace is shared namespace key for all process caches
         # we use it to send a global signal
         invalidation_namespace = 'repo_cache:1'
 
         inv_context_manager = rc_cache.InvalidationContext(
             uid=cache_namespace_uid, invalidation_namespace=invalidation_namespace)
         with inv_context_manager as invalidation_context:
             args = ('one', 'two')
             # re-compute and store cache if we get invalidate signal
             if invalidation_context.should_invalidate():
                 result = heavy_compute.refresh(*args)
             else:
                 result = heavy_compute(*args)
 
             compute_time = inv_context_manager.compute_time
             log.debug('result computed in %.4fs', compute_time)
 
         # To send global invalidation signal, simply run
         CacheKey.set_invalidate(invalidation_namespace)
 
     """
 
     def __repr__(self):
         return '<InvalidationContext:{}[{}]>'.format(
             safe_str(self.cache_key), safe_str(self.uid))
 
     def __init__(self, uid, invalidation_namespace='',
                  raise_exception=False, thread_scoped=None):
         self.uid = uid
         self.invalidation_namespace = invalidation_namespace
         self.raise_exception = raise_exception
         self.proc_id = safe_unicode(rhodecode.CONFIG.get('instance_id') or 'DEFAULT')
         self.thread_id = 'global'
 
         if thread_scoped is None:
             # if we set "default" we can override this via .ini settings
             thread_scoped = str2bool(rhodecode.CONFIG.get('cache_thread_scoped'))
 
         # Append the thread id to the cache key if this invalidation context
         # should be scoped to the current thread.
         if thread_scoped is True:
             self.thread_id = threading.current_thread().ident
 
         self.cache_key = compute_key_from_params(uid)
         self.cache_key = 'proc:{}|thread:{}|params:{}'.format(
             self.proc_id, self.thread_id, self.cache_key)
         self.compute_time = 0
 
     def get_or_create_cache_obj(self, cache_type, invalidation_namespace=''):
         invalidation_namespace = invalidation_namespace or self.invalidation_namespace
         # fetch all cache keys for this namespace and convert them to a map to find if we
         # have specific cache_key object registered. We do this because we want to have
         # all consistent cache_state_uid for newly registered objects
         cache_obj_map = CacheKey.get_namespace_map(invalidation_namespace)
         cache_obj = cache_obj_map.get(self.cache_key)
         log.debug('Fetched cache obj %s using %s cache key.', cache_obj, self.cache_key)
         if not cache_obj:
             new_cache_args = invalidation_namespace
             first_cache_obj = next(cache_obj_map.itervalues()) if cache_obj_map else None
             cache_state_uid = None
             if first_cache_obj:
                 cache_state_uid = first_cache_obj.cache_state_uid
             cache_obj = CacheKey(self.cache_key, cache_args=new_cache_args,
                                  cache_state_uid=cache_state_uid)
             cache_key_meta.cache_keys_by_pid.append(self.cache_key)
 
         return cache_obj
 
     def __enter__(self):
         """
         Test if current object is valid, and return CacheRegion function
         that does invalidation and calculation
         """
         log.debug('Entering cache invalidation check context: %s', self.invalidation_namespace)
         # register or get a new key based on uid
         self.cache_obj = self.get_or_create_cache_obj(cache_type=self.uid)
         cache_data = self.cache_obj.get_dict()
         self._start_time = time.time()
         if self.cache_obj.cache_active:
             # means our cache obj is existing and marked as it's
             # cache is not outdated, we return ActiveRegionCache
             self.skip_cache_active_change = True
 
             return ActiveRegionCache(context=self, cache_data=cache_data)
 
         # the key is either not existing or set to False, we return
         # the real invalidator which re-computes value. We additionally set
         # the flag to actually update the Database objects
         self.skip_cache_active_change = False
         return FreshRegionCache(context=self, cache_data=cache_data)
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         # save compute time
         self.compute_time = time.time() - self._start_time
 
         if self.skip_cache_active_change:
             return
 
         try:
             self.cache_obj.cache_active = True
             Session().add(self.cache_obj)
             Session().commit()
         except IntegrityError:
             # if we catch integrity error, it means we inserted this object
             # assumption is that's really an edge race-condition case and
             # it's safe is to skip it
             Session().rollback()
         except Exception:
             log.exception('Failed to commit on cache key update')
             Session().rollback()
             if self.raise_exception:
                 raise

rhodecode/lib/system_info.py

@@ -1,797 +1,853 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2017-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 import os
 import sys
 import time
 import platform
 import collections
+from functools import wraps
+
 import pkg_resources
 import logging
 import resource
 
 from pyramid.compat import configparser
 
 log = logging.getLogger(__name__)
 
 
 psutil = None
 
 try:
     # cygwin cannot have yet psutil support.
     import psutil as psutil
 except ImportError:
     pass
 
 
 _NA = 'NOT AVAILABLE'
 
 STATE_OK = 'ok'
 STATE_ERR = 'error'
 STATE_WARN = 'warning'
 
 STATE_OK_DEFAULT = {'message': '', 'type': STATE_OK}
 
 
+registered_helpers = {}
+
+
+def register_sysinfo(func):
+    """
+    @register_helper
+    def db_check():
+        pass
+
+    db_check == registered_helpers['db_check']
+    """
+    global registered_helpers
+    registered_helpers[func.__name__] = func
+
+    @wraps(func)
+    def _wrapper(*args, **kwargs):
+        return func(*args, **kwargs)
+    return _wrapper
+
+
 # HELPERS
 def percentage(part, whole):
     whole = float(whole)
     if whole > 0:
         return round(100 * float(part) / whole, 1)
     return 0.0
 
 
 def get_storage_size(storage_path):
     sizes = []
     for file_ in os.listdir(storage_path):
         storage_file = os.path.join(storage_path, file_)
         if os.path.isfile(storage_file):
             try:
                 sizes.append(os.path.getsize(storage_file))
             except OSError:
                 log.exception('Failed to get size of storage file %s', storage_file)
                 pass
 
     return sum(sizes)
 
 
 def get_resource(resource_type):
     try:
         return resource.getrlimit(resource_type)
     except Exception:
         return 'NOT_SUPPORTED'
 
 
 def get_cert_path(ini_path):
     default = '/etc/ssl/certs/ca-certificates.crt'
     control_ca_bundle = os.path.join(
         os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(ini_path)))),
         '.rccontrol-profile/etc/ca-bundle.crt')
     if os.path.isfile(control_ca_bundle):
         default = control_ca_bundle
 
     return default
 
 
 class SysInfoRes(object):
     def __init__(self, value, state=None, human_value=None):
         self.value = value
         self.state = state or STATE_OK_DEFAULT
         self.human_value = human_value or value
 
     def __json__(self):
         return {
             'value': self.value,
             'state': self.state,
             'human_value': self.human_value,
         }
 
     def get_value(self):
         return self.__json__()
 
     def __str__(self):
         return '<SysInfoRes({})>'.format(self.__json__())
 
 
 class SysInfo(object):
 
     def __init__(self, func_name, **kwargs):
         self.func_name = func_name
         self.value = _NA
         self.state = None
         self.kwargs = kwargs or {}
 
     def __call__(self):
         computed = self.compute(**self.kwargs)
         if not isinstance(computed, SysInfoRes):
             raise ValueError(
                 'computed value for {} is not instance of '
                 '{}, got {} instead'.format(
                     self.func_name, SysInfoRes, type(computed)))
         return computed.__json__()
 
     def __str__(self):
         return '<SysInfo({})>'.format(self.func_name)
 
     def compute(self, **kwargs):
         return self.func_name(**kwargs)
 
 
 # SysInfo functions
+@register_sysinfo
 def python_info():
     value = dict(version=' '.join(platform._sys_version()),
                  executable=sys.executable)
     return SysInfoRes(value=value)
 
 
+@register_sysinfo
 def py_modules():
     mods = dict([(p.project_name, {'version': p.version, 'location': p.location})
                  for p in pkg_resources.working_set])
 
     value = sorted(mods.items(), key=lambda k: k[0].lower())
     return SysInfoRes(value=value)
 
 
+@register_sysinfo
 def platform_type():
     from rhodecode.lib.utils import safe_unicode, generate_platform_uuid
 
     value = dict(
         name=safe_unicode(platform.platform()),
         uuid=generate_platform_uuid()
     )
     return SysInfoRes(value=value)
 
 
+@register_sysinfo
 def locale_info():
     import locale
 
     value = dict(
         locale_default=locale.getdefaultlocale(),
         locale_lc_all=locale.getlocale(locale.LC_ALL),
         lang_env=os.environ.get('LANG'),
         lc_all_env=os.environ.get('LC_ALL'),
         local_archive_env=os.environ.get('LOCALE_ARCHIVE'),
     )
     human_value = 'LANG: {}, locale LC_ALL: {},  Default locales: {}'.format(
         value['lang_env'], value['locale_lc_all'], value['locale_default'])
     return SysInfoRes(value=value, human_value=human_value)
 
 
+@register_sysinfo
 def ulimit_info():
     data = collections.OrderedDict([
         ('cpu time (seconds)', get_resource(resource.RLIMIT_CPU)),
         ('file size', get_resource(resource.RLIMIT_FSIZE)),
         ('stack size', get_resource(resource.RLIMIT_STACK)),
         ('core file size', get_resource(resource.RLIMIT_CORE)),
         ('address space size', get_resource(resource.RLIMIT_AS)),
         ('locked in mem size', get_resource(resource.RLIMIT_MEMLOCK)),
         ('heap size', get_resource(resource.RLIMIT_DATA)),
         ('rss size', get_resource(resource.RLIMIT_RSS)),
         ('number of processes', get_resource(resource.RLIMIT_NPROC)),
         ('open files', get_resource(resource.RLIMIT_NOFILE)),
     ])
 
     text = ', '.join('{}:{}'.format(k, v) for k, v in data.items())
 
     value = {
         'limits': data,
         'text': text,
     }
     return SysInfoRes(value=value)
 
 
+@register_sysinfo
 def uptime():
     from rhodecode.lib.helpers import age, time_to_datetime
     from rhodecode.translation import TranslationString
 
     value = dict(boot_time=0, uptime=0, text='')
     state = STATE_OK_DEFAULT
     if not psutil:
         return SysInfoRes(value=value, state=state)
 
     boot_time = psutil.boot_time()
     value['boot_time'] = boot_time
     value['uptime'] = time.time() - boot_time
 
     date_or_age = age(time_to_datetime(boot_time))
     if isinstance(date_or_age, TranslationString):
         date_or_age = date_or_age.interpolate()
 
     human_value = value.copy()
     human_value['boot_time'] = time_to_datetime(boot_time)
     human_value['uptime'] = age(time_to_datetime(boot_time), show_suffix=False)
 
     human_value['text'] = u'Server started {}'.format(date_or_age)
     return SysInfoRes(value=value, human_value=human_value)
 
 
+@register_sysinfo
 def memory():
     from rhodecode.lib.helpers import format_byte_size_binary
     value = dict(available=0, used=0, used_real=0, cached=0, percent=0,
                  percent_used=0, free=0, inactive=0, active=0, shared=0,
                  total=0, buffers=0, text='')
 
     state = STATE_OK_DEFAULT
     if not psutil:
         return SysInfoRes(value=value, state=state)
 
     value.update(dict(psutil.virtual_memory()._asdict()))
     value['used_real'] = value['total'] - value['available']
     value['percent_used'] = psutil._common.usage_percent(
         value['used_real'], value['total'], 1)
 
     human_value = value.copy()
     human_value['text'] = '%s/%s, %s%% used' % (
         format_byte_size_binary(value['used_real']),
         format_byte_size_binary(value['total']),
         value['percent_used'],)
 
     keys = value.keys()[::]
     keys.pop(keys.index('percent'))
     keys.pop(keys.index('percent_used'))
     keys.pop(keys.index('text'))
     for k in keys:
         human_value[k] = format_byte_size_binary(value[k])
 
     if state['type'] == STATE_OK and value['percent_used'] > 90:
         msg = 'Critical: your available RAM memory is very low.'
         state = {'message': msg, 'type': STATE_ERR}
 
     elif state['type'] == STATE_OK and value['percent_used'] > 70:
         msg = 'Warning: your available RAM memory is running low.'
         state = {'message': msg, 'type': STATE_WARN}
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
+@register_sysinfo
 def machine_load():
     value = {'1_min': _NA, '5_min': _NA, '15_min': _NA, 'text': ''}
     state = STATE_OK_DEFAULT
     if not psutil:
         return SysInfoRes(value=value, state=state)
 
     # load averages
     if hasattr(psutil.os, 'getloadavg'):
         value.update(dict(
             zip(['1_min', '5_min', '15_min'], psutil.os.getloadavg())))
 
     human_value = value.copy()
     human_value['text'] = '1min: {}, 5min: {}, 15min: {}'.format(
         value['1_min'], value['5_min'], value['15_min'])
 
     if state['type'] == STATE_OK and value['15_min'] > 5:
         msg = 'Warning: your machine load is very high.'
         state = {'message': msg, 'type': STATE_WARN}
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
+@register_sysinfo
 def cpu():
     value = {'cpu': 0, 'cpu_count': 0, 'cpu_usage': []}
     state = STATE_OK_DEFAULT
 
     if not psutil:
         return SysInfoRes(value=value, state=state)
 
     value['cpu'] = psutil.cpu_percent(0.5)
     value['cpu_usage'] = psutil.cpu_percent(0.5, percpu=True)
     value['cpu_count'] = psutil.cpu_count()
 
     human_value = value.copy()
     human_value['text'] = '{} cores at {} %'.format(
         value['cpu_count'], value['cpu'])
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
+@register_sysinfo
 def storage():
     from rhodecode.lib.helpers import format_byte_size_binary
     from rhodecode.model.settings import VcsSettingsModel
     path = VcsSettingsModel().get_repos_location()
 
     value = dict(percent=0, used=0, total=0, path=path, text='')
     state = STATE_OK_DEFAULT
     if not psutil:
         return SysInfoRes(value=value, state=state)
 
     try:
         value.update(dict(psutil.disk_usage(path)._asdict()))
     except Exception as e:
         log.exception('Failed to fetch disk info')
         state = {'message': str(e), 'type': STATE_ERR}
 
     human_value = value.copy()
     human_value['used'] = format_byte_size_binary(value['used'])
     human_value['total'] = format_byte_size_binary(value['total'])
     human_value['text'] = "{}/{}, {}% used".format(
         format_byte_size_binary(value['used']),
         format_byte_size_binary(value['total']),
         value['percent'])
 
     if state['type'] == STATE_OK and value['percent'] > 90:
         msg = 'Critical: your disk space is very low.'
         state = {'message': msg, 'type': STATE_ERR}
 
     elif state['type'] == STATE_OK and value['percent'] > 70:
         msg = 'Warning: your disk space is running low.'
         state = {'message': msg, 'type': STATE_WARN}
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
+@register_sysinfo
 def storage_inodes():
     from rhodecode.model.settings import VcsSettingsModel
     path = VcsSettingsModel().get_repos_location()
 
     value = dict(percent=0, free=0, used=0, total=0, path=path, text='')
     state = STATE_OK_DEFAULT
     if not psutil:
         return SysInfoRes(value=value, state=state)
 
     try:
         i_stat = os.statvfs(path)
         value['free'] = i_stat.f_ffree
         value['used'] = i_stat.f_files-i_stat.f_favail
         value['total'] = i_stat.f_files
         value['percent'] = percentage(value['used'], value['total'])
     except Exception as e:
         log.exception('Failed to fetch disk inodes info')
         state = {'message': str(e), 'type': STATE_ERR}
 
     human_value = value.copy()
     human_value['text'] = "{}/{}, {}% used".format(
         value['used'], value['total'], value['percent'])
 
     if state['type'] == STATE_OK and value['percent'] > 90:
         msg = 'Critical: your disk free inodes are very low.'
         state = {'message': msg, 'type': STATE_ERR}
 
     elif state['type'] == STATE_OK and value['percent'] > 70:
         msg = 'Warning: your disk free inodes are running low.'
         state = {'message': msg, 'type': STATE_WARN}
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
+@register_sysinfo
 def storage_archives():
     import rhodecode
     from rhodecode.lib.utils import safe_str
     from rhodecode.lib.helpers import format_byte_size_binary
 
     msg = 'Enable this by setting ' \
           'archive_cache_dir=/path/to/cache option in the .ini file'
     path = safe_str(rhodecode.CONFIG.get('archive_cache_dir', msg))
 
     value = dict(percent=0, used=0, total=0, items=0, path=path, text='')
     state = STATE_OK_DEFAULT
     try:
         items_count = 0
         used = 0
         for root, dirs, files in os.walk(path):
             if root == path:
                 items_count = len(files)
 
             for f in files:
                 try:
                     used += os.path.getsize(os.path.join(root, f))
                 except OSError:
                     pass
         value.update({
             'percent': 100,
             'used': used,
             'total': used,
             'items': items_count
         })
 
     except Exception as e:
         log.exception('failed to fetch archive cache storage')
         state = {'message': str(e), 'type': STATE_ERR}
 
     human_value = value.copy()
     human_value['used'] = format_byte_size_binary(value['used'])
     human_value['total'] = format_byte_size_binary(value['total'])
     human_value['text'] = "{} ({} items)".format(
         human_value['used'], value['items'])
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
+@register_sysinfo
 def storage_gist():
     from rhodecode.model.gist import GIST_STORE_LOC
     from rhodecode.model.settings import VcsSettingsModel
     from rhodecode.lib.utils import safe_str
     from rhodecode.lib.helpers import format_byte_size_binary
     path = safe_str(os.path.join(
         VcsSettingsModel().get_repos_location(), GIST_STORE_LOC))
 
     # gist storage
     value = dict(percent=0, used=0, total=0, items=0, path=path, text='')
     state = STATE_OK_DEFAULT
 
     try:
         items_count = 0
         used = 0
         for root, dirs, files in os.walk(path):
             if root == path:
                 items_count = len(dirs)
 
             for f in files:
                 try:
                     used += os.path.getsize(os.path.join(root, f))
                 except OSError:
                     pass
         value.update({
             'percent': 100,
             'used': used,
             'total': used,
             'items': items_count
         })
     except Exception as e:
         log.exception('failed to fetch gist storage items')
         state = {'message': str(e), 'type': STATE_ERR}
 
     human_value = value.copy()
     human_value['used'] = format_byte_size_binary(value['used'])
     human_value['total'] = format_byte_size_binary(value['total'])
     human_value['text'] = "{} ({} items)".format(
         human_value['used'], value['items'])
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
+@register_sysinfo
 def storage_temp():
     import tempfile
     from rhodecode.lib.helpers import format_byte_size_binary
 
     path = tempfile.gettempdir()
     value = dict(percent=0, used=0, total=0, items=0, path=path, text='')
     state = STATE_OK_DEFAULT
 
     if not psutil:
         return SysInfoRes(value=value, state=state)
 
     try:
         value.update(dict(psutil.disk_usage(path)._asdict()))
     except Exception as e:
         log.exception('Failed to fetch temp dir info')
         state = {'message': str(e), 'type': STATE_ERR}
 
     human_value = value.copy()
     human_value['used'] = format_byte_size_binary(value['used'])
     human_value['total'] = format_byte_size_binary(value['total'])
     human_value['text'] = "{}/{}, {}% used".format(
         format_byte_size_binary(value['used']),
         format_byte_size_binary(value['total']),
         value['percent'])
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
+@register_sysinfo
 def search_info():
     import rhodecode
     from rhodecode.lib.index import searcher_from_config
 
     backend = rhodecode.CONFIG.get('search.module', '')
     location = rhodecode.CONFIG.get('search.location', '')
 
     try:
         searcher = searcher_from_config(rhodecode.CONFIG)
         searcher = searcher.__class__.__name__
     except Exception:
         searcher = None
 
     value = dict(
         backend=backend, searcher=searcher, location=location, text='')
     state = STATE_OK_DEFAULT
 
     human_value = value.copy()
     human_value['text'] = "backend:`{}`".format(human_value['backend'])
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
+@register_sysinfo
 def git_info():
     from rhodecode.lib.vcs.backends import git
     state = STATE_OK_DEFAULT
     value = human_value = ''
     try:
         value = git.discover_git_version(raise_on_exc=True)
         human_value = 'version reported from VCSServer: {}'.format(value)
     except Exception as e:
         state = {'message': str(e), 'type': STATE_ERR}
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
+@register_sysinfo
 def hg_info():
     from rhodecode.lib.vcs.backends import hg
     state = STATE_OK_DEFAULT
     value = human_value = ''
     try:
         value = hg.discover_hg_version(raise_on_exc=True)
         human_value = 'version reported from VCSServer: {}'.format(value)
     except Exception as e:
         state = {'message': str(e), 'type': STATE_ERR}
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
+@register_sysinfo
 def svn_info():
     from rhodecode.lib.vcs.backends import svn
     state = STATE_OK_DEFAULT
     value = human_value = ''
     try:
         value = svn.discover_svn_version(raise_on_exc=True)
         human_value = 'version reported from VCSServer: {}'.format(value)
     except Exception as e:
         state = {'message': str(e), 'type': STATE_ERR}
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
+@register_sysinfo
 def vcs_backends():
     import rhodecode
     value = rhodecode.CONFIG.get('vcs.backends')
     human_value = 'Enabled backends in order: {}'.format(','.join(value))
     return SysInfoRes(value=value, human_value=human_value)
 
 
+@register_sysinfo
 def vcs_server():
     import rhodecode
     from rhodecode.lib.vcs.backends import get_vcsserver_service_data
 
     server_url = rhodecode.CONFIG.get('vcs.server')
     enabled = rhodecode.CONFIG.get('vcs.server.enable')
     protocol = rhodecode.CONFIG.get('vcs.server.protocol') or 'http'
     state = STATE_OK_DEFAULT
     version = None
     workers = 0
 
     try:
         data = get_vcsserver_service_data()
         if data and 'version' in data:
             version = data['version']
 
         if data and 'config' in data:
             conf = data['config']
             workers = conf.get('workers', 'NOT AVAILABLE')
 
         connection = 'connected'
     except Exception as e:
         connection = 'failed'
         state = {'message': str(e), 'type': STATE_ERR}
 
     value = dict(
         url=server_url,
         enabled=enabled,
         protocol=protocol,
         connection=connection,
         version=version,
         text='',
     )
 
     human_value = value.copy()
     human_value['text'] = \
         '{url}@ver:{ver} via {mode} mode[workers:{workers}], connection:{conn}'.format(
             url=server_url, ver=version, workers=workers, mode=protocol,
             conn=connection)
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
+@register_sysinfo
 def vcs_server_config():
     from rhodecode.lib.vcs.backends import get_vcsserver_service_data
     state = STATE_OK_DEFAULT
 
     value = {}
     try:
         data = get_vcsserver_service_data()
         value = data['app_config']
     except Exception as e:
         state = {'message': str(e), 'type': STATE_ERR}
 
     human_value = value.copy()
     human_value['text'] = 'VCS Server config'
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
+@register_sysinfo
 def rhodecode_app_info():
     import rhodecode
     edition = rhodecode.CONFIG.get('rhodecode.edition')
 
     value = dict(
         rhodecode_version=rhodecode.__version__,
         rhodecode_lib_path=os.path.abspath(rhodecode.__file__),
         text=''
     )
     human_value = value.copy()
     human_value['text'] = 'RhodeCode {edition}, version {ver}'.format(
         edition=edition, ver=value['rhodecode_version']
     )
     return SysInfoRes(value=value, human_value=human_value)
 
 
+@register_sysinfo
 def rhodecode_config():
     import rhodecode
     path = rhodecode.CONFIG.get('__file__')
     rhodecode_ini_safe = rhodecode.CONFIG.copy()
     cert_path = get_cert_path(path)
 
     try:
         config = configparser.ConfigParser()
         config.read(path)
         parsed_ini = config
         if parsed_ini.has_section('server:main'):
             parsed_ini = dict(parsed_ini.items('server:main'))
     except Exception:
         log.exception('Failed to read .ini file for display')
         parsed_ini = {}
 
     rhodecode_ini_safe['server:main'] = parsed_ini
 
     blacklist = [
         'rhodecode_license_key',
         'routes.map',
         'sqlalchemy.db1.url',
         'channelstream.secret',
         'beaker.session.secret',
         'rhodecode.encrypted_values.secret',
         'rhodecode_auth_github_consumer_key',
         'rhodecode_auth_github_consumer_secret',
         'rhodecode_auth_google_consumer_key',
         'rhodecode_auth_google_consumer_secret',
         'rhodecode_auth_bitbucket_consumer_secret',
         'rhodecode_auth_bitbucket_consumer_key',
         'rhodecode_auth_twitter_consumer_secret',
         'rhodecode_auth_twitter_consumer_key',
 
         'rhodecode_auth_twitter_secret',
         'rhodecode_auth_github_secret',
         'rhodecode_auth_google_secret',
         'rhodecode_auth_bitbucket_secret',
 
         'appenlight.api_key',
         ('app_conf', 'sqlalchemy.db1.url')
     ]
     for k in blacklist:
         if isinstance(k, tuple):
             section, key = k
             if section in rhodecode_ini_safe:
                 rhodecode_ini_safe[section] = '**OBFUSCATED**'
         else:
             rhodecode_ini_safe.pop(k, None)
 
     # TODO: maybe put some CONFIG checks here ?
     return SysInfoRes(value={'config': rhodecode_ini_safe,
                              'path': path, 'cert_path': cert_path})
 
 
+@register_sysinfo
 def database_info():
     import rhodecode
     from sqlalchemy.engine import url as engine_url
     from rhodecode.model.meta import Base as sql_base, Session
     from rhodecode.model.db import DbMigrateVersion
 
     state = STATE_OK_DEFAULT
 
     db_migrate = DbMigrateVersion.query().filter(
         DbMigrateVersion.repository_id == 'rhodecode_db_migrations').one()
 
     db_url_obj = engine_url.make_url(rhodecode.CONFIG['sqlalchemy.db1.url'])
 
     try:
         engine = sql_base.metadata.bind
         db_server_info = engine.dialect._get_server_version_info(
             Session.connection(bind=engine))
         db_version = '.'.join(map(str, db_server_info))
     except Exception:
         log.exception('failed to fetch db version')
         db_version = 'UNKNOWN'
 
     db_info = dict(
         migrate_version=db_migrate.version,
         type=db_url_obj.get_backend_name(),
         version=db_version,
         url=repr(db_url_obj)
     )
     current_version = db_migrate.version
     expected_version = rhodecode.__dbversion__
     if state['type'] == STATE_OK and current_version != expected_version:
         msg = 'Critical: database schema mismatch, ' \
               'expected version {}, got {}. ' \
               'Please run migrations on your database.'.format(
             expected_version, current_version)
         state = {'message': msg, 'type': STATE_ERR}
 
     human_value = db_info.copy()
     human_value['url'] = "{} @ migration version: {}".format(
         db_info['url'], db_info['migrate_version'])
     human_value['version'] = "{} {}".format(db_info['type'], db_info['version'])
     return SysInfoRes(value=db_info, state=state, human_value=human_value)
 
 
+@register_sysinfo
 def server_info(environ):
     import rhodecode
     from rhodecode.lib.base import get_server_ip_addr, get_server_port
 
     value = {
         'server_ip': '%s:%s' % (
             get_server_ip_addr(environ, log_errors=False),
             get_server_port(environ)
         ),
         'server_id': rhodecode.CONFIG.get('instance_id'),
     }
     return SysInfoRes(value=value)
 
 
+@register_sysinfo
 def usage_info():
     from rhodecode.model.db import User, Repository
     value = {
         'users': User.query().count(),
         'users_active': User.query().filter(User.active == True).count(),
         'repositories': Repository.query().count(),
         'repository_types': {
             'hg': Repository.query().filter(
                 Repository.repo_type == 'hg').count(),
             'git': Repository.query().filter(
                 Repository.repo_type == 'git').count(),
             'svn': Repository.query().filter(
                 Repository.repo_type == 'svn').count(),
         },
     }
     return SysInfoRes(value=value)
 
 
 def get_system_info(environ):
     environ = environ or {}
     return {
         'rhodecode_app': SysInfo(rhodecode_app_info)(),
         'rhodecode_config': SysInfo(rhodecode_config)(),
         'rhodecode_usage': SysInfo(usage_info)(),
         'python': SysInfo(python_info)(),
         'py_modules': SysInfo(py_modules)(),
 
         'platform': SysInfo(platform_type)(),
         'locale': SysInfo(locale_info)(),
         'server': SysInfo(server_info, environ=environ)(),
         'database': SysInfo(database_info)(),
         'ulimit': SysInfo(ulimit_info)(),
         'storage': SysInfo(storage)(),
         'storage_inodes': SysInfo(storage_inodes)(),
         'storage_archive': SysInfo(storage_archives)(),
         'storage_gist': SysInfo(storage_gist)(),
         'storage_temp': SysInfo(storage_temp)(),
 
         'search': SysInfo(search_info)(),
 
         'uptime': SysInfo(uptime)(),
         'load': SysInfo(machine_load)(),
         'cpu': SysInfo(cpu)(),
         'memory': SysInfo(memory)(),
 
         'vcs_backends': SysInfo(vcs_backends)(),
         'vcs_server': SysInfo(vcs_server)(),
 
         'vcs_server_config': SysInfo(vcs_server_config)(),
 
         'git': SysInfo(git_info)(),
         'hg': SysInfo(hg_info)(),
         'svn': SysInfo(svn_info)(),
     }
+
+
+def load_system_info(key):
+    """
+    get_sys_info('vcs_server')
+    get_sys_info('database')
+    """
+    return SysInfo(registered_helpers[key])()