configs: refactor and updated for python3 migrations

rhodecode/config/licenses.json

@@ -184,17 +184,6 b''
     {
         "license": [
             {
-                "fullName": "MIT License",
-                "shortName": "mit",
-                "spdxId": "MIT",
-                "url": "http://spdx.org/licenses/MIT.html"
-            }
-        ],
-        "name": "python2.7-pathlib2-2.3.5"
-    },
-    {
-        "license": [
-            {
                 "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                 "shortName": "bsdOriginal",
                 "spdxId": "BSD-4-Clause",
@@ -1868,17 +1857,6 b''
                 "url": "http://spdx.org/licenses/BSD-4-Clause.html"
             }
         ],
-        "name": "python2.7-configobj-5.0.6"
-    },
-    {
-        "license": [
-            {
-                "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
-                "shortName": "bsdOriginal",
-                "spdxId": "BSD-4-Clause",
-                "url": "http://spdx.org/licenses/BSD-4-Clause.html"
-            }
-        ],
         "name": "python2.7-channelstream-0.6.14"
     },
     {

rhodecode/config/middleware.py

@@ -120,8 +120,7 b' def make_pyramid_app(global_config, **se'
     meta.Session.remove()
 
     total_time = time.time() - start_time
-    log.info('Pyramid app `%s` created and configured in %.2fs',
-             getattr(pyramid_app, 'func_name', 'pyramid_app'), total_time)
+    log.info('Pyramid app created and configured in %.2fs', total_time)
     return pyramid_app
 
 
@@ -241,11 +240,20 b' def error_handler(exception, request):'
     c.exception_id_url = request.route_url(
         'admin_settings_exception_tracker_show', exception_id=c.exception_id)
 
+    debug_mode = rhodecode.ConfigGet().get_bool('debug')
     if c.show_exception_id:
         store_exception(c.exception_id, exc_info)
-    c.exception_debug = str2bool(rhodecode.CONFIG.get('debug'))
+    c.exception_debug = debug_mode
     c.exception_config_ini = rhodecode.CONFIG.get('__file__')
 
+    if debug_mode:
+        try:
+            from rich.traceback import install
+            install(show_locals=True)
+            log.debug('Installing rich tracebacks...')
+        except ImportError:
+            pass
+
     response = render_to_response(
         '/errors/error_document.mako', {'c': c, 'h': helpers}, request=request,
         response=base_response)
@@ -283,6 +291,17 b' def includeme_first(config):'
         '_static/rhodecode', path='rhodecode:public', cache_max_age=3600 * 24)
 
 
+ce_auth_resources = [
+    'rhodecode.authentication.plugins.auth_crowd',
+    'rhodecode.authentication.plugins.auth_headers',
+    'rhodecode.authentication.plugins.auth_jasig_cas',
+    'rhodecode.authentication.plugins.auth_ldap',
+    'rhodecode.authentication.plugins.auth_pam',
+    'rhodecode.authentication.plugins.auth_rhodecode',
+    'rhodecode.authentication.plugins.auth_token',
+]
+
+
 def includeme(config, auth_resources=None):
     from rhodecode.lib.celerylib.loader import configure_celery
     log.debug('Initializing main includeme from %s', os.path.basename(__file__))
@@ -306,21 +325,14 b' def includeme(config, auth_resources=Non'
     config.include('pyramid_mako')
     config.include('rhodecode.lib.rc_beaker')
     config.include('rhodecode.lib.rc_cache')
+    config.include('rhodecode.lib.rc_cache.archive_cache')
+
     config.include('rhodecode.apps._base.navigation')
     config.include('rhodecode.apps._base.subscribers')
     config.include('rhodecode.tweens')
     config.include('rhodecode.authentication')
 
     if load_all:
-        ce_auth_resources = [
-            'rhodecode.authentication.plugins.auth_crowd',
-            'rhodecode.authentication.plugins.auth_headers',
-            'rhodecode.authentication.plugins.auth_jasig_cas',
-            'rhodecode.authentication.plugins.auth_ldap',
-            'rhodecode.authentication.plugins.auth_pam',
-            'rhodecode.authentication.plugins.auth_rhodecode',
-            'rhodecode.authentication.plugins.auth_token',
-        ]
 
         # load CE authentication plugins
 
@@ -384,9 +396,6 b' def includeme(config, auth_resources=Non'
         config.add_subscriber(write_js_routes_if_enabled,
                               pyramid.events.ApplicationCreated)
 
-    # Set the authorization policy.
-    authz_policy = ACLAuthorizationPolicy()
-    config.set_authorization_policy(authz_policy)
 
     # Set the default renderer for HTML templates to mako.
     config.add_mako_renderer('.html')
@@ -448,7 +457,7 b' def wrap_app_in_wsgi_middlewares(pyramid'
             # In a single threaded mode server, on non sqlite db we should have
             # '0 Current Checked out connections' at the end of a request,
             # if not, then something, somewhere is leaving a connection open
-            pool = meta.Base.metadata.bind.engine.pool
+            pool = meta.get_engine().pool
             log.debug('sa pool status: %s', pool.status())
             total = time.time() - start
             log.debug('Request processing finalized: %.4fs', total)
@@ -531,7 +540,7 b' def sanitize_settings_and_apply_defaults'
 
     settings_maker.make_setting('vcs.svn.compatible_version', '')
     settings_maker.make_setting('vcs.hooks.protocol', 'http')
-    settings_maker.make_setting('vcs.hooks.host', '127.0.0.1')
+    settings_maker.make_setting('vcs.hooks.host', '*')
     settings_maker.make_setting('vcs.scm_app_implementation', 'http')
     settings_maker.make_setting('vcs.server', '')
     settings_maker.make_setting('vcs.server.protocol', 'http')
@@ -586,17 +595,17 b' def sanitize_settings_and_apply_defaults'
     # cache_perms
     settings_maker.make_setting('rc_cache.cache_perms.backend', 'dogpile.cache.rc.file_namespace')
     settings_maker.make_setting('rc_cache.cache_perms.expiration_time', 60 * 60, parser='int')
-    settings_maker.make_setting('rc_cache.cache_perms.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_perms.db'))
+    settings_maker.make_setting('rc_cache.cache_perms.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_perms_db'))
 
     # cache_repo
     settings_maker.make_setting('rc_cache.cache_repo.backend', 'dogpile.cache.rc.file_namespace')
     settings_maker.make_setting('rc_cache.cache_repo.expiration_time', 60 * 60 * 24 * 30, parser='int')
-    settings_maker.make_setting('rc_cache.cache_repo.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_repo.db'))
+    settings_maker.make_setting('rc_cache.cache_repo.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_repo_db'))
 
     # cache_license
     settings_maker.make_setting('rc_cache.cache_license.backend', 'dogpile.cache.rc.file_namespace')
     settings_maker.make_setting('rc_cache.cache_license.expiration_time', 60 * 5, parser='int')
-    settings_maker.make_setting('rc_cache.cache_license.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_license.db'))
+    settings_maker.make_setting('rc_cache.cache_license.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_license_db'))
 
     # cache_repo_longterm memory, 96H
     settings_maker.make_setting('rc_cache.cache_repo_longterm.backend', 'dogpile.cache.rc.memory_lru')
@@ -608,6 +617,11 b' def sanitize_settings_and_apply_defaults'
     settings_maker.make_setting('rc_cache.sql_cache_short.expiration_time', 30, parser='int')
     settings_maker.make_setting('rc_cache.sql_cache_short.max_size', 10000, parser='int')
 
+    # archive_cache
+    settings_maker.make_setting('archive_cache.store_dir', os.path.join(default_cache_dir, 'archive_cache'), default_when_empty=True,)
+    settings_maker.make_setting('archive_cache.cache_size_gb', 10, parser='float')
+    settings_maker.make_setting('archive_cache.cache_shards', 10, parser='int')
+
     settings_maker.env_expand()
 
     # configure instance id

rhodecode/config/patches.py

@@ -94,6 +94,7 b' def inspect_getargspec():'
         args, varargs, varkw = inspect.getargs(func.func_code)
         return inspect.ArgSpec(args, varargs, varkw, func.func_defaults)
 
-    inspect.getargspec = custom_getargspec
+    #TODO: fix it and test it on python3.11
+    inspect.getargspec = inspect.getfullargspec #custom_getargspec
 
     return inspect

rhodecode/config/rcextensions/helpers/extract_pre_files.py

@@ -32,7 +32,7 b' import json'
 from rhodecode.lib import diffs
 from rhodecode.lib.vcs.backends.hg.diff import MercurialDiff
 from rhodecode.lib.vcs.backends.git.diff import GitDiff
-from vcsserver.utils import safe_int
+from rhodecode.lib.str_utils import safe_int
 
 
 def get_svn_files(repo, vcs_repo, refs):
@@ -108,7 +108,7 b' def get_git_files(repo, vcs_repo, refs):'
         stdout, stderr = vcs_repo.run_git_command(cmd, extra_env=git_env)
         vcs_diff = GitDiff(stdout)
 
-        diff_processor = diffs.DiffProcessor(vcs_diff, format='newdiff')
+        diff_processor = diffs.DiffProcessor(vcs_diff, diff_format='newdiff')
         # this is list of dicts with diff information
         # _parsed[0].keys()
         # ['raw_diff', 'old_revision', 'stats', 'original_filename',

rhodecode/config/rcextensions/utils.py

@@ -21,7 +21,9 b' import os'
 import string
 import functools
 import collections
-import urllib.request, urllib.parse, urllib.error
+import urllib.request
+import urllib.parse
+import urllib.error
 
 log = logging.getLogger('rhodecode.' + __name__)
 
@@ -48,7 +50,7 b' class DotDict(dict):'
     def __contains__(self, k):
         try:
             return dict.__contains__(self, k) or hasattr(self, k)
-        except:
+        except Exception:
             return False
 
     # only called if k not found in normal places
@@ -153,14 +155,13 b' def maybe_log_call(name, args, kwargs):'
         calls[name].append((args, kwargs))
 
 
-def str2bool(_str):
+def str2bool(_str) -> bool:
     """
     returns True/False value from given string, it tries to translate the
     string into boolean
 
     :param _str: string value to translate into boolean
-    :rtype: boolean
-    :returns: boolean from given string
+    :returns: bool from given string
     """
     if _str is None:
         return False

rhodecode/config/settings_maker.py

@@ -24,55 +24,17 b' import functools'
 import logging
 import tempfile
 import logging.config
+from rhodecode.lib.type_utils import str2bool, aslist
+
 log = logging.getLogger(__name__)
 
+
 # skip keys, that are set here, so we don't double process those
 set_keys = {
     '__file__': ''
 }
 
 
-def str2bool(_str):
-    """
-    returns True/False value from given string, it tries to translate the
-    string into boolean
-
-    :param _str: string value to translate into boolean
-    :rtype: boolean
-    :returns: boolean from given string
-    """
-    if _str is None:
-        return False
-    if _str in (True, False):
-        return _str
-    _str = str(_str).strip().lower()
-    return _str in ('t', 'true', 'y', 'yes', 'on', '1')
-
-
-def aslist(obj, sep=None, strip=True):
-    """
-    Returns given string separated by sep as list
-
-    :param obj:
-    :param sep:
-    :param strip:
-    """
-    if isinstance(obj, str):
-        if obj in ['', ""]:
-            return []
-
-        lst = obj.split(sep)
-        if strip:
-            lst = [v.strip() for v in lst]
-        return lst
-    elif isinstance(obj, (list, tuple)):
-        return obj
-    elif obj is None:
-        return []
-    else:
-        return [obj]
-
-
 class SettingsMaker(object):
 
     def __init__(self, app_settings):
@@ -87,6 +49,10 b' class SettingsMaker(object):'
         return int(input_val)
 
     @classmethod
+    def _float_func(cls, input_val):
+        return float(input_val)
+
+    @classmethod
     def _list_func(cls, input_val, sep=','):
         return aslist(input_val, sep=sep)
 
@@ -97,10 +63,6 b' class SettingsMaker(object):'
         return input_val
 
     @classmethod
-    def _float_func(cls, input_val):
-        return float(input_val)
-
-    @classmethod
     def _dir_func(cls, input_val, ensure_dir=False, mode=0o755):
 
         # ensure we have our dir created
@@ -160,7 +122,7 b' class SettingsMaker(object):'
                       'file does not exist.... specify path using logging.logging_conf_file= config setting. ', logging_conf)
             return
 
-        with open(logging_conf, 'rb') as f:
+        with open(logging_conf, 'rt') as f:
             ini_template = textwrap.dedent(f.read())
             ini_template = string.Template(ini_template).safe_substitute(
                 RC_LOGGING_LEVEL=os.environ.get('RC_LOGGING_LEVEL', '') or level,
@@ -184,6 +146,7 b' class SettingsMaker(object):'
         parser_func = {
             'bool': self._bool_func,
             'int': self._int_func,
+            'float': self._float_func,
             'list': self._list_func,
             'list:newline': functools.partial(self._list_func, sep='/n'),
             'list:spacesep': functools.partial(self._list_func, sep=' '),

rhodecode/config/utils.py

@@ -95,9 +95,25 b' def set_instance_id(config):'
 
 
 def get_default_user_id():
-    from rhodecode.model.db import User, Session
-    user_id = Session()\
-        .query(User.user_id)\
-        .filter(User.username == User.DEFAULT_USER)\
-        .scalar()
+    DEFAULT_USER = 'default'
+    from sqlalchemy import text
+    from rhodecode.model import meta
+
+    engine = meta.get_engine()
+    with meta.SA_Session(engine) as session:
+        result = session.execute(text("SELECT user_id from users where username = :uname"), {'uname': DEFAULT_USER})
+        user_id = result.first()[0]
+
     return user_id
+
+
+def get_default_base_path():
+    from sqlalchemy import text
+    from rhodecode.model import meta
+
+    engine = meta.get_engine()
+    with meta.SA_Session(engine) as session:
+        result = session.execute(text("SELECT ui_value from rhodecode_ui where ui_key = '/'"))
+        base_path = result.first()[0]
+
+    return base_path