celery: ensure celery workers are compatible with python3.11 and also fix CLI option for celery 5.X

rhodecode/config/patches.py

@@ -30,6 +30,68 b' Please keep the following principles in '
 """
 
 
+def inspect_formatargspec():
+
+    import inspect
+    from inspect import formatannotation
+
+    def backport_inspect_formatargspec(
+            args, varargs=None, varkw=None, defaults=None,
+            kwonlyargs=(), kwonlydefaults={}, annotations={},
+            formatarg=str,
+            formatvarargs=lambda name: '*' + name,
+            formatvarkw=lambda name: '**' + name,
+            formatvalue=lambda value: '=' + repr(value),
+            formatreturns=lambda text: ' -> ' + text,
+            formatannotation=formatannotation):
+        """Copy formatargspec from python 3.7 standard library.
+        Python 3 has deprecated formatargspec and requested that Signature
+        be used instead, however this requires a full reimplementation
+        of formatargspec() in terms of creating Parameter objects and such.
+        Instead of introducing all the object-creation overhead and having
+        to reinvent from scratch, just copy their compatibility routine.
+        Utimately we would need to rewrite our "decorator" routine completely
+        which is not really worth it right now, until all Python 2.x support
+        is dropped.
+        """
+
+        def formatargandannotation(arg):
+            result = formatarg(arg)
+            if arg in annotations:
+                result += ': ' + formatannotation(annotations[arg])
+            return result
+
+        specs = []
+        if defaults:
+            firstdefault = len(args) - len(defaults)
+        for i, arg in enumerate(args):
+            spec = formatargandannotation(arg)
+            if defaults and i >= firstdefault:
+                spec = spec + formatvalue(defaults[i - firstdefault])
+            specs.append(spec)
+        if varargs is not None:
+            specs.append(formatvarargs(formatargandannotation(varargs)))
+        else:
+            if kwonlyargs:
+                specs.append('*')
+        if kwonlyargs:
+            for kwonlyarg in kwonlyargs:
+                spec = formatargandannotation(kwonlyarg)
+                if kwonlydefaults and kwonlyarg in kwonlydefaults:
+                    spec += formatvalue(kwonlydefaults[kwonlyarg])
+                specs.append(spec)
+        if varkw is not None:
+            specs.append(formatvarkw(formatargandannotation(varkw)))
+        result = '(' + ', '.join(specs) + ')'
+        if 'return' in annotations:
+            result += formatreturns(formatannotation(annotations['return']))
+        return result
+
+    # NOTE: inject for python3.11
+    inspect.formatargspec = backport_inspect_formatargspec
+    return inspect
+
+
 def inspect_getargspec():
     """
     Pyramid rely on inspect.getargspec to lookup the signature of
@@ -92,7 +154,7 b' def inspect_getargspec():'
         args, varargs, varkw = inspect.getargs(func.func_code)
         return inspect.ArgSpec(args, varargs, varkw, func.func_defaults)
 
-    #TODO: fix it and test it on python3.11
-    inspect.getargspec = inspect.getfullargspec #custom_getargspec
+    # NOTE: inject for python3.11
+    inspect.getargspec = inspect.getfullargspec
 
     return inspect

rhodecode/lib/celerylib/loader.py

@@ -1,4 +1,3 b''
-
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
@@ -28,14 +27,19 b' Celery loader, run with::'
         --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler \
         --loglevel DEBUG --ini=.dev/dev.ini
 """
-import os
+from rhodecode.config.patches import inspect_getargspec, inspect_formatargspec
+inspect_getargspec()
+inspect_formatargspec()
+# python3.11 inspect patches for backward compat on `paste` code
+
 import logging
 import importlib
 
+import click
 from celery import Celery
 from celery import signals
 from celery import Task
-from celery import exceptions  # pragma: no cover
+from celery import exceptions  # noqa
 from kombu.serialization import register
 
 import rhodecode
@@ -82,13 +86,13 b' base_celery_config = {'
 }
 
 
-def add_preload_arguments(parser):
-    parser.add_argument(
-        '--ini', default=None,
+preload_option_ini = click.Option(
+    ('--ini',),
         help='Path to ini configuration file.'
     )
-    parser.add_argument(
-        '--ini-var', default=None,
+
+preload_option_ini_var = click.Option(
+    ('--ini-var',),
         help='Comma separated list of key=value to pass to ini.'
     )
 
@@ -108,7 +112,8 b' def get_logger(obj):'
 
 # init main celery app
 celery_app = Celery()
-celery_app.user_options['preload'].add(add_preload_arguments)
+celery_app.user_options['preload'].add(preload_option_ini)
+celery_app.user_options['preload'].add(preload_option_ini_var)
 
 
 @signals.setup_logging.connect
@@ -199,7 +204,6 b' def task_success_signal(result, **kwargs'
         closer()
 
 
-
 @signals.task_retry.connect
 def task_retry_signal(
         request, reason, einfo, **kwargs):