##// END OF EJS Templates
celery: ensure celery workers are compatible with python3.11 and also fix CLI option for celery 5.X
super-admin -
r5102:4a1c6bb5 default
parent child Browse files
Show More
@@ -30,6 +30,68 b' Please keep the following principles in '
30 """
30 """
31
31
32
32
33 def inspect_formatargspec():
34
35 import inspect
36 from inspect import formatannotation
37
38 def backport_inspect_formatargspec(
39 args, varargs=None, varkw=None, defaults=None,
40 kwonlyargs=(), kwonlydefaults={}, annotations={},
41 formatarg=str,
42 formatvarargs=lambda name: '*' + name,
43 formatvarkw=lambda name: '**' + name,
44 formatvalue=lambda value: '=' + repr(value),
45 formatreturns=lambda text: ' -> ' + text,
46 formatannotation=formatannotation):
47 """Copy formatargspec from python 3.7 standard library.
48 Python 3 has deprecated formatargspec and requested that Signature
49 be used instead, however this requires a full reimplementation
50 of formatargspec() in terms of creating Parameter objects and such.
51 Instead of introducing all the object-creation overhead and having
52 to reinvent from scratch, just copy their compatibility routine.
53 Utimately we would need to rewrite our "decorator" routine completely
54 which is not really worth it right now, until all Python 2.x support
55 is dropped.
56 """
57
58 def formatargandannotation(arg):
59 result = formatarg(arg)
60 if arg in annotations:
61 result += ': ' + formatannotation(annotations[arg])
62 return result
63
64 specs = []
65 if defaults:
66 firstdefault = len(args) - len(defaults)
67 for i, arg in enumerate(args):
68 spec = formatargandannotation(arg)
69 if defaults and i >= firstdefault:
70 spec = spec + formatvalue(defaults[i - firstdefault])
71 specs.append(spec)
72 if varargs is not None:
73 specs.append(formatvarargs(formatargandannotation(varargs)))
74 else:
75 if kwonlyargs:
76 specs.append('*')
77 if kwonlyargs:
78 for kwonlyarg in kwonlyargs:
79 spec = formatargandannotation(kwonlyarg)
80 if kwonlydefaults and kwonlyarg in kwonlydefaults:
81 spec += formatvalue(kwonlydefaults[kwonlyarg])
82 specs.append(spec)
83 if varkw is not None:
84 specs.append(formatvarkw(formatargandannotation(varkw)))
85 result = '(' + ', '.join(specs) + ')'
86 if 'return' in annotations:
87 result += formatreturns(formatannotation(annotations['return']))
88 return result
89
90 # NOTE: inject for python3.11
91 inspect.formatargspec = backport_inspect_formatargspec
92 return inspect
93
94
33 def inspect_getargspec():
95 def inspect_getargspec():
34 """
96 """
35 Pyramid rely on inspect.getargspec to lookup the signature of
97 Pyramid rely on inspect.getargspec to lookup the signature of
@@ -92,7 +154,7 b' def inspect_getargspec():'
92 args, varargs, varkw = inspect.getargs(func.func_code)
154 args, varargs, varkw = inspect.getargs(func.func_code)
93 return inspect.ArgSpec(args, varargs, varkw, func.func_defaults)
155 return inspect.ArgSpec(args, varargs, varkw, func.func_defaults)
94
156
95 #TODO: fix it and test it on python3.11
157 # NOTE: inject for python3.11
96 inspect.getargspec = inspect.getfullargspec #custom_getargspec
158 inspect.getargspec = inspect.getfullargspec
97
159
98 return inspect
160 return inspect
@@ -1,4 +1,3 b''
1
2 # Copyright (C) 2010-2023 RhodeCode GmbH
1 # Copyright (C) 2010-2023 RhodeCode GmbH
3 #
2 #
4 # This program is free software: you can redistribute it and/or modify
3 # This program is free software: you can redistribute it and/or modify
@@ -28,14 +27,19 b' Celery loader, run with::'
28 --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler \
27 --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler \
29 --loglevel DEBUG --ini=.dev/dev.ini
28 --loglevel DEBUG --ini=.dev/dev.ini
30 """
29 """
31 import os
30 from rhodecode.config.patches import inspect_getargspec, inspect_formatargspec
31 inspect_getargspec()
32 inspect_formatargspec()
33 # python3.11 inspect patches for backward compat on `paste` code
34
32 import logging
35 import logging
33 import importlib
36 import importlib
34
37
38 import click
35 from celery import Celery
39 from celery import Celery
36 from celery import signals
40 from celery import signals
37 from celery import Task
41 from celery import Task
38 from celery import exceptions # pragma: no cover
42 from celery import exceptions # noqa
39 from kombu.serialization import register
43 from kombu.serialization import register
40
44
41 import rhodecode
45 import rhodecode
@@ -82,13 +86,13 b' base_celery_config = {'
82 }
86 }
83
87
84
88
85 def add_preload_arguments(parser):
89 preload_option_ini = click.Option(
86 parser.add_argument(
90 ('--ini',),
87 '--ini', default=None,
88 help='Path to ini configuration file.'
91 help='Path to ini configuration file.'
89 )
92 )
90 parser.add_argument(
93
91 '--ini-var', default=None,
94 preload_option_ini_var = click.Option(
95 ('--ini-var',),
92 help='Comma separated list of key=value to pass to ini.'
96 help='Comma separated list of key=value to pass to ini.'
93 )
97 )
94
98
@@ -108,7 +112,8 b' def get_logger(obj):'
108
112
109 # init main celery app
113 # init main celery app
110 celery_app = Celery()
114 celery_app = Celery()
111 celery_app.user_options['preload'].add(add_preload_arguments)
115 celery_app.user_options['preload'].add(preload_option_ini)
116 celery_app.user_options['preload'].add(preload_option_ini_var)
112
117
113
118
114 @signals.setup_logging.connect
119 @signals.setup_logging.connect
@@ -199,7 +204,6 b' def task_success_signal(result, **kwargs'
199 closer()
204 closer()
200
205
201
206
202
203 @signals.task_retry.connect
207 @signals.task_retry.connect
204 def task_retry_signal(
208 def task_retry_signal(
205 request, reason, einfo, **kwargs):
209 request, reason, einfo, **kwargs):
General Comments 0
You need to be logged in to leave comments. Login now