celery: upgrade to Celery 4

Celery 3 doesn't support Python 3.7 or later. This upgrade is thus essential
for full Python 3 support. But note that
https://docs.celeryproject.org/en/4.4.0/faq.html#does-celery-support-windows
says "No".

The names of config settings changed in Celery 3 to 4, as described on
https://docs.celeryproject.org/en/3.0/whatsnew-4.0.html#lowercase-setting-names .

Celery 4 config settings can now be specified in Kallithea .ini files by prefixing
with `celery.` - for example as `celery.broker_url`.

Remain backwards compatible for the usual settings, and map old names to the
new names.

development.ini

@@ -250,6 +250,7 b' ssh_enabled = false'
 ###        CELERY CONFIG        ####
 ####################################
 
+## Note: Celery doesn't support Windows.
 use_celery = false
 
 ## Example: use the message queue on the local virtual host 'kallitheavhost' as the RabbitMQ user 'kallithea':

docs/setup.rst

@@ -332,11 +332,11 b' To enable it, simply set::'
 
   use_celery = true
 
-and add or change the ``celery.*`` and ``broker.*`` configuration variables.
+and add or change the ``celery.*`` configuration variables.
 
-Remember that the ini files use the format with '.' and not with '_' like
-Celery. So for example setting `BROKER_HOST` in Celery means setting
-`broker.host` in the configuration file.
+Configuration settings are prefixed with 'celery.', so for example setting
+`broker_url` in Celery means setting `celery.broker_url` in the configuration
+file.
 
 To start the Celery process, run::
 

kallithea/lib/celerypylons/__init__.py

@@ -23,11 +23,24 b' import kallithea'
 
 
 class CeleryConfig(object):
-    CELERY_IMPORTS = ['kallithea.lib.celerylib.tasks']
-    CELERY_ACCEPT_CONTENT = ['json']
-    CELERY_RESULT_SERIALIZER = 'json'
-    CELERY_TASK_SERIALIZER = 'json'
-    CELERY_ALWAYS_EAGER = False
+    imports = ['kallithea.lib.celerylib.tasks']
+    task_always_eager = False
+
+# map from Kallithea .ini Celery 3 config names to Celery 4 config names
+celery3_compat = {
+    'broker.url': 'broker_url',
+    'celery.accept.content': 'accept_content',
+    'celery.always.eager': 'task_always_eager',
+    'celery.amqp.task.result.expires': 'result_expires',
+    'celeryd.concurrency': 'worker_concurrency',
+    'celeryd.max.tasks.per.child': 'worker_max_tasks_per_child',
+    #'celery.imports' ends up unchanged
+    'celery.result.backend': 'result_backend',
+    'celery.result.serializer': 'result_serializer',
+    'celery.task.serializer': 'task_serializer',
+}
+
+list_config_names = """imports accept_content""".split()
 
 
 desupported = set([
@@ -45,18 +58,20 b' def make_celery_config(config):'
 
     celery_config = CeleryConfig()
 
-    PREFIXES = """ADMINS BROKER CASSANDRA CELERYBEAT CELERYD CELERYMON CELERY EMAIL SERVER""".split()
-    LIST_PARAMS = """CELERY_IMPORTS CELERY_ACCEPT_CONTENT""".split()
-
     for config_key, config_value in sorted(config.items()):
         if config_key in desupported and config_value:
             log.error('Celery configuration setting %r is no longer supported', config_key)
-        celery_key = config_key.replace('.', '_').upper()
-        if celery_key.split('_', 1)[0] not in PREFIXES:
+        celery_key = celery3_compat.get(config_key)
+        parts = config_key.split('.', 1)
+        if celery_key:  # explicit Celery 3 backwards compatibility
+            pass
+        elif parts[0] == 'celery' and len(parts) == 2:  # Celery 4 config key
+            celery_key = parts[1]
+        else:
             continue
         if not isinstance(config_value, str):
             continue
-        if celery_key in LIST_PARAMS:
+        if celery_key in list_config_names:
             celery_value = config_value.split()
         elif config_value.isdigit():
             celery_value = int(config_value)
@@ -72,6 +87,6 b' def make_app():'
     """Create celery app from the TurboGears configuration file"""
     app = celery.Celery()
     celery_config = make_celery_config(tg.config)
-    kallithea.CELERY_EAGER = celery_config.CELERY_ALWAYS_EAGER
+    kallithea.CELERY_EAGER = celery_config.task_always_eager
     app.config_from_object(celery_config)
     return app

kallithea/lib/paster_commands/template.ini.mako

@@ -356,6 +356,7 b' ssh_locale = ${ssh_locale}'
 <%text>###        CELERY CONFIG        ####</%text>
 <%text>####################################</%text>
 
+<%text>## Note: Celery doesn't support Windows.</%text>
 use_celery = false
 
 <%text>## Example: use the message queue on the local virtual host 'kallitheavhost' as the RabbitMQ user 'kallithea':</%text>

setup.py

@@ -54,7 +54,7 b' requirements = ['
     "Mako >= 0.9.1, < 1.2",
     "Pygments >= 2.2.0, < 2.6",
     "Whoosh >= 2.7.1, < 2.8",
-    "celery >= 3.1, < 4.0", # TODO: celery 4 doesn't work
+    "celery >= 4.3, < 4.5",
     "Babel >= 1.3, < 2.9",
     "python-dateutil >= 2.1.0, < 2.9",
     "Markdown >= 2.2.1, < 3.2",