celery: upgrade to Celery 5.0 ... and adjust for Click API

Celery 5 has apparently no relevant API or config changes.

Celery is however switching to click. run_from_argv goes away, and there is no
simple way to do as before and start the worker with our Celery app but still
use Celery's own command line parser.

Apply hacks to make sure it still is possible to run like:

  kallithea-cli celery-run -c my.ini -- --help

  kallithea-cli celery-run -c my.ini -- --loglevel=ERROR broker_url=amqp://u:p@localhost:5672/v

kallithea/bin/kallithea_cli_celery.py

@@ -1,46 +1,53 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import celery.bin.worker
 import click
+from celery.bin.celery import celery as celery_command
 
 import kallithea
 import kallithea.bin.kallithea_cli_base as cli_base
 from kallithea.lib import celery_app
 from kallithea.lib.utils2 import asbool
 
 
 @cli_base.register_command(needs_config_file=True)
 @click.argument('celery_args', nargs=-1)
 def celery_run(celery_args, config):
     """Start Celery worker(s) for asynchronous tasks.
 
     This commands starts the Celery daemon which will spawn workers to handle
     certain asynchronous tasks for Kallithea.
 
     Any extra arguments you pass to this command will be passed through to
     Celery. Use '--' before such extra arguments to avoid options to be parsed
     by this CLI command.
     """
 
     if not asbool(config.get('use_celery')):
         raise Exception('Please set use_celery = true in .ini config '
                         'file before running this command')
 
     kallithea.CELERY_APP.config_from_object(celery_app.make_celery_config(config))
 
     kallithea.CELERY_APP.loader.on_worker_process_init = lambda: kallithea.config.application.make_app(config.global_conf, **config.local_conf)
 
-    cmd = celery.bin.worker.worker(kallithea.CELERY_APP)
-    return cmd.run_from_argv(None, command='celery-run -c CONFIG_FILE --', argv=list(celery_args))
+    args = list(celery_args)
+    # args[0] is generally ignored when prog_name is specified, but -h *needs* it to be 'worker' ... but will also suggest that users specify 'worker' explicitly
+    if not args or args[0] != 'worker':
+        args.insert(0, 'worker')
+
+    # inline kallithea.CELERY_APP.start in order to allow specifying prog_name
+    assert celery_command.params[0].name == 'app'
+    celery_command.params[0].default = kallithea.CELERY_APP
+    celery_command.main(args=args, prog_name='kallithea-cli celery-run -c CONFIG_FILE --')

setup.py

@@ -1,161 +1,161 b''
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 import os
 import platform
 import re
 import sys
 
 import setuptools
 # monkey patch setuptools to use distutils owner/group functionality
 from setuptools.command import sdist
 
 
 if sys.version_info < (3, 6):
     raise Exception('Kallithea requires Python 3.6 or later')
 
 
 here = os.path.abspath(os.path.dirname(__file__))
 
 
 def _get_meta_var(name, data, callback_handler=None):
     matches = re.compile(r'(?:%s)\s*=\s*(.*)' % name).search(data)
     if matches:
         s = eval(matches.groups()[0])
         if callable(callback_handler):
             return callback_handler(s)
         return s
 
 _meta = open(os.path.join(here, 'kallithea', '__init__.py'), 'r')
 _metadata = _meta.read()
 _meta.close()
 
 def callback(V):
     return '.'.join(map(str, V[:3])) + '.'.join(V[3:])
 __version__ = _get_meta_var('VERSION', _metadata, callback)
 __license__ = _get_meta_var('__license__', _metadata)
 __author__ = _get_meta_var('__author__', _metadata)
 __url__ = _get_meta_var('__url__', _metadata)
 # defines current platform
 __platform__ = platform.system()
 
 is_windows = __platform__ in ['Windows']
 
 requirements = [
     "alembic >= 1.0.10, < 1.5",
     "gearbox >= 0.1.0, < 1",
     "waitress >= 0.8.8, < 1.5",
     "WebOb >= 1.8, < 1.9",
     "backlash >= 0.1.2, < 1",
     "TurboGears2 >= 2.4, < 2.5",
     "tgext.routes >= 0.2.0, < 1",
     "Beaker >= 1.10.1, < 2",
     "WebHelpers2 >= 2.0, < 2.1",
     "FormEncode >= 1.3.1, < 1.4",
     "SQLAlchemy >= 1.2.9, < 1.4",
     "Mako >= 0.9.1, < 1.2",
     "Pygments >= 2.2.0, < 2.7",
     "Whoosh >= 2.7.1, < 2.8",
-    "celery >= 4.3, < 4.5, != 4.4.4", # 4.4.4 is broken due to unexpressed dependency on 'future', see https://github.com/celery/celery/pull/6146
+    "celery >= 5, < 5.1",
     "Babel >= 1.3, < 2.9",
     "python-dateutil >= 2.1.0, < 2.9",
     "Markdown >= 2.2.1, < 3.2",
     "docutils >= 0.11, < 0.17",
     "URLObject >= 2.3.4, < 2.5",
     "Routes >= 2.0, < 2.5",
     "dulwich >= 0.19.0, < 0.20",
     "mercurial >= 5.2, < 5.7",
     "decorator >= 4.2.1, < 4.5",
     "Paste >= 2.0.3, < 3.5",
     "bleach >= 3.0, < 3.1.4",
     "Click >= 7.0, < 8",
     "ipaddr >= 2.2.0, < 2.3",
     "paginate >= 0.5, < 0.6",
     "paginate_sqlalchemy >= 0.3.0, < 0.4",
     "bcrypt >= 3.1.0, < 3.2",
     "pip >= 20.0, < 999",
 ]
 
 dependency_links = [
 ]
 
 classifiers = [
     'Development Status :: 4 - Beta',
     'Environment :: Web Environment',
     'Framework :: Pylons',
     'Intended Audience :: Developers',
     'License :: OSI Approved :: GNU General Public License (GPL)',
     'Operating System :: OS Independent',
     'Programming Language :: Python :: 3.6',
     'Programming Language :: Python :: 3.7',
     'Programming Language :: Python :: 3.8',
     'Topic :: Software Development :: Version Control',
 ]
 
 
 # additional files from project that goes somewhere in the filesystem
 # relative to sys.prefix
 data_files = []
 
 description = ('Kallithea is a fast and powerful management tool '
                'for Mercurial and Git with a built in push/pull server, '
                'full text search and code-review.')
 
 keywords = ' '.join([
     'kallithea', 'mercurial', 'git', 'code review',
     'repo groups', 'ldap', 'repository management', 'hgweb replacement',
     'hgwebdir', 'gitweb replacement', 'serving hgweb',
 ])
 
 # long description
 README_FILE = 'README.rst'
 try:
     long_description = open(README_FILE).read()
 except IOError as err:
     sys.stderr.write(
         "[WARNING] Cannot find file specified as long_description (%s): %s\n"
         % (README_FILE, err)
     )
     long_description = description
 
 
 sdist_org = sdist.sdist
 class sdist_new(sdist_org):
     def initialize_options(self):
         sdist_org.initialize_options(self)
         self.owner = self.group = 'root'
 sdist.sdist = sdist_new
 
 packages = setuptools.find_packages(exclude=['ez_setup'])
 
 setuptools.setup(
     name='Kallithea',
     version=__version__,
     description=description,
     long_description=long_description,
     keywords=keywords,
     license=__license__,
     author=__author__,
     author_email='[e-mail]',
     dependency_links=dependency_links,
     url=__url__,
     install_requires=requirements,
     classifiers=classifiers,
     data_files=data_files,
     packages=packages,
     include_package_data=True,
     message_extractors={'kallithea': [
             ('**.py', 'python', None),
             ('templates/**.mako', 'mako', {'input_encoding': 'utf-8'}),
             ('templates/**.html', 'mako', {'input_encoding': 'utf-8'}),
             ('public/**', 'ignore', None)]},
     zip_safe=False,
     entry_points="""
     [console_scripts]
     kallithea-api =    kallithea.bin.kallithea_api:main
     kallithea-gist =   kallithea.bin.kallithea_gist:main
     kallithea-cli =    kallithea.bin.kallithea_cli:cli
 
     [paste.app_factory]
     main = kallithea.config.application:make_app
     """,
 )