celery: upgrade to Celery 3

Celery 3 is more than 2 and is the future and should be more stable than Celery 2.

There might be problems like
  PRECONDITION_FAILED - cannot redeclare exchange 'celeryresults' in vhost 'kallithea' with different type, durable, internal or autodelete value
when celery is upgraded and it tries to upgrade existing tasks.

One way to avoid that problem:
- make sure celery has run to completion and there are no important pending tasks,
- delete the old vhost,
- upgrade celery,
- create new vhost
- set permissions for vhost
- profit

kallithea/lib/celerypylons/__init__.py

@@ -1,36 +1,36 b''
 # -*- coding: utf-8 -*-
 
 """
 Kallithea wrapper of Celery
 
 The Celery configuration is in the normal Pylons ini file. We thus have to set
 the `CELERY_LOADER` environment variable to point at a custom "loader" that can
 read it. That environment variable must be set *before* importing celery. To
 ensure that, we wrap celery in this module.
 
 Also, the loader depends on Pylons being configured to it can read the Celery
 configuration out of it. To make sure that really is the case and give an early
 warning, we check one of the mandatory settings.
 
 This module must thus not be imported in global scope but must be imported on
 demand in function scope.
 """
 
 import os
 import warnings
 
 # Verify Pylons configuration has been loaded
 from pylons import config
 assert config['celery.imports'] == 'kallithea.lib.celerylib.tasks', 'Kallithea Celery configuration has not been loaded'
 
 # Prepare environment to point at Kallithea Pylons loader
 CELERYPYLONS_LOADER = 'kallithea.lib.celerypylons.loader.PylonsLoader'
 if os.environ.get('CELERY_LOADER', CELERYPYLONS_LOADER) != CELERYPYLONS_LOADER:
     warnings.warn("'CELERY_LOADER' environment variable will be overridden by celery-pylons.")
 os.environ['CELERY_LOADER'] = CELERYPYLONS_LOADER
 
 # Import (and expose) celery, thus immediately triggering use of the custom Pylons loader
 import celery.app as app
 import celery.result as result
 from celery.task import task
-from celery.bin import celeryd
+from celery.bin import worker

kallithea/lib/paster_commands/celeryd.py

@@ -1,46 +1,46 b''
 # -*- coding: utf-8 -*-
 
 import kallithea
 from kallithea.lib.paster_commands.common import BasePasterCommand
 from kallithea.lib.utils import load_rcextensions
 from kallithea.lib.utils2 import str2bool
 
 __all__ = ['Command']
 
 
 class Command(BasePasterCommand):
     """Start the celery worker
 
     Starts the celery worker that uses a paste.deploy configuration
     file.
     """
 
     usage = 'CONFIG_FILE [celeryd options...]'
     summary = __doc__.splitlines()[0]
     description = "".join(__doc__.splitlines()[2:])
     group_name = "Kallithea"
 
     parser = BasePasterCommand.standard_parser(quiet=True)
 
     def update_parser(self):
         from kallithea.lib import celerypylons
-        cmd = celerypylons.celeryd.WorkerCommand(celerypylons.app.app_or_default())
+        cmd = celerypylons.worker.worker(celerypylons.app.app_or_default())
         for x in cmd.get_options():
             self.parser.add_option(x)
 
     def command(self):
         from kallithea.lib import celerypylons
         from pylons import config
         try:
             CELERY_ON = str2bool(config['app_conf'].get('use_celery'))
         except KeyError:
             CELERY_ON = False
 
         if not CELERY_ON:
             raise Exception('Please set use_celery = true in .ini config '
                             'file before running celeryd')
         kallithea.CELERY_ON = CELERY_ON
 
         load_rcextensions(config['here'])
-        cmd = celerypylons.celeryd.WorkerCommand(celerypylons.app.app_or_default())
+        cmd = celerypylons.worker.worker(celerypylons.app.app_or_default())
         return cmd.run(**vars(self.options))

setup.py

@@ -1,172 +1,172 b''
 #!/usr/bin/env python2
 # -*- coding: utf-8 -*-
 import os
 import sys
 import platform
 
 if sys.version_info < (2, 6) or sys.version_info >= (3,):
     raise Exception('Kallithea requires python 2.6 or 2.7')
 
 
 here = os.path.abspath(os.path.dirname(__file__))
 
 
 def _get_meta_var(name, data, callback_handler=None):
     import re
     matches = re.compile(r'(?:%s)\s*=\s*(.*)' % name).search(data)
     if matches:
         if not callable(callback_handler):
             callback_handler = lambda v: v
 
         return callback_handler(eval(matches.groups()[0]))
 
 _meta = open(os.path.join(here, 'kallithea', '__init__.py'), 'rb')
 _metadata = _meta.read()
 _meta.close()
 
 callback = lambda V: ('.'.join(map(str, V[:3])) + '.'.join(V[3:]))
 __version__ = _get_meta_var('VERSION', _metadata, callback)
 __license__ = _get_meta_var('__license__', _metadata)
 __author__ = _get_meta_var('__author__', _metadata)
 __url__ = _get_meta_var('__url__', _metadata)
 # defines current platform
 __platform__ = platform.system()
 
 is_windows = __platform__ in ['Windows']
 
 requirements = [
     "alembic>=0.8.0,<0.9",
     "waitress>=0.8.8,<1.0",
     "webob>=1.0.8,<=1.1.1",
     "Pylons>=1.0.0,<=1.0.2",
     "WebTest<2.0", # make the Pylons dependency pick a version that supports our WebOb version
     "Beaker==1.6.4",
     "WebHelpers==1.3",
     "formencode>=1.2.4,<=1.2.6",
     "SQLAlchemy>=1.0,<1.1",
     "Mako>=0.9.0,<=1.0.0",
     "pygments>=1.5",
     "whoosh>=2.4.0,<=2.5.7",
-    "celery>=2.2.5,<2.3",
+    "celery>=3.1,<3.2",
     "babel>=0.9.6,<=1.3",
     "python-dateutil>=1.5.0,<2.0.0",
     "markdown==2.2.1",
     "docutils>=0.8.1,<=0.11",
     "URLObject==2.3.4",
     "Routes==1.13",
     "dulwich>=0.14.1",
     "mercurial>=2.9,<4.0",
 ]
 
 if sys.version_info < (2, 7):
     requirements.append("importlib==1.0.1")
     requirements.append("argparse")
 
 if not is_windows:
     requirements.append("bcrypt>=2.0.0")
 
 dependency_links = [
 ]
 
 classifiers = [
     'Development Status :: 4 - Beta',
     'Environment :: Web Environment',
     'Framework :: Pylons',
     'Intended Audience :: Developers',
     'License :: OSI Approved :: GNU General Public License (GPL)',
     'Operating System :: OS Independent',
     'Programming Language :: Python',
     'Programming Language :: Python :: 2.6',
     'Programming Language :: Python :: 2.7',
     'Topic :: Software Development :: Version Control',
 ]
 
 
 # additional files from project that goes somewhere in the filesystem
 # relative to sys.prefix
 data_files = []
 
 description = ('Kallithea is a fast and powerful management tool '
                'for Mercurial and Git with a built in push/pull server, '
                'full text search and code-review.')
 
 keywords = ' '.join([
     'kallithea', 'mercurial', 'git', 'code review',
     'repo groups', 'ldap', 'repository management', 'hgweb replacement',
     'hgwebdir', 'gitweb replacement', 'serving hgweb',
 ])
 
 # long description
 README_FILE = 'README.rst'
 try:
     long_description = open(README_FILE).read()
 except IOError as err:
     sys.stderr.write(
         "[WARNING] Cannot find file specified as long_description (%s)\n"
         % README_FILE
     )
     long_description = description
 
 import setuptools
 
 # monkey patch setuptools to use distutils owner/group functionality
 from setuptools.command import sdist
 sdist_org = sdist.sdist
 class sdist_new(sdist_org):
     def initialize_options(self):
         sdist_org.initialize_options(self)
         self.owner = self.group = 'root'
 sdist.sdist = sdist_new
 
 packages = setuptools.find_packages(exclude=['ez_setup'])
 
 setuptools.setup(
     name='Kallithea',
     version=__version__,
     description=description,
     long_description=long_description,
     keywords=keywords,
     license=__license__,
     author=__author__,
     author_email='[e-mail]',
     dependency_links=dependency_links,
     url=__url__,
     install_requires=requirements,
     classifiers=classifiers,
     setup_requires=['PasteScript>=1.6.3'],
     data_files=data_files,
     packages=packages,
     include_package_data=True,
     message_extractors={'kallithea': [
             ('**.py', 'python', None),
             ('templates/**.mako', 'mako', {'input_encoding': 'utf-8'}),
             ('templates/**.html', 'mako', {'input_encoding': 'utf-8'}),
             ('public/**', 'ignore', None)]},
     zip_safe=False,
     paster_plugins=['PasteScript', 'Pylons'],
     entry_points="""
     [console_scripts]
     kallithea-api =    kallithea.bin.kallithea_api:main
     kallithea-gist =   kallithea.bin.kallithea_gist:main
     kallithea-config = kallithea.bin.kallithea_config:main
 
     [paste.app_factory]
     main = kallithea.config.middleware:make_app
 
     [paste.app_install]
     main = pylons.util:PylonsInstaller
 
     [paste.global_paster_command]
     setup-db=kallithea.lib.paster_commands.setup_db:Command
     cleanup-repos=kallithea.lib.paster_commands.cleanup:Command
     update-repoinfo=kallithea.lib.paster_commands.update_repoinfo:Command
     make-rcext=kallithea.lib.paster_commands.make_rcextensions:Command
     repo-scan=kallithea.lib.paster_commands.repo_scan:Command
     cache-keys=kallithea.lib.paster_commands.cache_keys:Command
     ishell=kallithea.lib.paster_commands.ishell:Command
     make-index=kallithea.lib.paster_commands.make_index:Command
     upgrade-db=kallithea.lib.dbmigrate:UpgradeDb
     celeryd=kallithea.lib.paster_commands.celeryd:Command
     install-iis=kallithea.lib.paster_commands.install_iis:Command
     """,
 )