diff --git a/pkgs/python-packages.nix b/pkgs/python-packages.nix --- a/pkgs/python-packages.nix +++ b/pkgs/python-packages.nix @@ -22,14 +22,14 @@ self: super: { }; }; "amqp" = super.buildPythonPackage { - name = "amqp-2.5.2"; + name = "amqp-2.6.1"; doCheck = false; propagatedBuildInputs = [ self."vine" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/92/1d/433541994a5a69f4ad2fff39746ddbb0bdedb0ea0d85673eb0db68a7edd9/amqp-2.5.2.tar.gz"; - sha256 = "13dhhfxjrqcjybnq4zahg92mydhpg2l76nxcmq7d560687wsxwbp"; + url = "https://files.pythonhosted.org/packages/37/9f/d54494a157d0dcd1673fe7a1bcce7ac70d3eb6d5d6149749450c87a2c959/amqp-2.6.1.tar.gz"; + sha256 = "70cdb10628468ff14e57ec2f751c7aa9e48e7e3651cfd62d431213c0c4e58f21"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; @@ -149,11 +149,11 @@ self: super: { }; }; "billiard" = super.buildPythonPackage { - name = "billiard-3.6.1.0"; + name = "billiard-3.6.4.0"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/68/1d/2aea8fbb0b1e1260a8a2e77352de2983d36d7ac01207cf14c2b9c6cc860e/billiard-3.6.1.0.tar.gz"; - sha256 = "09hzy3aqi7visy4vmf4xiish61n0rq5nd3iwjydydps8yrs9r05q"; + url = "https://files.pythonhosted.org/packages/92/91/40de1901da8ec9eeb7c6a22143ba5d55d8aaa790761ca31342cedcd5c793/billiard-3.6.4.0.tar.gz"; + sha256 = "299de5a8da28a783d51b197d496bef4f1595dd023a93a4f59dde1886ae905547"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; @@ -197,7 +197,7 @@ self: super: { }; }; "celery" = super.buildPythonPackage { - name = "celery-4.3.0"; + name = "celery-4.4.7"; doCheck = false; propagatedBuildInputs = [ self."pytz" @@ -206,8 +206,8 @@ self: super: { self."vine" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/a2/4b/d020836f751617e907e84753a41c92231cd4b673ff991b8ee9da52361323/celery-4.3.0.tar.gz"; - sha256 = "1y8y0gbgkwimpxqnxq2rm5qz2vy01fvjiybnpm00y5rzd2m34iac"; + url = "https://files.pythonhosted.org/packages/fe/58/c7ced9705c2cedf526e183e428d1b145910cb8bc7ea537a2ec9a6552c056/celery-4.4.7.tar.gz"; + sha256 = "d220b13a8ed57c78149acf82c006785356071844afe0b27012a4991d44026f9f"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; @@ -938,15 +938,15 @@ self: super: { }; }; "kombu" = super.buildPythonPackage { - name = "kombu-4.6.6"; + name = "kombu-4.6.10"; doCheck = false; propagatedBuildInputs = [ self."amqp" self."importlib-metadata" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/20/e6/bc2d9affba6138a1dc143f77fef253e9e08e238fa7c0688d917c09005e96/kombu-4.6.6.tar.gz"; - sha256 = "11mxpcy8mg1l35bgbhba70v29bydr2hrhdbdlb4lg98m3m5vaq0p"; + url = "https://files.pythonhosted.org/packages/73/ec/22996f5f58669d3acc7f0e58adb919bfa7d3c5744b0a6384740690db4748/kombu-4.6.10.tar.gz"; + sha256 = "437b9cdea193cc2ed0b8044c85fd0f126bb3615ca2f4d4a35b39de7cacfa3c1a"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ ## dependencies -amqp==2.5.2 +amqp==2.6.1 babel==1.3 beaker==1.9.1 bleach==3.1.3 -celery==4.3.0 +celery==4.4.7 channelstream==0.6.14 click==7.0 colander==1.7.0 @@ -22,7 +22,7 @@ futures==3.0.2 infrae.cache==1.0.1 iso8601==0.1.12 itsdangerous==1.1.0 -kombu==4.6.6 +kombu==4.6.10 lxml==4.2.5 mako==1.1.0 markdown==2.6.11 diff --git a/requirements_pinned.txt b/requirements_pinned.txt --- a/requirements_pinned.txt +++ b/requirements_pinned.txt @@ -3,7 +3,7 @@ atomicwrites==1.3.0 attrs==19.3.0 asn1crypto==0.24.0 -billiard==3.6.1.0 +billiard==3.6.4.0 cffi==1.12.3 chameleon==2.24 configparser==4.0.2 diff --git a/rhodecode/lib/celerylib/loader.py b/rhodecode/lib/celerylib/loader.py --- a/rhodecode/lib/celerylib/loader.py +++ b/rhodecode/lib/celerylib/loader.py @@ -55,30 +55,6 @@ register('json_ext', json.dumps, json.lo log = logging.getLogger('celery.rhodecode.loader') -def add_preload_arguments(parser): - parser.add_argument( - '--ini', default=None, - help='Path to ini configuration file.' - ) - parser.add_argument( - '--ini-var', default=None, - help='Comma separated list of key=value to pass to ini.' - ) - - -def get_logger(obj): - custom_log = logging.getLogger( - 'rhodecode.task.{}'.format(obj.__class__.__name__)) - - if rhodecode.CELERY_ENABLED: - try: - custom_log = obj.get_logger() - except Exception: - pass - - return custom_log - - imports = ['rhodecode.lib.celerylib.tasks'] try: @@ -104,26 +80,50 @@ base_celery_config = { 'group': 'beat_groupmeta', } } + + +def add_preload_arguments(parser): + parser.add_argument( + '--ini', default=None, + help='Path to ini configuration file.' + ) + parser.add_argument( + '--ini-var', default=None, + help='Comma separated list of key=value to pass to ini.' + ) + + +def get_logger(obj): + custom_log = logging.getLogger( + 'rhodecode.task.{}'.format(obj.__class__.__name__)) + + if rhodecode.CELERY_ENABLED: + try: + custom_log = obj.get_logger() + except Exception: + pass + + return custom_log + + # init main celery app celery_app = Celery() celery_app.user_options['preload'].add(add_preload_arguments) -ini_file_glob = None @signals.setup_logging.connect def setup_logging_callback(**kwargs): - setup_logging(ini_file_glob) + ini_file = celery_app.conf['RC_INI_FILE'] + setup_logging(ini_file) @signals.user_preload_options.connect def on_preload_parsed(options, **kwargs): - from rhodecode.config.middleware import get_celery_config - ini_location = options['ini'] + ini_file = options['ini'] ini_vars = options['ini_var'] - celery_app.conf['INI_PYRAMID'] = options['ini'] - if ini_location is None: + if ini_file is None: print('You must provide the paste --ini argument') exit(-1) @@ -131,16 +131,24 @@ def on_preload_parsed(options, **kwargs) if ini_vars is not None: options = parse_ini_vars(ini_vars) - global ini_file_glob - ini_file_glob = ini_location + celery_app.conf['RC_INI_FILE'] = ini_file + celery_app.conf['RC_INI_OPTIONS'] = options + setup_logging(ini_file) + + +@signals.celeryd_init.connect +def on_celeryd_init(sender=None, conf=None, **kwargs): + from rhodecode.config.middleware import get_celery_config log.debug('Bootstrapping RhodeCode application...') + ini_file = conf['RC_INI_FILE'] + options = conf['RC_INI_OPTIONS'] + try: - env = bootstrap(ini_location, options=options) + env = bootstrap(ini_file, options=options) except Exception: log.exception('Failed to bootstrap RhodeCode APP') - raise log.debug('Got Pyramid ENV: %s', env)