celery: bumped to 4.4.7 and fixed loader logic/logging

pkgs/python-packages.nix

@@ -22,14 +22,14 b' self: super: {'
     };
   };
   "amqp" = super.buildPythonPackage {
-    name = "amqp-2.5.2";
+    name = "amqp-2.6.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."vine"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/92/1d/433541994a5a69f4ad2fff39746ddbb0bdedb0ea0d85673eb0db68a7edd9/amqp-2.5.2.tar.gz";
-      sha256 = "13dhhfxjrqcjybnq4zahg92mydhpg2l76nxcmq7d560687wsxwbp";
+      url = "https://files.pythonhosted.org/packages/37/9f/d54494a157d0dcd1673fe7a1bcce7ac70d3eb6d5d6149749450c87a2c959/amqp-2.6.1.tar.gz";
+      sha256 = "70cdb10628468ff14e57ec2f751c7aa9e48e7e3651cfd62d431213c0c4e58f21";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
@@ -149,11 +149,11 b' self: super: {'
     };
   };
   "billiard" = super.buildPythonPackage {
-    name = "billiard-3.6.1.0";
+    name = "billiard-3.6.4.0";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/68/1d/2aea8fbb0b1e1260a8a2e77352de2983d36d7ac01207cf14c2b9c6cc860e/billiard-3.6.1.0.tar.gz";
-      sha256 = "09hzy3aqi7visy4vmf4xiish61n0rq5nd3iwjydydps8yrs9r05q";
+      url = "https://files.pythonhosted.org/packages/92/91/40de1901da8ec9eeb7c6a22143ba5d55d8aaa790761ca31342cedcd5c793/billiard-3.6.4.0.tar.gz";
+      sha256 = "299de5a8da28a783d51b197d496bef4f1595dd023a93a4f59dde1886ae905547";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
@@ -197,7 +197,7 b' self: super: {'
     };
   };
   "celery" = super.buildPythonPackage {
-    name = "celery-4.3.0";
+    name = "celery-4.4.7";
     doCheck = false;
     propagatedBuildInputs = [
       self."pytz"
@@ -206,8 +206,8 b' self: super: {'
       self."vine"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/a2/4b/d020836f751617e907e84753a41c92231cd4b673ff991b8ee9da52361323/celery-4.3.0.tar.gz";
-      sha256 = "1y8y0gbgkwimpxqnxq2rm5qz2vy01fvjiybnpm00y5rzd2m34iac";
+      url = "https://files.pythonhosted.org/packages/fe/58/c7ced9705c2cedf526e183e428d1b145910cb8bc7ea537a2ec9a6552c056/celery-4.4.7.tar.gz";
+      sha256 = "d220b13a8ed57c78149acf82c006785356071844afe0b27012a4991d44026f9f";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
@@ -938,15 +938,15 b' self: super: {'
     };
   };
   "kombu" = super.buildPythonPackage {
-    name = "kombu-4.6.6";
+    name = "kombu-4.6.10";
     doCheck = false;
     propagatedBuildInputs = [
       self."amqp"
       self."importlib-metadata"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/20/e6/bc2d9affba6138a1dc143f77fef253e9e08e238fa7c0688d917c09005e96/kombu-4.6.6.tar.gz";
-      sha256 = "11mxpcy8mg1l35bgbhba70v29bydr2hrhdbdlb4lg98m3m5vaq0p";
+      url = "https://files.pythonhosted.org/packages/73/ec/22996f5f58669d3acc7f0e58adb919bfa7d3c5744b0a6384740690db4748/kombu-4.6.10.tar.gz";
+      sha256 = "437b9cdea193cc2ed0b8044c85fd0f126bb3615ca2f4d4a35b39de7cacfa3c1a";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];

requirements.txt

@@ -1,10 +1,10 b''
 ## dependencies
 
-amqp==2.5.2
+amqp==2.6.1
 babel==1.3
 beaker==1.9.1
 bleach==3.1.3
-celery==4.3.0
+celery==4.4.7
 channelstream==0.6.14
 click==7.0
 colander==1.7.0
@@ -22,7 +22,7 b' futures==3.0.2'
 infrae.cache==1.0.1
 iso8601==0.1.12
 itsdangerous==1.1.0
-kombu==4.6.6
+kombu==4.6.10
 lxml==4.2.5
 mako==1.1.0
 markdown==2.6.11

requirements_pinned.txt

@@ -3,7 +3,7 b''
 atomicwrites==1.3.0
 attrs==19.3.0
 asn1crypto==0.24.0
-billiard==3.6.1.0
+billiard==3.6.4.0
 cffi==1.12.3
 chameleon==2.24
 configparser==4.0.2

rhodecode/lib/celerylib/loader.py

@@ -55,30 +55,6 b" register('json_ext', json.dumps, json.lo"
 log = logging.getLogger('celery.rhodecode.loader')
 
 
-def add_preload_arguments(parser):
-    parser.add_argument(
-        '--ini', default=None,
-        help='Path to ini configuration file.'
-    )
-    parser.add_argument(
-        '--ini-var', default=None,
-        help='Comma separated list of key=value to pass to ini.'
-    )
-
-
-def get_logger(obj):
-    custom_log = logging.getLogger(
-        'rhodecode.task.{}'.format(obj.__class__.__name__))
-
-    if rhodecode.CELERY_ENABLED:
-        try:
-            custom_log = obj.get_logger()
-        except Exception:
-            pass
-
-    return custom_log
-
-
 imports = ['rhodecode.lib.celerylib.tasks']
 
 try:
@@ -104,26 +80,50 b' base_celery_config = {'
         'group': 'beat_groupmeta',
     }
 }
+
+
+def add_preload_arguments(parser):
+    parser.add_argument(
+        '--ini', default=None,
+        help='Path to ini configuration file.'
+    )
+    parser.add_argument(
+        '--ini-var', default=None,
+        help='Comma separated list of key=value to pass to ini.'
+    )
+
+
+def get_logger(obj):
+    custom_log = logging.getLogger(
+        'rhodecode.task.{}'.format(obj.__class__.__name__))
+
+    if rhodecode.CELERY_ENABLED:
+        try:
+            custom_log = obj.get_logger()
+        except Exception:
+            pass
+
+    return custom_log
+
+
 # init main celery app
 celery_app = Celery()
 celery_app.user_options['preload'].add(add_preload_arguments)
-ini_file_glob = None
 
 
 @signals.setup_logging.connect
 def setup_logging_callback(**kwargs):
-    setup_logging(ini_file_glob)
+    ini_file = celery_app.conf['RC_INI_FILE']
+    setup_logging(ini_file)
 
 
 @signals.user_preload_options.connect
 def on_preload_parsed(options, **kwargs):
-    from rhodecode.config.middleware import get_celery_config
 
-    ini_location = options['ini']
+    ini_file = options['ini']
     ini_vars = options['ini_var']
-    celery_app.conf['INI_PYRAMID'] = options['ini']
 
-    if ini_location is None:
+    if ini_file is None:
         print('You must provide the paste --ini argument')
         exit(-1)
 
@@ -131,16 +131,24 b' def on_preload_parsed(options, **kwargs)'
     if ini_vars is not None:
         options = parse_ini_vars(ini_vars)
 
-    global ini_file_glob
-    ini_file_glob = ini_location
+    celery_app.conf['RC_INI_FILE'] = ini_file
+    celery_app.conf['RC_INI_OPTIONS'] = options
+    setup_logging(ini_file)
+
+
[e-mail]_init.connect
+def on_celeryd_init(sender=None, conf=None, **kwargs):
+    from rhodecode.config.middleware import get_celery_config
 
     log.debug('Bootstrapping RhodeCode application...')
 
+    ini_file = conf['RC_INI_FILE']
+    options = conf['RC_INI_OPTIONS']
+
     try:
-        env = bootstrap(ini_location, options=options)
+        env = bootstrap(ini_file, options=options)
     except Exception:
         log.exception('Failed to bootstrap RhodeCode APP')
-        raise
 
     log.debug('Got Pyramid ENV: %s', env)