##// END OF EJS Templates
packages: updated celery to 4.3.0 and switch default backend to redis...
marcink -
r3913:dfb2da80 default
parent child
Show More
@@ -307,8 +307,11 file_store.storage_path = %(here)s/data/
307
307
308 use_celery = false
308 use_celery = false
309
309
310 ## connection url to the message broker (default rabbitmq)
310 ## connection url to the message broker (default redis)
311 celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost
311 celery.broker_url = redis://localhost:6379/8
312
313 ## rabbitmq example
314 #celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost
312
315
313 ## maximum tasks to execute before worker restart
316 ## maximum tasks to execute before worker restart
314 celery.max_tasks_per_child = 100
317 celery.max_tasks_per_child = 100
@@ -282,8 +282,11 file_store.storage_path = %(here)s/data/
282
282
283 use_celery = false
283 use_celery = false
284
284
285 ## connection url to the message broker (default rabbitmq)
285 ## connection url to the message broker (default redis)
286 celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost
286 celery.broker_url = redis://localhost:6379/8
287
288 ## rabbitmq example
289 #celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost
287
290
288 ## maximum tasks to execute before worker restart
291 ## maximum tasks to execute before worker restart
289 celery.max_tasks_per_child = 100
292 celery.max_tasks_per_child = 100
@@ -57,7 +57,7 Here's an overview what components shoul
57 - `nginx` acting as a load-balancer.
57 - `nginx` acting as a load-balancer.
58 - `postgresql-server` used for database and sessions.
58 - `postgresql-server` used for database and sessions.
59 - `redis-server` used for storing shared caches.
59 - `redis-server` used for storing shared caches.
60 - optionally `rabbitmq-server` for `Celery` if used.
60 - optionally `rabbitmq-server` or `redis` for `Celery` if used.
61 - optionally if `Celery` is used Enterprise/Community instance + VCSServer.
61 - optionally if `Celery` is used Enterprise/Community instance + VCSServer.
62 - optionally mailserver that can be shared by other instances.
62 - optionally mailserver that can be shared by other instances.
63 - optionally channelstream server to handle live communication for all instances.
63 - optionally channelstream server to handle live communication for all instances.
@@ -372,16 +372,16 Using Celery with cluster
372
372
373
373
374 If `Celery` is used we recommend setting also an instance of Enterprise/Community+VCSserver
374 If `Celery` is used we recommend setting also an instance of Enterprise/Community+VCSserver
375 on the node that is running `RabbitMQ`_. Those instances will be used to executed async
375 on the node that is running `RabbitMQ`_ or `Redis`_. Those instances will be used to
376 tasks on the `rc-node-1`. This is the most efficient setup. `Celery` usually
376 executed async tasks on the `rc-node-1`. This is the most efficient setup.
377 handles tasks such as sending emails, forking repositories, importing
377 `Celery` usually handles tasks such as sending emails, forking repositories, importing
378 repositories from external location etc. Using workers on instance that has
378 repositories from external location etc. Using workers on instance that has
379 the direct access to disks used by NFS as well as email server gives noticeable
379 the direct access to disks used by NFS as well as email server gives noticeable
380 performance boost. Running local workers to the NFS storage results in faster
380 performance boost. Running local workers to the NFS storage results in faster
381 execution of forking large repositories or sending lots of emails.
381 execution of forking large repositories or sending lots of emails.
382
382
383 Those instances need to be configured in the same way as for other nodes.
383 Those instances need to be configured in the same way as for other nodes.
384 The instance in rc-node-1 can be added to the cluser, but we don't recommend doing it.
384 The instance in rc-node-1 can be added to the cluster, but we don't recommend doing it.
385 For best results let it be isolated to only executing `Celery` tasks in the cluster setup.
385 For best results let it be isolated to only executing `Celery` tasks in the cluster setup.
386
386
387
387
@@ -11,16 +11,20 and import repositories in async way. It
11 repository sync in scheduler.
11 repository sync in scheduler.
12
12
13 If you decide to use Celery you also need a working message queue.
13 If you decide to use Celery you also need a working message queue.
14 The recommended and fully supported message broker is rabbitmq_.
14 There are two fully supported message brokers is rabbitmq_ and redis_ (recommended).
15
16 Since release 4.18.X we recommend using redis_ as a backend since it's generally
17 easier to work with, and results in simpler stack as redis is generally recommended
18 for caching purposes.
15
19
16
20
17 In order to install and configure Celery, follow these steps:
21 In order to install and configure Celery, follow these steps:
18
22
19 1. Install RabbitMQ, see the documentation on the Celery website for
23 1. Install RabbitMQ or Redis for a message queue, see the documentation on the Celery website for
20 `rabbitmq installation`_, or `rabbitmq website installation`_
24 `redis installation`_ or `rabbitmq installation`_
21
25
22
26
23 1a. As en example configuration after installation, you can run::
27 1a. If you choose RabbitMQ example configuration after installation would look like that::
24
28
25 sudo rabbitmqctl add_user rcuser secret_password
29 sudo rabbitmqctl add_user rcuser secret_password
26 sudo rabbitmqctl add_vhost rhodevhost
30 sudo rabbitmqctl add_vhost rhodevhost
@@ -45,6 +49,10 3. Configure Celery in the
45 Set the broker_url as minimal settings required to enable operation.
49 Set the broker_url as minimal settings required to enable operation.
46 If used our example data from pt 1a, here is how the broker url should look like::
50 If used our example data from pt 1a, here is how the broker url should look like::
47
51
52 # for Redis
53 celery.broker_url = redis://localhost:6379/8
54
55 # for RabbitMQ
48 celery.broker_url = amqp://rcuser:secret_password@localhost:5672/rhodevhost
56 celery.broker_url = amqp://rcuser:secret_password@localhost:5672/rhodevhost
49
57
50 Full configuration example is below:
58 Full configuration example is below:
@@ -57,7 +65,7 3. Configure Celery in the
57 ####################################
65 ####################################
58
66
59 use_celery = true
67 use_celery = true
60 celery.broker_url = amqp://rcuser:secret@localhost:5672/rhodevhost
68 celery.broker_url = redis://localhost:6379/8
61
69
62 # maximum tasks to execute before worker restart
70 # maximum tasks to execute before worker restart
63 celery.max_tasks_per_child = 100
71 celery.max_tasks_per_child = 100
@@ -69,6 +77,8 3. Configure Celery in the
69 .. _python: http://www.python.org/
77 .. _python: http://www.python.org/
70 .. _mercurial: http://mercurial.selenic.com/
78 .. _mercurial: http://mercurial.selenic.com/
71 .. _celery: http://celeryproject.org/
79 .. _celery: http://celeryproject.org/
80 .. _redis: http://redis.io
81 .. _redis installation: https://redis.io/topics/quickstart
72 .. _rabbitmq: http://www.rabbitmq.com/
82 .. _rabbitmq: http://www.rabbitmq.com/
73 .. _rabbitmq installation: http://docs.celeryproject.org/en/latest/getting-started/brokers/rabbitmq.html
83 .. _rabbitmq installation: http://docs.celeryproject.org/en/latest/getting-started/brokers/rabbitmq.html
74 .. _rabbitmq website installation: http://www.rabbitmq.com/download.html
84 .. _rabbitmq website installation: http://www.rabbitmq.com/download.html
@@ -22,14 +22,14 self: super: {
22 };
22 };
23 };
23 };
24 "amqp" = super.buildPythonPackage {
24 "amqp" = super.buildPythonPackage {
25 name = "amqp-2.3.1";
25 name = "amqp-2.5.1";
26 doCheck = false;
26 doCheck = false;
27 propagatedBuildInputs = [
27 propagatedBuildInputs = [
28 self."vine"
28 self."vine"
29 ];
29 ];
30 src = fetchurl {
30 src = fetchurl {
31 url = "https://files.pythonhosted.org/packages/1b/32/242ff76cd802766f11c89c72f3389b5c8de4bdfbab406137b90c5fae8b05/amqp-2.3.1.tar.gz";
31 url = "https://files.pythonhosted.org/packages/b5/f5/70e364a1f5fbafc742c098ad88a064b801b0d69cf56bfad13be2c08be4e2/amqp-2.5.1.tar.gz";
32 sha256 = "0wlfnvhmfrn7c8qif2jyvsm63ibdxp02ss564qwrvqfhz0di72s0";
32 sha256 = "0s2yxnnhhx9hww0n33yn22q6sgnbd6n2nw92050qv2qpc3i1ga8r";
33 };
33 };
34 meta = {
34 meta = {
35 license = [ pkgs.lib.licenses.bsdOriginal ];
35 license = [ pkgs.lib.licenses.bsdOriginal ];
@@ -135,11 +135,11 self: super: {
135 };
135 };
136 };
136 };
137 "billiard" = super.buildPythonPackage {
137 "billiard" = super.buildPythonPackage {
138 name = "billiard-3.5.0.3";
138 name = "billiard-3.6.1.0";
139 doCheck = false;
139 doCheck = false;
140 src = fetchurl {
140 src = fetchurl {
141 url = "https://files.pythonhosted.org/packages/39/ac/f5571210cca2e4f4532e38aaff242f26c8654c5e2436bee966c230647ccc/billiard-3.5.0.3.tar.gz";
141 url = "https://files.pythonhosted.org/packages/68/1d/2aea8fbb0b1e1260a8a2e77352de2983d36d7ac01207cf14c2b9c6cc860e/billiard-3.6.1.0.tar.gz";
142 sha256 = "1riwiiwgb141151md4ykx49qrz749akj5k8g290ji9bsqjyj4yqx";
142 sha256 = "09hzy3aqi7visy4vmf4xiish61n0rq5nd3iwjydydps8yrs9r05q";
143 };
143 };
144 meta = {
144 meta = {
145 license = [ pkgs.lib.licenses.bsdOriginal ];
145 license = [ pkgs.lib.licenses.bsdOriginal ];
@@ -172,16 +172,17 self: super: {
172 };
172 };
173 };
173 };
174 "celery" = super.buildPythonPackage {
174 "celery" = super.buildPythonPackage {
175 name = "celery-4.1.1";
175 name = "celery-4.3.0";
176 doCheck = false;
176 doCheck = false;
177 propagatedBuildInputs = [
177 propagatedBuildInputs = [
178 self."pytz"
178 self."pytz"
179 self."billiard"
179 self."billiard"
180 self."kombu"
180 self."kombu"
181 self."vine"
181 ];
182 ];
182 src = fetchurl {
183 src = fetchurl {
183 url = "https://files.pythonhosted.org/packages/e9/cf/a4c0597effca20c57eb586324e41d1180bc8f13a933da41e0646cff69f02/celery-4.1.1.tar.gz";
184 url = "https://files.pythonhosted.org/packages/a2/4b/d020836f751617e907e84753a41c92231cd4b673ff991b8ee9da52361323/celery-4.3.0.tar.gz";
184 sha256 = "1xbir4vw42n2ir9lanhwl7w69zpmj7lbi66fxm2b7pyvkcss7wni";
185 sha256 = "1y8y0gbgkwimpxqnxq2rm5qz2vy01fvjiybnpm00y5rzd2m34iac";
185 };
186 };
186 meta = {
187 meta = {
187 license = [ pkgs.lib.licenses.bsdOriginal ];
188 license = [ pkgs.lib.licenses.bsdOriginal ];
@@ -284,6 +285,17 self: super: {
284 license = [ pkgs.lib.licenses.mit ];
285 license = [ pkgs.lib.licenses.mit ];
285 };
286 };
286 };
287 };
288 "contextlib2" = super.buildPythonPackage {
289 name = "contextlib2-0.5.5";
290 doCheck = false;
291 src = fetchurl {
292 url = "https://files.pythonhosted.org/packages/6e/db/41233498c210b03ab8b072c8ee49b1cd63b3b0c76f8ea0a0e5d02df06898/contextlib2-0.5.5.tar.gz";
293 sha256 = "0j6ad6lwwyc9kv71skj098v5l7x5biyj2hs4lc5x1kcixqcr97sh";
294 };
295 meta = {
296 license = [ pkgs.lib.licenses.psfl ];
297 };
298 };
287 "cov-core" = super.buildPythonPackage {
299 "cov-core" = super.buildPythonPackage {
288 name = "cov-core-1.15.0";
300 name = "cov-core-1.15.0";
289 doCheck = false;
301 doCheck = false;
@@ -665,6 +677,23 self: super: {
665 license = [ pkgs.lib.licenses.mit ];
677 license = [ pkgs.lib.licenses.mit ];
666 };
678 };
667 };
679 };
680 "importlib-metadata" = super.buildPythonPackage {
681 name = "importlib-metadata-0.20";
682 doCheck = false;
683 propagatedBuildInputs = [
684 self."zipp"
685 self."contextlib2"
686 self."configparser"
687 self."pathlib2"
688 ];
689 src = fetchurl {
690 url = "https://files.pythonhosted.org/packages/05/41/7d339dd7b507e97f67be812fdf29c4ad991ddd34b1ed0f3c54e8f1c4e0b3/importlib_metadata-0.20.tar.gz";
691 sha256 = "13bshj8i98l9gxi6df4xbw1262phmawgr527as20brblwf93a55p";
692 };
693 meta = {
694 license = [ pkgs.lib.licenses.asl20 ];
695 };
696 };
668 "infrae.cache" = super.buildPythonPackage {
697 "infrae.cache" = super.buildPythonPackage {
669 name = "infrae.cache-1.0.1";
698 name = "infrae.cache-1.0.1";
670 doCheck = false;
699 doCheck = false;
@@ -848,14 +877,15 self: super: {
848 };
877 };
849 };
878 };
850 "kombu" = super.buildPythonPackage {
879 "kombu" = super.buildPythonPackage {
851 name = "kombu-4.2.1";
880 name = "kombu-4.6.4";
852 doCheck = false;
881 doCheck = false;
853 propagatedBuildInputs = [
882 propagatedBuildInputs = [
854 self."amqp"
883 self."amqp"
884 self."importlib-metadata"
855 ];
885 ];
856 src = fetchurl {
886 src = fetchurl {
857 url = "https://files.pythonhosted.org/packages/39/9f/556b988833abede4a80dbd18b2bdf4e8ff4486dd482ed45da961347e8ed2/kombu-4.2.1.tar.gz";
887 url = "https://files.pythonhosted.org/packages/52/f2/5a64fc850b0533d2daf09a523406e51e85a8b2a4a2bc87a922a8906ba2aa/kombu-4.6.4.tar.gz";
858 sha256 = "10lh3hncvw67fz0k5vgbx3yh9gjfpqdlia1f13i28cgnc1nfrbc6";
888 sha256 = "16w02mvkxchz7041yia4h8xmqavci88szk18ynxvw4chzcnk3w75";
859 };
889 };
860 meta = {
890 meta = {
861 license = [ pkgs.lib.licenses.bsdOriginal ];
891 license = [ pkgs.lib.licenses.bsdOriginal ];
@@ -1636,11 +1666,11 self: super: {
1636 };
1666 };
1637 };
1667 };
1638 "pytz" = super.buildPythonPackage {
1668 "pytz" = super.buildPythonPackage {
1639 name = "pytz-2018.4";
1669 name = "pytz-2019.2";
1640 doCheck = false;
1670 doCheck = false;
1641 src = fetchurl {
1671 src = fetchurl {
1642 url = "https://files.pythonhosted.org/packages/10/76/52efda4ef98e7544321fd8d5d512e11739c1df18b0649551aeccfb1c8376/pytz-2018.4.tar.gz";
1672 url = "https://files.pythonhosted.org/packages/27/c0/fbd352ca76050952a03db776d241959d5a2ee1abddfeb9e2a53fdb489be4/pytz-2019.2.tar.gz";
1643 sha256 = "0jgpqx3kk2rhv81j1izjxvmx8d0x7hzs1857pgqnixic5wq2ar60";
1673 sha256 = "0ckb27hhjc8i8gcdvk4d9avld62b7k52yjijc60s2m3y8cpb7h16";
1644 };
1674 };
1645 meta = {
1675 meta = {
1646 license = [ pkgs.lib.licenses.mit ];
1676 license = [ pkgs.lib.licenses.mit ];
@@ -2271,6 +2301,20 self: super: {
2271 license = [ { fullName = "PSF or ZPL"; } ];
2301 license = [ { fullName = "PSF or ZPL"; } ];
2272 };
2302 };
2273 };
2303 };
2304 "zipp" = super.buildPythonPackage {
2305 name = "zipp-0.6.0";
2306 doCheck = false;
2307 propagatedBuildInputs = [
2308 self."more-itertools"
2309 ];
2310 src = fetchurl {
2311 url = "https://files.pythonhosted.org/packages/57/dd/585d728479d97d25aeeb9aa470d36a4ad8d0ba5610f84e14770128ce6ff7/zipp-0.6.0.tar.gz";
2312 sha256 = "13ndkf7vklw978a4gdl1yfvn8hch28429a0iam67sg4nrp5v261p";
2313 };
2314 meta = {
2315 license = [ pkgs.lib.licenses.mit ];
2316 };
2317 };
2274 "zope.cachedescriptors" = super.buildPythonPackage {
2318 "zope.cachedescriptors" = super.buildPythonPackage {
2275 name = "zope.cachedescriptors-4.3.1";
2319 name = "zope.cachedescriptors-4.3.1";
2276 doCheck = false;
2320 doCheck = false;
@@ -1,11 +1,11
1 ## dependencies
1 ## dependencies
2
2
3 amqp==2.3.1
3 amqp==2.5.1
4
4
5 babel==1.3
5 babel==1.3
6 beaker==1.9.1
6 beaker==1.9.1
7 bleach==3.1.0
7 bleach==3.1.0
8 celery==4.1.1
8 celery==4.3.0
9 channelstream==0.5.2
9 channelstream==0.5.2
10 click==7.0
10 click==7.0
11 colander==1.7.0
11 colander==1.7.0
@@ -24,7 +24,7 futures==3.0.2
24 infrae.cache==1.0.1
24 infrae.cache==1.0.1
25 iso8601==0.1.12
25 iso8601==0.1.12
26 itsdangerous==0.24
26 itsdangerous==0.24
27 kombu==4.2.1
27 kombu==4.6.4
28 lxml==4.2.5
28 lxml==4.2.5
29 mako==1.0.7
29 mako==1.0.7
30 markdown==2.6.11
30 markdown==2.6.11
@@ -52,7 +52,7 python-ldap==3.1.0
52 python-memcached==1.59
52 python-memcached==1.59
53 python-pam==1.8.4
53 python-pam==1.8.4
54 python-saml==2.4.2
54 python-saml==2.4.2
55 pytz==2018.4
55 pytz==2019.2
56 tzlocal==1.5.1
56 tzlocal==1.5.1
57 pyzmq==14.6.0
57 pyzmq==14.6.0
58 py-gfm==0.1.4
58 py-gfm==0.1.4
@@ -2,7 +2,7
2
2
3 atomicwrites==1.2.1
3 atomicwrites==1.2.1
4 attrs==18.2.0
4 attrs==18.2.0
5 billiard==3.5.0.3
5 billiard==3.6.1.0
6 chameleon==2.24
6 chameleon==2.24
7 cffi==1.12.2
7 cffi==1.12.2
8 ecdsa==0.13.2
8 ecdsa==0.13.2
General Comments 0
You need to be logged in to leave comments. Login now