Show More
@@ -307,8 +307,11 b' file_store.storage_path = %(here)s/data/' | |||
|
307 | 307 | |
|
308 | 308 | use_celery = false |
|
309 | 309 | |
|
310 |
## connection url to the message broker (default r |
|
|
311 |
celery.broker_url = |
|
|
310 | ## connection url to the message broker (default redis) | |
|
311 | celery.broker_url = redis://localhost:6379/8 | |
|
312 | ||
|
313 | ## rabbitmq example | |
|
314 | #celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost | |
|
312 | 315 | |
|
313 | 316 | ## maximum tasks to execute before worker restart |
|
314 | 317 | celery.max_tasks_per_child = 100 |
@@ -282,8 +282,11 b' file_store.storage_path = %(here)s/data/' | |||
|
282 | 282 | |
|
283 | 283 | use_celery = false |
|
284 | 284 | |
|
285 |
## connection url to the message broker (default r |
|
|
286 |
celery.broker_url = |
|
|
285 | ## connection url to the message broker (default redis) | |
|
286 | celery.broker_url = redis://localhost:6379/8 | |
|
287 | ||
|
288 | ## rabbitmq example | |
|
289 | #celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost | |
|
287 | 290 | |
|
288 | 291 | ## maximum tasks to execute before worker restart |
|
289 | 292 | celery.max_tasks_per_child = 100 |
@@ -57,7 +57,7 b" Here's an overview what components shoul" | |||
|
57 | 57 | - `nginx` acting as a load-balancer. |
|
58 | 58 | - `postgresql-server` used for database and sessions. |
|
59 | 59 | - `redis-server` used for storing shared caches. |
|
60 | - optionally `rabbitmq-server` for `Celery` if used. | |
|
60 | - optionally `rabbitmq-server` or `redis` for `Celery` if used. | |
|
61 | 61 | - optionally if `Celery` is used Enterprise/Community instance + VCSServer. |
|
62 | 62 | - optionally mailserver that can be shared by other instances. |
|
63 | 63 | - optionally channelstream server to handle live communication for all instances. |
@@ -372,16 +372,16 b' Using Celery with cluster' | |||
|
372 | 372 | |
|
373 | 373 | |
|
374 | 374 | If `Celery` is used we recommend setting also an instance of Enterprise/Community+VCSserver |
|
375 |
on the node that is running `RabbitMQ`_. Those instances will be used to |
|
|
376 |
tasks on the `rc-node-1`. This is the most efficient setup. |
|
|
377 | handles tasks such as sending emails, forking repositories, importing | |
|
375 | on the node that is running `RabbitMQ`_ or `Redis`_. Those instances will be used to | |
|
376 | executed async tasks on the `rc-node-1`. This is the most efficient setup. | |
|
377 | `Celery` usually handles tasks such as sending emails, forking repositories, importing | |
|
378 | 378 | repositories from external location etc. Using workers on instance that has |
|
379 | 379 | the direct access to disks used by NFS as well as email server gives noticeable |
|
380 | 380 | performance boost. Running local workers to the NFS storage results in faster |
|
381 | 381 | execution of forking large repositories or sending lots of emails. |
|
382 | 382 | |
|
383 | 383 | Those instances need to be configured in the same way as for other nodes. |
|
384 | The instance in rc-node-1 can be added to the cluser, but we don't recommend doing it. | |
|
384 | The instance in rc-node-1 can be added to the cluster, but we don't recommend doing it. | |
|
385 | 385 | For best results let it be isolated to only executing `Celery` tasks in the cluster setup. |
|
386 | 386 | |
|
387 | 387 |
@@ -11,16 +11,20 b' and import repositories in async way. It' | |||
|
11 | 11 | repository sync in scheduler. |
|
12 | 12 | |
|
13 | 13 | If you decide to use Celery you also need a working message queue. |
|
14 |
The |
|
|
14 | There are two fully supported message brokers is rabbitmq_ and redis_ (recommended). | |
|
15 | ||
|
16 | Since release 4.18.X we recommend using redis_ as a backend since it's generally | |
|
17 | easier to work with, and results in simpler stack as redis is generally recommended | |
|
18 | for caching purposes. | |
|
15 | 19 | |
|
16 | 20 | |
|
17 | 21 | In order to install and configure Celery, follow these steps: |
|
18 | 22 | |
|
19 | 1. Install RabbitMQ, see the documentation on the Celery website for | |
|
20 | `rabbitmq installation`_, or `rabbitmq website installation`_ | |
|
23 | 1. Install RabbitMQ or Redis for a message queue, see the documentation on the Celery website for | |
|
24 | `redis installation`_ or `rabbitmq installation`_ | |
|
21 | 25 | |
|
22 | 26 | |
|
23 |
1a. |
|
|
27 | 1a. If you choose RabbitMQ example configuration after installation would look like that:: | |
|
24 | 28 | |
|
25 | 29 | sudo rabbitmqctl add_user rcuser secret_password |
|
26 | 30 | sudo rabbitmqctl add_vhost rhodevhost |
@@ -45,6 +49,10 b' 3. Configure Celery in the' | |||
|
45 | 49 | Set the broker_url as minimal settings required to enable operation. |
|
46 | 50 | If used our example data from pt 1a, here is how the broker url should look like:: |
|
47 | 51 | |
|
52 | # for Redis | |
|
53 | celery.broker_url = redis://localhost:6379/8 | |
|
54 | ||
|
55 | # for RabbitMQ | |
|
48 | 56 | celery.broker_url = amqp://rcuser:secret_password@localhost:5672/rhodevhost |
|
49 | 57 |
|
|
50 | 58 | Full configuration example is below: |
@@ -57,7 +65,7 b' 3. Configure Celery in the' | |||
|
57 | 65 | #################################### |
|
58 | 66 | |
|
59 | 67 | use_celery = true |
|
60 |
celery.broker_url = |
|
|
68 | celery.broker_url = redis://localhost:6379/8 | |
|
61 | 69 | |
|
62 | 70 | # maximum tasks to execute before worker restart |
|
63 | 71 | celery.max_tasks_per_child = 100 |
@@ -69,6 +77,8 b' 3. Configure Celery in the' | |||
|
69 | 77 | .. _python: http://www.python.org/ |
|
70 | 78 | .. _mercurial: http://mercurial.selenic.com/ |
|
71 | 79 | .. _celery: http://celeryproject.org/ |
|
80 | .. _redis: http://redis.io | |
|
81 | .. _redis installation: https://redis.io/topics/quickstart | |
|
72 | 82 | .. _rabbitmq: http://www.rabbitmq.com/ |
|
73 | 83 | .. _rabbitmq installation: http://docs.celeryproject.org/en/latest/getting-started/brokers/rabbitmq.html |
|
74 | 84 | .. _rabbitmq website installation: http://www.rabbitmq.com/download.html |
@@ -22,14 +22,14 b' self: super: {' | |||
|
22 | 22 | }; |
|
23 | 23 | }; |
|
24 | 24 | "amqp" = super.buildPythonPackage { |
|
25 |
name = "amqp-2. |
|
|
25 | name = "amqp-2.5.1"; | |
|
26 | 26 | doCheck = false; |
|
27 | 27 | propagatedBuildInputs = [ |
|
28 | 28 | self."vine" |
|
29 | 29 | ]; |
|
30 | 30 | src = fetchurl { |
|
31 |
url = "https://files.pythonhosted.org/packages/ |
|
|
32 | sha256 = "0wlfnvhmfrn7c8qif2jyvsm63ibdxp02ss564qwrvqfhz0di72s0"; | |
|
31 | url = "https://files.pythonhosted.org/packages/b5/f5/70e364a1f5fbafc742c098ad88a064b801b0d69cf56bfad13be2c08be4e2/amqp-2.5.1.tar.gz"; | |
|
32 | sha256 = "0s2yxnnhhx9hww0n33yn22q6sgnbd6n2nw92050qv2qpc3i1ga8r"; | |
|
33 | 33 | }; |
|
34 | 34 | meta = { |
|
35 | 35 | license = [ pkgs.lib.licenses.bsdOriginal ]; |
@@ -135,11 +135,11 b' self: super: {' | |||
|
135 | 135 | }; |
|
136 | 136 | }; |
|
137 | 137 | "billiard" = super.buildPythonPackage { |
|
138 |
name = "billiard-3. |
|
|
138 | name = "billiard-3.6.1.0"; | |
|
139 | 139 | doCheck = false; |
|
140 | 140 | src = fetchurl { |
|
141 |
url = "https://files.pythonhosted.org/packages/ |
|
|
142 | sha256 = "1riwiiwgb141151md4ykx49qrz749akj5k8g290ji9bsqjyj4yqx"; | |
|
141 | url = "https://files.pythonhosted.org/packages/68/1d/2aea8fbb0b1e1260a8a2e77352de2983d36d7ac01207cf14c2b9c6cc860e/billiard-3.6.1.0.tar.gz"; | |
|
142 | sha256 = "09hzy3aqi7visy4vmf4xiish61n0rq5nd3iwjydydps8yrs9r05q"; | |
|
143 | 143 | }; |
|
144 | 144 | meta = { |
|
145 | 145 | license = [ pkgs.lib.licenses.bsdOriginal ]; |
@@ -172,16 +172,17 b' self: super: {' | |||
|
172 | 172 | }; |
|
173 | 173 | }; |
|
174 | 174 | "celery" = super.buildPythonPackage { |
|
175 |
name = "celery-4. |
|
|
175 | name = "celery-4.3.0"; | |
|
176 | 176 | doCheck = false; |
|
177 | 177 | propagatedBuildInputs = [ |
|
178 | 178 | self."pytz" |
|
179 | 179 | self."billiard" |
|
180 | 180 | self."kombu" |
|
181 | self."vine" | |
|
181 | 182 | ]; |
|
182 | 183 | src = fetchurl { |
|
183 |
url = "https://files.pythonhosted.org/packages/e9 |
|
|
184 | sha256 = "1xbir4vw42n2ir9lanhwl7w69zpmj7lbi66fxm2b7pyvkcss7wni"; | |
|
184 | url = "https://files.pythonhosted.org/packages/a2/4b/d020836f751617e907e84753a41c92231cd4b673ff991b8ee9da52361323/celery-4.3.0.tar.gz"; | |
|
185 | sha256 = "1y8y0gbgkwimpxqnxq2rm5qz2vy01fvjiybnpm00y5rzd2m34iac"; | |
|
185 | 186 | }; |
|
186 | 187 | meta = { |
|
187 | 188 | license = [ pkgs.lib.licenses.bsdOriginal ]; |
@@ -284,6 +285,17 b' self: super: {' | |||
|
284 | 285 | license = [ pkgs.lib.licenses.mit ]; |
|
285 | 286 | }; |
|
286 | 287 | }; |
|
288 | "contextlib2" = super.buildPythonPackage { | |
|
289 | name = "contextlib2-0.5.5"; | |
|
290 | doCheck = false; | |
|
291 | src = fetchurl { | |
|
292 | url = "https://files.pythonhosted.org/packages/6e/db/41233498c210b03ab8b072c8ee49b1cd63b3b0c76f8ea0a0e5d02df06898/contextlib2-0.5.5.tar.gz"; | |
|
293 | sha256 = "0j6ad6lwwyc9kv71skj098v5l7x5biyj2hs4lc5x1kcixqcr97sh"; | |
|
294 | }; | |
|
295 | meta = { | |
|
296 | license = [ pkgs.lib.licenses.psfl ]; | |
|
297 | }; | |
|
298 | }; | |
|
287 | 299 | "cov-core" = super.buildPythonPackage { |
|
288 | 300 | name = "cov-core-1.15.0"; |
|
289 | 301 | doCheck = false; |
@@ -665,6 +677,23 b' self: super: {' | |||
|
665 | 677 | license = [ pkgs.lib.licenses.mit ]; |
|
666 | 678 | }; |
|
667 | 679 | }; |
|
680 | "importlib-metadata" = super.buildPythonPackage { | |
|
681 | name = "importlib-metadata-0.20"; | |
|
682 | doCheck = false; | |
|
683 | propagatedBuildInputs = [ | |
|
684 | self."zipp" | |
|
685 | self."contextlib2" | |
|
686 | self."configparser" | |
|
687 | self."pathlib2" | |
|
688 | ]; | |
|
689 | src = fetchurl { | |
|
690 | url = "https://files.pythonhosted.org/packages/05/41/7d339dd7b507e97f67be812fdf29c4ad991ddd34b1ed0f3c54e8f1c4e0b3/importlib_metadata-0.20.tar.gz"; | |
|
691 | sha256 = "13bshj8i98l9gxi6df4xbw1262phmawgr527as20brblwf93a55p"; | |
|
692 | }; | |
|
693 | meta = { | |
|
694 | license = [ pkgs.lib.licenses.asl20 ]; | |
|
695 | }; | |
|
696 | }; | |
|
668 | 697 | "infrae.cache" = super.buildPythonPackage { |
|
669 | 698 | name = "infrae.cache-1.0.1"; |
|
670 | 699 | doCheck = false; |
@@ -848,14 +877,15 b' self: super: {' | |||
|
848 | 877 | }; |
|
849 | 878 | }; |
|
850 | 879 | "kombu" = super.buildPythonPackage { |
|
851 |
name = "kombu-4. |
|
|
880 | name = "kombu-4.6.4"; | |
|
852 | 881 | doCheck = false; |
|
853 | 882 | propagatedBuildInputs = [ |
|
854 | 883 | self."amqp" |
|
884 | self."importlib-metadata" | |
|
855 | 885 | ]; |
|
856 | 886 | src = fetchurl { |
|
857 |
url = "https://files.pythonhosted.org/packages/ |
|
|
858 | sha256 = "10lh3hncvw67fz0k5vgbx3yh9gjfpqdlia1f13i28cgnc1nfrbc6"; | |
|
887 | url = "https://files.pythonhosted.org/packages/52/f2/5a64fc850b0533d2daf09a523406e51e85a8b2a4a2bc87a922a8906ba2aa/kombu-4.6.4.tar.gz"; | |
|
888 | sha256 = "16w02mvkxchz7041yia4h8xmqavci88szk18ynxvw4chzcnk3w75"; | |
|
859 | 889 | }; |
|
860 | 890 | meta = { |
|
861 | 891 | license = [ pkgs.lib.licenses.bsdOriginal ]; |
@@ -1636,11 +1666,11 b' self: super: {' | |||
|
1636 | 1666 | }; |
|
1637 | 1667 | }; |
|
1638 | 1668 | "pytz" = super.buildPythonPackage { |
|
1639 |
name = "pytz-201 |
|
|
1669 | name = "pytz-2019.2"; | |
|
1640 | 1670 | doCheck = false; |
|
1641 | 1671 | src = fetchurl { |
|
1642 |
url = "https://files.pythonhosted.org/packages/ |
|
|
1643 | sha256 = "0jgpqx3kk2rhv81j1izjxvmx8d0x7hzs1857pgqnixic5wq2ar60"; | |
|
1672 | url = "https://files.pythonhosted.org/packages/27/c0/fbd352ca76050952a03db776d241959d5a2ee1abddfeb9e2a53fdb489be4/pytz-2019.2.tar.gz"; | |
|
1673 | sha256 = "0ckb27hhjc8i8gcdvk4d9avld62b7k52yjijc60s2m3y8cpb7h16"; | |
|
1644 | 1674 | }; |
|
1645 | 1675 | meta = { |
|
1646 | 1676 | license = [ pkgs.lib.licenses.mit ]; |
@@ -2271,6 +2301,20 b' self: super: {' | |||
|
2271 | 2301 | license = [ { fullName = "PSF or ZPL"; } ]; |
|
2272 | 2302 | }; |
|
2273 | 2303 | }; |
|
2304 | "zipp" = super.buildPythonPackage { | |
|
2305 | name = "zipp-0.6.0"; | |
|
2306 | doCheck = false; | |
|
2307 | propagatedBuildInputs = [ | |
|
2308 | self."more-itertools" | |
|
2309 | ]; | |
|
2310 | src = fetchurl { | |
|
2311 | url = "https://files.pythonhosted.org/packages/57/dd/585d728479d97d25aeeb9aa470d36a4ad8d0ba5610f84e14770128ce6ff7/zipp-0.6.0.tar.gz"; | |
|
2312 | sha256 = "13ndkf7vklw978a4gdl1yfvn8hch28429a0iam67sg4nrp5v261p"; | |
|
2313 | }; | |
|
2314 | meta = { | |
|
2315 | license = [ pkgs.lib.licenses.mit ]; | |
|
2316 | }; | |
|
2317 | }; | |
|
2274 | 2318 | "zope.cachedescriptors" = super.buildPythonPackage { |
|
2275 | 2319 | name = "zope.cachedescriptors-4.3.1"; |
|
2276 | 2320 | doCheck = false; |
@@ -1,11 +1,11 b'' | |||
|
1 | 1 | ## dependencies |
|
2 | 2 | |
|
3 |
amqp==2. |
|
|
3 | amqp==2.5.1 | |
|
4 | 4 | |
|
5 | 5 | babel==1.3 |
|
6 | 6 | beaker==1.9.1 |
|
7 | 7 | bleach==3.1.0 |
|
8 |
celery==4. |
|
|
8 | celery==4.3.0 | |
|
9 | 9 | channelstream==0.5.2 |
|
10 | 10 | click==7.0 |
|
11 | 11 | colander==1.7.0 |
@@ -24,7 +24,7 b' futures==3.0.2' | |||
|
24 | 24 | infrae.cache==1.0.1 |
|
25 | 25 | iso8601==0.1.12 |
|
26 | 26 | itsdangerous==0.24 |
|
27 |
kombu==4. |
|
|
27 | kombu==4.6.4 | |
|
28 | 28 | lxml==4.2.5 |
|
29 | 29 | mako==1.0.7 |
|
30 | 30 | markdown==2.6.11 |
@@ -52,7 +52,7 b' python-ldap==3.1.0' | |||
|
52 | 52 | python-memcached==1.59 |
|
53 | 53 | python-pam==1.8.4 |
|
54 | 54 | python-saml==2.4.2 |
|
55 |
pytz==201 |
|
|
55 | pytz==2019.2 | |
|
56 | 56 | tzlocal==1.5.1 |
|
57 | 57 | pyzmq==14.6.0 |
|
58 | 58 | py-gfm==0.1.4 |
General Comments 0
You need to be logged in to leave comments.
Login now