Show More
@@ -0,0 +1,16 b'' | |||
|
1 | """ | |
|
2 | Automatically sets the environment variable `CELERY_LOADER` to | |
|
3 | `celerypylons.loader:PylonsLoader`. This ensures the loader is | |
|
4 | specified when accessing the rest of this package, and allows celery | |
|
5 | to be installed in a webapp just by importing celerypylons:: | |
|
6 | ||
|
7 | import celerypylons | |
|
8 | ||
|
9 | """ | |
|
10 | import os | |
|
11 | import warnings | |
|
12 | ||
|
13 | CELERYPYLONS_LOADER = 'rhodecode.lib.celerypylons.loader.PylonsLoader' | |
|
14 | if os.environ.get('CELERY_LOADER', CELERYPYLONS_LOADER) != CELERYPYLONS_LOADER: | |
|
15 | warnings.warn("'CELERY_LOADER' environment variable will be overridden by celery-pylons.") | |
|
16 | os.environ['CELERY_LOADER'] = CELERYPYLONS_LOADER |
@@ -0,0 +1,143 b'' | |||
|
1 | import os | |
|
2 | from paste.script.command import Command, BadCommand | |
|
3 | import paste.deploy | |
|
4 | from pylons import config | |
|
5 | ||
|
6 | ||
|
7 | __all__ = ['CeleryDaemonCommand', 'CeleryBeatCommand', | |
|
8 | 'CAMQPAdminCommand', 'CeleryEventCommand'] | |
|
9 | ||
|
10 | ||
|
11 | class CeleryCommand(Command): | |
|
12 | """ | |
|
13 | Abstract Base Class for celery commands. | |
|
14 | ||
|
15 | The celery commands are somewhat aggressive about loading | |
|
16 | celery.conf, and since our module sets the `CELERY_LOADER` | |
|
17 | environment variable to our loader, we have to bootstrap a bit and | |
|
18 | make sure we've had a chance to load the pylons config off of the | |
|
19 | command line, otherwise everything fails. | |
|
20 | """ | |
|
21 | min_args = 1 | |
|
22 | min_args_error = "Please provide a paster config file as an argument." | |
|
23 | takes_config_file = 1 | |
|
24 | requires_config_file = True | |
|
25 | ||
|
26 | def run(self, args): | |
|
27 | """ | |
|
28 | Overrides Command.run | |
|
29 | ||
|
30 | Checks for a config file argument and loads it. | |
|
31 | """ | |
|
32 | if len(args) < self.min_args: | |
|
33 | raise BadCommand( | |
|
34 | self.min_args_error % {'min_args': self.min_args, | |
|
35 | 'actual_args': len(args)}) | |
|
36 | # Decrement because we're going to lob off the first argument. | |
|
37 | # @@ This is hacky | |
|
38 | self.min_args -= 1 | |
|
39 | self.bootstrap_config(args[0]) | |
|
40 | self.update_parser() | |
|
41 | return super(CeleryCommand, self).run(args[1:]) | |
|
42 | ||
|
43 | def update_parser(self): | |
|
44 | """ | |
|
45 | Abstract method. Allows for the class's parser to be updated | |
|
46 | before the superclass's `run` method is called. Necessary to | |
|
47 | allow options/arguments to be passed through to the underlying | |
|
48 | celery command. | |
|
49 | """ | |
|
50 | raise NotImplementedError("Abstract Method.") | |
|
51 | ||
|
52 | def bootstrap_config(self, conf): | |
|
53 | """ | |
|
54 | Loads the pylons configuration. | |
|
55 | """ | |
|
56 | path_to_ini_file = os.path.realpath(conf) | |
|
57 | conf = paste.deploy.appconfig('config:' + path_to_ini_file) | |
|
58 | config.init_app(conf.global_conf, conf.local_conf) | |
|
59 | ||
|
60 | ||
|
61 | class CeleryDaemonCommand(CeleryCommand): | |
|
62 | """Start the celery worker | |
|
63 | ||
|
64 | Starts the celery worker that uses a paste.deploy configuration | |
|
65 | file. | |
|
66 | """ | |
|
67 | usage = 'CONFIG_FILE [celeryd options...]' | |
|
68 | summary = __doc__.splitlines()[0] | |
|
69 | description = "".join(__doc__.splitlines()[2:]) | |
|
70 | ||
|
71 | parser = Command.standard_parser(quiet=True) | |
|
72 | ||
|
73 | def update_parser(self): | |
|
74 | from celery.bin import celeryd | |
|
75 | for x in celeryd.WorkerCommand().get_options(): | |
|
76 | self.parser.add_option(x) | |
|
77 | ||
|
78 | def command(self): | |
|
79 | from celery.bin import celeryd | |
|
80 | return celeryd.WorkerCommand().run(**vars(self.options)) | |
|
81 | ||
|
82 | ||
|
83 | class CeleryBeatCommand(CeleryCommand): | |
|
84 | """Start the celery beat server | |
|
85 | ||
|
86 | Starts the celery beat server using a paste.deploy configuration | |
|
87 | file. | |
|
88 | """ | |
|
89 | usage = 'CONFIG_FILE [celerybeat options...]' | |
|
90 | summary = __doc__.splitlines()[0] | |
|
91 | description = "".join(__doc__.splitlines()[2:]) | |
|
92 | ||
|
93 | parser = Command.standard_parser(quiet=True) | |
|
94 | ||
|
95 | def update_parser(self): | |
|
96 | from celery.bin import celerybeat | |
|
97 | for x in celerybeat.BeatCommand().get_options(): | |
|
98 | self.parser.add_option(x) | |
|
99 | ||
|
100 | def command(self): | |
|
101 | from celery.bin import celerybeat | |
|
102 | return celerybeat.BeatCommand(**vars(self.options)) | |
|
103 | ||
|
104 | class CAMQPAdminCommand(CeleryCommand): | |
|
105 | """CAMQP Admin | |
|
106 | ||
|
107 | CAMQP celery admin tool. | |
|
108 | """ | |
|
109 | usage = 'CONFIG_FILE [camqadm options...]' | |
|
110 | summary = __doc__.splitlines()[0] | |
|
111 | description = "".join(__doc__.splitlines()[2:]) | |
|
112 | ||
|
113 | parser = Command.standard_parser(quiet=True) | |
|
114 | ||
|
115 | def update_parser(self): | |
|
116 | from celery.bin import camqadm | |
|
117 | for x in camqadm.OPTION_LIST: | |
|
118 | self.parser.add_option(x) | |
|
119 | ||
|
120 | def command(self): | |
|
121 | from celery.bin import camqadm | |
|
122 | return camqadm.camqadm(*self.args, **vars(self.options)) | |
|
123 | ||
|
124 | ||
|
125 | class CeleryEventCommand(CeleryCommand): | |
|
126 | """Celery event commandd. | |
|
127 | ||
|
128 | Capture celery events. | |
|
129 | """ | |
|
130 | usage = 'CONFIG_FILE [celeryev options...]' | |
|
131 | summary = __doc__.splitlines()[0] | |
|
132 | description = "".join(__doc__.splitlines()[2:]) | |
|
133 | ||
|
134 | parser = Command.standard_parser(quiet=True) | |
|
135 | ||
|
136 | def update_parser(self): | |
|
137 | from celery.bin import celeryev | |
|
138 | for x in celeryev.OPTION_LIST: | |
|
139 | self.parser.add_option(x) | |
|
140 | ||
|
141 | def command(self): | |
|
142 | from celery.bin import celeryev | |
|
143 | return celeryev.run_celeryev(**vars(self.options)) |
@@ -0,0 +1,55 b'' | |||
|
1 | from celery.loaders.base import BaseLoader | |
|
2 | from pylons import config | |
|
3 | ||
|
4 | to_pylons = lambda x: x.replace('_', '.').lower() | |
|
5 | to_celery = lambda x: x.replace('.', '_').upper() | |
|
6 | ||
|
7 | LIST_PARAMS = """CELERY_IMPORTS ADMINS ROUTES""".split() | |
|
8 | ||
|
9 | ||
|
10 | class PylonsSettingsProxy(object): | |
|
11 | """Pylons Settings Proxy | |
|
12 | ||
|
13 | Proxies settings from pylons.config | |
|
14 | ||
|
15 | """ | |
|
16 | def __getattr__(self, key): | |
|
17 | pylons_key = to_pylons(key) | |
|
18 | try: | |
|
19 | value = config[pylons_key] | |
|
20 | if key in LIST_PARAMS: return value.split() | |
|
21 | return self.type_converter(value) | |
|
22 | except KeyError: | |
|
23 | raise AttributeError(pylons_key) | |
|
24 | ||
|
25 | def __setattr__(self, key, value): | |
|
26 | pylons_key = to_pylons(key) | |
|
27 | config[pylons_key] = value | |
|
28 | ||
|
29 | ||
|
30 | def type_converter(self, value): | |
|
31 | #cast to int | |
|
32 | if value.isdigit(): | |
|
33 | return int(value) | |
|
34 | ||
|
35 | #cast to bool | |
|
36 | if value.lower() in ['true', 'false']: | |
|
37 | return value.lower() == 'true' | |
|
38 | ||
|
39 | return value | |
|
40 | ||
|
41 | class PylonsLoader(BaseLoader): | |
|
42 | """Pylons celery loader | |
|
43 | ||
|
44 | Maps the celery config onto pylons.config | |
|
45 | ||
|
46 | """ | |
|
47 | def read_configuration(self): | |
|
48 | self.configured = True | |
|
49 | return PylonsSettingsProxy() | |
|
50 | ||
|
51 | def on_worker_init(self): | |
|
52 | """ | |
|
53 | Import task modules. | |
|
54 | """ | |
|
55 | self.import_default_modules() |
@@ -1,6 +1,6 b'' | |||
|
1 | 1 | ################################################################################ |
|
2 | 2 | ################################################################################ |
|
3 |
# |
|
|
3 | # RhodeCode - Pylons environment configuration # | |
|
4 | 4 | # # |
|
5 | 5 | # The %(here)s variable will be replaced with the parent directory of this file# |
|
6 | 6 | ################################################################################ |
@@ -9,8 +9,8 b'' | |||
|
9 | 9 | debug = true |
|
10 | 10 | ################################################################################ |
|
11 | 11 | ## Uncomment and replace with the address which should receive ## |
|
12 |
## any error reports after application crash |
|
|
13 |
## Additionally those settings will be used by |
|
|
12 | ## any error reports after application crash ## | |
|
13 | ## Additionally those settings will be used by RhodeCode mailing system ## | |
|
14 | 14 | ################################################################################ |
|
15 | 15 | #email_to = admin@localhost |
|
16 | 16 | #error_email_from = paste_error@localhost |
@@ -19,15 +19,16 b' debug = true' | |||
|
19 | 19 | |
|
20 | 20 | #smtp_server = mail.server.com |
|
21 | 21 | #smtp_username = |
|
22 | #smtp_password = | |
|
22 | #smtp_password = | |
|
23 | 23 | #smtp_port = |
|
24 | #smtp_use_tls = | |
|
24 | #smtp_use_tls = false | |
|
25 | #smtp_use_ssl = true | |
|
25 | 26 | |
|
26 | 27 | [server:main] |
|
27 | 28 | ##nr of threads to spawn |
|
28 | 29 | threadpool_workers = 5 |
|
29 | 30 | |
|
30 | ##max request before | |
|
31 | ##max request before thread respawn | |
|
31 | 32 | threadpool_max_requests = 6 |
|
32 | 33 | |
|
33 | 34 | ##option to use threads of process |
@@ -46,6 +47,33 b' cache_dir = %(here)s/data' | |||
|
46 | 47 | index_dir = %(here)s/data/index |
|
47 | 48 | |
|
48 | 49 | #################################### |
|
50 | ### CELERY CONFIG #### | |
|
51 | #################################### | |
|
52 | use_celery = false | |
|
53 | broker.host = localhost | |
|
54 | broker.vhost = rabbitmqhost | |
|
55 | broker.port = 5672 | |
|
56 | broker.user = rabbitmq | |
|
57 | broker.password = qweqwe | |
|
58 | ||
|
59 | celery.imports = rhodecode.lib.celerylib.tasks | |
|
60 | ||
|
61 | celery.result.backend = amqp | |
|
62 | celery.result.dburi = amqp:// | |
|
63 | celery.result.serialier = json | |
|
64 | ||
|
65 | #celery.send.task.error.emails = true | |
|
66 | #celery.amqp.task.result.expires = 18000 | |
|
67 | ||
|
68 | celeryd.concurrency = 2 | |
|
69 | #celeryd.log.file = celeryd.log | |
|
70 | celeryd.log.level = debug | |
|
71 | celeryd.max.tasks.per.child = 3 | |
|
72 | ||
|
73 | #tasks will never be sent to the queue, but executed locally instead. | |
|
74 | celery.always.eager = false | |
|
75 | ||
|
76 | #################################### | |
|
49 | 77 | ### BEAKER CACHE #### |
|
50 | 78 | #################################### |
|
51 | 79 | beaker.cache.data_dir=/%(here)s/data/cache/data |
@@ -61,9 +89,8 b' beaker.cache.short_term.expire=60' | |||
|
61 | 89 | beaker.cache.long_term.type=memory |
|
62 | 90 | beaker.cache.long_term.expire=36000 |
|
63 | 91 | |
|
64 | ||
|
65 | 92 | beaker.cache.sql_cache_short.type=memory |
|
66 |
beaker.cache.sql_cache_short.expire= |
|
|
93 | beaker.cache.sql_cache_short.expire=10 | |
|
67 | 94 | |
|
68 | 95 | beaker.cache.sql_cache_med.type=memory |
|
69 | 96 | beaker.cache.sql_cache_med.expire=360 |
@@ -75,7 +102,7 b' beaker.cache.sql_cache_long.expire=3600' | |||
|
75 | 102 | ### BEAKER SESSION #### |
|
76 | 103 | #################################### |
|
77 | 104 | ## Type of storage used for the session, current types are |
|
78 |
## |
|
|
105 | ## dbm, file, memcached, database, and memory. | |
|
79 | 106 | ## The storage uses the Container API |
|
80 | 107 | ##that is also used by the cache system. |
|
81 | 108 | beaker.session.type = file |
@@ -47,6 +47,33 b' cache_dir = %(here)s/data' | |||
|
47 | 47 | index_dir = %(here)s/data/index |
|
48 | 48 | |
|
49 | 49 | #################################### |
|
50 | ### CELERY CONFIG #### | |
|
51 | #################################### | |
|
52 | use_celery = false | |
|
53 | broker.host = localhost | |
|
54 | broker.vhost = rabbitmqhost | |
|
55 | broker.port = 5672 | |
|
56 | broker.user = rabbitmq | |
|
57 | broker.password = qweqwe | |
|
58 | ||
|
59 | celery.imports = rhodecode.lib.celerylib.tasks | |
|
60 | ||
|
61 | celery.result.backend = amqp | |
|
62 | celery.result.dburi = amqp:// | |
|
63 | celery.result.serialier = json | |
|
64 | ||
|
65 | #celery.send.task.error.emails = true | |
|
66 | #celery.amqp.task.result.expires = 18000 | |
|
67 | ||
|
68 | celeryd.concurrency = 2 | |
|
69 | #celeryd.log.file = celeryd.log | |
|
70 | celeryd.log.level = debug | |
|
71 | celeryd.max.tasks.per.child = 3 | |
|
72 | ||
|
73 | #tasks will never be sent to the queue, but executed locally instead. | |
|
74 | celery.always.eager = false | |
|
75 | ||
|
76 | #################################### | |
|
50 | 77 | ### BEAKER CACHE #### |
|
51 | 78 | #################################### |
|
52 | 79 | beaker.cache.data_dir=/%(here)s/data/cache/data |
@@ -1,6 +1,6 b'' | |||
|
1 | 1 | ################################################################################ |
|
2 | 2 | ################################################################################ |
|
3 |
# |
|
|
3 | # RhodeCode - Pylons environment configuration # | |
|
4 | 4 | # # |
|
5 | 5 | # The %(here)s variable will be replaced with the parent directory of this file# |
|
6 | 6 | ################################################################################ |
@@ -10,7 +10,7 b' debug = true' | |||
|
10 | 10 | ################################################################################ |
|
11 | 11 | ## Uncomment and replace with the address which should receive ## |
|
12 | 12 | ## any error reports after application crash ## |
|
13 |
## Additionally those settings will be used by |
|
|
13 | ## Additionally those settings will be used by RhodeCode mailing system ## | |
|
14 | 14 | ################################################################################ |
|
15 | 15 | #email_to = admin@localhost |
|
16 | 16 | #error_email_from = paste_error@localhost |
@@ -48,6 +48,33 b' index_dir = %(here)s/data/index' | |||
|
48 | 48 | app_instance_uuid = ${app_instance_uuid} |
|
49 | 49 | |
|
50 | 50 | #################################### |
|
51 | ### CELERY CONFIG #### | |
|
52 | #################################### | |
|
53 | use_celery = false | |
|
54 | broker.host = localhost | |
|
55 | broker.vhost = rabbitmqhost | |
|
56 | broker.port = 5672 | |
|
57 | broker.user = rabbitmq | |
|
58 | broker.password = qweqwe | |
|
59 | ||
|
60 | celery.imports = rhodecode.lib.celerylib.tasks | |
|
61 | ||
|
62 | celery.result.backend = amqp | |
|
63 | celery.result.dburi = amqp:// | |
|
64 | celery.result.serialier = json | |
|
65 | ||
|
66 | #celery.send.task.error.emails = true | |
|
67 | #celery.amqp.task.result.expires = 18000 | |
|
68 | ||
|
69 | celeryd.concurrency = 2 | |
|
70 | #celeryd.log.file = celeryd.log | |
|
71 | celeryd.log.level = debug | |
|
72 | celeryd.max.tasks.per.child = 3 | |
|
73 | ||
|
74 | #tasks will never be sent to the queue, but executed locally instead. | |
|
75 | celery.always.eager = false | |
|
76 | ||
|
77 | #################################### | |
|
51 | 78 | ### BEAKER CACHE #### |
|
52 | 79 | #################################### |
|
53 | 80 | beaker.cache.data_dir=/%(here)s/data/cache/data |
@@ -64,7 +91,7 b' beaker.cache.long_term.type=memory' | |||
|
64 | 91 | beaker.cache.long_term.expire=36000 |
|
65 | 92 | |
|
66 | 93 | beaker.cache.sql_cache_short.type=memory |
|
67 |
beaker.cache.sql_cache_short.expire= |
|
|
94 | beaker.cache.sql_cache_short.expire=10 | |
|
68 | 95 | |
|
69 | 96 | beaker.cache.sql_cache_med.type=memory |
|
70 | 97 | beaker.cache.sql_cache_med.expire=360 |
@@ -1,37 +1,47 b'' | |||
|
1 | import os | |
|
2 | import sys | |
|
3 | import socket | |
|
4 | import traceback | |
|
5 | import logging | |
|
6 | ||
|
1 | 7 | from rhodecode.lib.pidlock import DaemonLock, LockHeld |
|
2 | 8 | from vcs.utils.lazy import LazyProperty |
|
3 | 9 | from decorator import decorator |
|
4 | import logging | |
|
5 | import os | |
|
6 | import sys | |
|
7 | import traceback | |
|
8 | 10 | from hashlib import md5 |
|
9 | import socket | |
|
11 | from pylons import config | |
|
12 | ||
|
10 | 13 | log = logging.getLogger(__name__) |
|
11 | 14 | |
|
15 | def str2bool(v): | |
|
16 | return v.lower() in ["yes", "true", "t", "1"] if v else None | |
|
17 | ||
|
18 | CELERY_ON = str2bool(config['app_conf'].get('use_celery')) | |
|
19 | ||
|
12 | 20 | class ResultWrapper(object): |
|
13 | 21 | def __init__(self, task): |
|
14 | 22 | self.task = task |
|
15 | ||
|
23 | ||
|
16 | 24 | @LazyProperty |
|
17 | 25 | def result(self): |
|
18 | 26 | return self.task |
|
19 | 27 | |
|
20 | 28 | def run_task(task, *args, **kwargs): |
|
21 | try: | |
|
22 | t = task.delay(*args, **kwargs) | |
|
23 | log.info('running task %s', t.task_id) | |
|
24 | return t | |
|
25 | except socket.error, e: | |
|
26 | if e.errno == 111: | |
|
27 | log.debug('Unable to connect to celeryd. Sync execution') | |
|
28 | else: | |
|
29 | log.error(traceback.format_exc()) | |
|
30 | except KeyError, e: | |
|
31 | log.debug('Unable to connect to celeryd. Sync execution') | |
|
32 | except Exception, e: | |
|
33 | log.error(traceback.format_exc()) | |
|
34 | ||
|
29 | if CELERY_ON: | |
|
30 | try: | |
|
31 | t = task.delay(*args, **kwargs) | |
|
32 | log.info('running task %s:%s', t.task_id, task) | |
|
33 | return t | |
|
34 | except socket.error, e: | |
|
35 | if e.errno == 111: | |
|
36 | log.debug('Unable to connect to celeryd. Sync execution') | |
|
37 | else: | |
|
38 | log.error(traceback.format_exc()) | |
|
39 | except KeyError, e: | |
|
40 | log.debug('Unable to connect to celeryd. Sync execution') | |
|
41 | except Exception, e: | |
|
42 | log.error(traceback.format_exc()) | |
|
43 | ||
|
44 | log.debug('executing task %s in sync mode', task) | |
|
35 | 45 | return ResultWrapper(task(*args, **kwargs)) |
|
36 | 46 | |
|
37 | 47 | |
@@ -39,7 +49,7 b' def locked_task(func):' | |||
|
39 | 49 | def __wrapper(func, *fargs, **fkwargs): |
|
40 | 50 | params = list(fargs) |
|
41 | 51 | params.extend(['%s-%s' % ar for ar in fkwargs.items()]) |
|
42 | ||
|
52 | ||
|
43 | 53 | lockkey = 'task_%s' % \ |
|
44 | 54 | md5(str(func.__name__) + '-' + \ |
|
45 | 55 | '-'.join(map(str, params))).hexdigest() |
@@ -51,14 +61,14 b' def locked_task(func):' | |||
|
51 | 61 | return ret |
|
52 | 62 | except LockHeld: |
|
53 | 63 | log.info('LockHeld') |
|
54 |
return 'Task with key %s already running' % lockkey |
|
|
64 | return 'Task with key %s already running' % lockkey | |
|
55 | 65 | |
|
56 |
return decorator(__wrapper, func) |
|
|
57 | ||
|
66 | return decorator(__wrapper, func) | |
|
67 | ||
|
68 | ||
|
58 | 69 | |
|
59 | ||
|
60 | ||
|
61 | ||
|
62 | ||
|
63 | ||
|
64 | ||
|
70 | ||
|
71 | ||
|
72 | ||
|
73 | ||
|
74 |
@@ -2,16 +2,24 b' from celery.decorators import task' | |||
|
2 | 2 | |
|
3 | 3 | import os |
|
4 | 4 | import traceback |
|
5 | import beaker | |
|
5 | 6 | from time import mktime |
|
6 | ||
|
7 | 7 | from operator import itemgetter |
|
8 | ||
|
9 | from pylons import config | |
|
8 | 10 | from pylons.i18n.translation import _ |
|
9 | from rhodecode.lib.celerylib import run_task, locked_task | |
|
11 | ||
|
12 | from rhodecode.lib.celerylib import run_task, locked_task, str2bool | |
|
10 | 13 | from rhodecode.lib.helpers import person |
|
11 | 14 | from rhodecode.lib.smtp_mailer import SmtpMailer |
|
12 | 15 | from rhodecode.lib.utils import OrderedDict |
|
16 | from rhodecode.model import init_model | |
|
17 | from rhodecode.model import meta | |
|
18 | from rhodecode.model.db import RhodeCodeUi | |
|
19 | ||
|
13 | 20 | from vcs.backends import get_repo |
|
14 | from rhodecode.model.db import RhodeCodeUi | |
|
21 | ||
|
22 | from sqlalchemy import engine_from_config | |
|
15 | 23 | |
|
16 | 24 | try: |
|
17 | 25 | import json |
@@ -19,31 +27,16 b' except ImportError:' | |||
|
19 | 27 | #python 2.5 compatibility |
|
20 | 28 | import simplejson as json |
|
21 | 29 | |
|
22 | try: | |
|
23 | from celeryconfig import PYLONS_CONFIG as config | |
|
24 | celery_on = True | |
|
25 | except ImportError: | |
|
26 | #if celeryconfig is not present let's just load our pylons | |
|
27 | #config instead | |
|
28 | from pylons import config | |
|
29 | celery_on = False | |
|
30 | ||
|
31 | ||
|
32 | 30 | __all__ = ['whoosh_index', 'get_commits_stats', |
|
33 | 31 | 'reset_user_password', 'send_email'] |
|
34 | 32 | |
|
33 | CELERY_ON = str2bool(config['app_conf'].get('use_celery')) | |
|
34 | ||
|
35 | 35 | def get_session(): |
|
36 | if celery_on: | |
|
37 | from sqlalchemy import engine_from_config | |
|
38 | from sqlalchemy.orm import sessionmaker, scoped_session | |
|
39 | engine = engine_from_config(dict(config.items('app:main')), | |
|
40 | 'sqlalchemy.db1.') | |
|
41 | sa = scoped_session(sessionmaker(bind=engine)) | |
|
42 | else: | |
|
43 | #If we don't use celery reuse our current application Session | |
|
44 | from rhodecode.model.meta import Session | |
|
45 | sa = Session() | |
|
46 | ||
|
36 | if CELERY_ON: | |
|
37 | engine = engine_from_config(config, 'sqlalchemy.db1.') | |
|
38 | init_model(engine) | |
|
39 | sa = meta.Session() | |
|
47 | 40 | return sa |
|
48 | 41 | |
|
49 | 42 | def get_repos_path(): |
@@ -56,7 +49,7 b' def get_repos_path():' | |||
|
56 | 49 | def whoosh_index(repo_location, full_index): |
|
57 | 50 | log = whoosh_index.get_logger() |
|
58 | 51 | from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon |
|
59 |
index_location = |
|
|
52 | index_location = config['index_dir'] | |
|
60 | 53 | WhooshIndexingDaemon(index_location=index_location, |
|
61 | 54 | repo_location=repo_location).run(full_index=full_index) |
|
62 | 55 | |
@@ -235,6 +228,7 b' def reset_user_password(user_email):' | |||
|
235 | 228 | except: |
|
236 | 229 | log.error('Failed to update user password') |
|
237 | 230 | log.error(traceback.format_exc()) |
|
231 | ||
|
238 | 232 | return True |
|
239 | 233 | |
|
240 | 234 | @task |
@@ -249,14 +243,11 b' def send_email(recipients, subject, body' | |||
|
249 | 243 | :param body: body of the mail |
|
250 | 244 | """ |
|
251 | 245 | log = send_email.get_logger() |
|
252 |
email_config = |
|
|
246 | email_config = config | |
|
253 | 247 | |
|
254 | 248 | if not recipients: |
|
255 | 249 | recipients = [email_config.get('email_to')] |
|
256 | 250 | |
|
257 | def str2bool(v): | |
|
258 | return v.lower() in ["yes", "true", "t", "1"] if v else None | |
|
259 | ||
|
260 | 251 | mail_from = email_config.get('app_email_from') |
|
261 | 252 | user = email_config.get('smtp_username') |
|
262 | 253 | passwd = email_config.get('smtp_password') |
@@ -293,12 +284,58 b' def create_repo_fork(form_data, cur_user' | |||
|
293 | 284 | backend(str(repo_fork_path), create=True, src_url=str(repo_path)) |
|
294 | 285 | |
|
295 | 286 | def __get_codes_stats(repo_name): |
|
296 |
LANGUAGES_EXTENSIONS = |
|
|
297 | 'aspx', 'asx', 'axd', 'c', 'cfg', 'cfm', 'cpp', 'cs', 'diff', 'do', 'el', | |
|
298 | 'erl', 'h', 'java', 'js', 'jsp', 'jspx', 'lisp', 'lua', 'm', 'mako', 'ml', | |
|
299 | 'pas', 'patch', 'php', 'php3', 'php4', 'phtml', 'pm', 'py', 'rb', 'rst', | |
|
300 | 's', 'sh', 'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt', 'yaws'] | |
|
301 | ||
|
287 | LANGUAGES_EXTENSIONS_MAP = {'scm': 'Scheme', 'asmx': 'VbNetAspx', 'Rout': | |
|
288 | 'RConsole', 'rest': 'Rst', 'abap': 'ABAP', 'go': 'Go', 'phtml': 'HtmlPhp', | |
|
289 | 'ns2': 'Newspeak', 'xml': 'EvoqueXml', 'sh-session': 'BashSession', 'ads': | |
|
290 | 'Ada', 'clj': 'Clojure', 'll': 'Llvm', 'ebuild': 'Bash', 'adb': 'Ada', | |
|
291 | 'ada': 'Ada', 'c++-objdump': 'CppObjdump', 'aspx': | |
|
292 | 'VbNetAspx', 'ksh': 'Bash', 'coffee': 'CoffeeScript', 'vert': 'GLShader', | |
|
293 | 'Makefile.*': 'Makefile', 'di': 'D', 'dpatch': 'DarcsPatch', 'rake': | |
|
294 | 'Ruby', 'moo': 'MOOCode', 'erl-sh': 'ErlangShell', 'geo': 'GLShader', | |
|
295 | 'pov': 'Povray', 'bas': 'VbNet', 'bat': 'Batch', 'd': 'D', 'lisp': | |
|
296 | 'CommonLisp', 'h': 'C', 'rbx': 'Ruby', 'tcl': 'Tcl', 'c++': 'Cpp', 'md': | |
|
297 | 'MiniD', '.vimrc': 'Vim', 'xsd': 'Xml', 'ml': 'Ocaml', 'el': 'CommonLisp', | |
|
298 | 'befunge': 'Befunge', 'xsl': 'Xslt', 'pyx': 'Cython', 'cfm': | |
|
299 | 'ColdfusionHtml', 'evoque': 'Evoque', 'cfg': 'Ini', 'htm': 'Html', | |
|
300 | 'Makefile': 'Makefile', 'cfc': 'ColdfusionHtml', 'tex': 'Tex', 'cs': | |
|
301 | 'CSharp', 'mxml': 'Mxml', 'patch': 'Diff', 'apache.conf': 'ApacheConf', | |
|
302 | 'scala': 'Scala', 'applescript': 'AppleScript', 'GNUmakefile': 'Makefile', | |
|
303 | 'c-objdump': 'CObjdump', 'lua': 'Lua', 'apache2.conf': 'ApacheConf', 'rb': | |
|
304 | 'Ruby', 'gemspec': 'Ruby', 'rl': 'RagelObjectiveC', 'vala': 'Vala', 'tmpl': | |
|
305 | 'Cheetah', 'bf': 'Brainfuck', 'plt': 'Gnuplot', 'G': 'AntlrRuby', 'xslt': | |
|
306 | 'Xslt', 'flxh': 'Felix', 'asax': 'VbNetAspx', 'Rakefile': 'Ruby', 'S': 'S', | |
|
307 | 'wsdl': 'Xml', 'js': 'Javascript', 'autodelegate': 'Myghty', 'properties': | |
|
308 | 'Ini', 'bash': 'Bash', 'c': 'C', 'g': 'AntlrRuby', 'r3': 'Rebol', 's': | |
|
309 | 'Gas', 'ashx': 'VbNetAspx', 'cxx': 'Cpp', 'boo': 'Boo', 'prolog': 'Prolog', | |
|
310 | 'sqlite3-console': 'SqliteConsole', 'cl': 'CommonLisp', 'cc': 'Cpp', 'pot': | |
|
311 | 'Gettext', 'vim': 'Vim', 'pxi': 'Cython', 'yaml': 'Yaml', 'SConstruct': | |
|
312 | 'Python', 'diff': 'Diff', 'txt': 'Text', 'cw': 'Redcode', 'pxd': 'Cython', | |
|
313 | 'plot': 'Gnuplot', 'java': 'Java', 'hrl': 'Erlang', 'py': 'Python', | |
|
314 | 'makefile': 'Makefile', 'squid.conf': 'SquidConf', 'asm': 'Nasm', 'toc': | |
|
315 | 'Tex', 'kid': 'Genshi', 'rhtml': 'Rhtml', 'po': 'Gettext', 'pl': 'Prolog', | |
|
316 | 'pm': 'Perl', 'hx': 'Haxe', 'ascx': 'VbNetAspx', 'ooc': 'Ooc', 'asy': | |
|
317 | 'Asymptote', 'hs': 'Haskell', 'SConscript': 'Python', 'pytb': | |
|
318 | 'PythonTraceback', 'myt': 'Myghty', 'hh': 'Cpp', 'R': 'S', 'aux': 'Tex', | |
|
319 | 'rst': 'Rst', 'cpp-objdump': 'CppObjdump', 'lgt': 'Logtalk', 'rss': 'Xml', | |
|
320 | 'flx': 'Felix', 'b': 'Brainfuck', 'f': 'Fortran', 'rbw': 'Ruby', | |
|
321 | '.htaccess': 'ApacheConf', 'cxx-objdump': 'CppObjdump', 'j': 'ObjectiveJ', | |
|
322 | 'mll': 'Ocaml', 'yml': 'Yaml', 'mu': 'MuPAD', 'r': 'Rebol', 'ASM': 'Nasm', | |
|
323 | 'erl': 'Erlang', 'mly': 'Ocaml', 'mo': 'Modelica', 'def': 'Modula2', 'ini': | |
|
324 | 'Ini', 'control': 'DebianControl', 'vb': 'VbNet', 'vapi': 'Vala', 'pro': | |
|
325 | 'Prolog', 'spt': 'Cheetah', 'mli': 'Ocaml', 'as': 'ActionScript3', 'cmd': | |
|
326 | 'Batch', 'cpp': 'Cpp', 'io': 'Io', 'tac': 'Python', 'haml': 'Haml', 'rkt': | |
|
327 | 'Racket', 'st':'Smalltalk', 'inc': 'Povray', 'pas': 'Delphi', 'cmake': | |
|
328 | 'CMake', 'csh':'Tcsh', 'hpp': 'Cpp', 'feature': 'Gherkin', 'html': 'Html', | |
|
329 | 'php':'Php', 'php3':'Php', 'php4':'Php', 'php5':'Php', 'xhtml': 'Html', | |
|
330 | 'hxx': 'Cpp', 'eclass': 'Bash', 'css': 'Css', | |
|
331 | 'frag': 'GLShader', 'd-objdump': 'DObjdump', 'weechatlog': 'IrcLogs', | |
|
332 | 'tcsh': 'Tcsh', 'objdump': 'Objdump', 'pyw': 'Python', 'h++': 'Cpp', | |
|
333 | 'py3tb': 'Python3Traceback', 'jsp': 'Jsp', 'sql': 'Sql', 'mak': 'Makefile', | |
|
334 | 'php': 'Php', 'mao': 'Mako', 'man': 'Groff', 'dylan': 'Dylan', 'sass': | |
|
335 | 'Sass', 'cfml': 'ColdfusionHtml', 'darcspatch': 'DarcsPatch', 'tpl': | |
|
336 | 'Smarty', 'm': 'ObjectiveC', 'f90': 'Fortran', 'mod': 'Modula2', 'sh': | |
|
337 | 'Bash', 'lhs': 'LiterateHaskell', 'sources.list': 'SourcesList', 'axd': | |
|
338 | 'VbNetAspx', 'sc': 'Python'} | |
|
302 | 339 | |
|
303 | 340 | repos_path = get_repos_path() |
|
304 | 341 | p = os.path.join(repos_path, repo_name) |
@@ -308,12 +345,14 b' def __get_codes_stats(repo_name):' | |||
|
308 | 345 | |
|
309 | 346 | def aggregate(cs): |
|
310 | 347 | for f in cs[2]: |
|
311 |
|
|
|
312 |
|
|
|
313 | if code_stats.has_key(k): | |
|
314 | code_stats[k] += 1 | |
|
348 | ext = f.extension | |
|
349 | key = LANGUAGES_EXTENSIONS_MAP.get(ext, ext) | |
|
350 | key = key or ext | |
|
351 | if ext in LANGUAGES_EXTENSIONS_MAP.keys(): | |
|
352 | if code_stats.has_key(key): | |
|
353 | code_stats[key] += 1 | |
|
315 | 354 | else: |
|
316 | code_stats[k] = 1 | |
|
355 | code_stats[key] = 1 | |
|
317 | 356 | |
|
318 | 357 | map(aggregate, tip.walk('/')) |
|
319 | 358 |
@@ -46,6 +46,33 b' cache_dir = %(here)s/data' | |||
|
46 | 46 | index_dir = /tmp/index |
|
47 | 47 | |
|
48 | 48 | #################################### |
|
49 | ### CELERY CONFIG #### | |
|
50 | #################################### | |
|
51 | use_celery = false | |
|
52 | broker.host = localhost | |
|
53 | broker.vhost = rabbitmqhost | |
|
54 | broker.port = 5672 | |
|
55 | broker.user = rabbitmq | |
|
56 | broker.password = qweqwe | |
|
57 | ||
|
58 | celery.imports = rhodecode.lib.celerylib.tasks | |
|
59 | ||
|
60 | celery.result.backend = amqp | |
|
61 | celery.result.dburi = amqp:// | |
|
62 | celery.result.serialier = json | |
|
63 | ||
|
64 | #celery.send.task.error.emails = true | |
|
65 | #celery.amqp.task.result.expires = 18000 | |
|
66 | ||
|
67 | celeryd.concurrency = 2 | |
|
68 | #celeryd.log.file = celeryd.log | |
|
69 | celeryd.log.level = debug | |
|
70 | celeryd.max.tasks.per.child = 3 | |
|
71 | ||
|
72 | #tasks will never be sent to the queue, but executed locally instead. | |
|
73 | celery.always.eager = false | |
|
74 | ||
|
75 | #################################### | |
|
49 | 76 | ### BEAKER CACHE #### |
|
50 | 77 | #################################### |
|
51 | 78 | beaker.cache.data_dir=/%(here)s/data/cache/data |
|
1 | NO CONTENT: file was removed |
General Comments 0
You need to be logged in to leave comments.
Login now