##// END OF EJS Templates
subscribers: make creation of routes.js safer....
marcink -
r2823:e48f0fdb default
parent child Browse files
Show More
@@ -1,324 +1,327 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2010-2018 RhodeCode GmbH
3 # Copyright (C) 2010-2018 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 import io
20 import io
21 import re
21 import re
22 import datetime
22 import datetime
23 import logging
23 import logging
24 import Queue
24 import Queue
25 import subprocess32
25 import subprocess32
26 import os
26 import os
27
27
28
28
29 from dateutil.parser import parse
29 from dateutil.parser import parse
30 from pyramid.i18n import get_localizer
30 from pyramid.i18n import get_localizer
31 from pyramid.threadlocal import get_current_request
31 from pyramid.threadlocal import get_current_request
32 from pyramid.interfaces import IRoutesMapper
32 from pyramid.interfaces import IRoutesMapper
33 from pyramid.settings import asbool
33 from pyramid.settings import asbool
34 from pyramid.path import AssetResolver
34 from pyramid.path import AssetResolver
35 from threading import Thread
35 from threading import Thread
36
36
37 from rhodecode.translation import _ as tsf
37 from rhodecode.translation import _ as tsf
38 from rhodecode.config.jsroutes import generate_jsroutes_content
38 from rhodecode.config.jsroutes import generate_jsroutes_content
39 from rhodecode.lib import auth
39 from rhodecode.lib import auth
40 from rhodecode.lib.base import get_auth_user
40 from rhodecode.lib.base import get_auth_user
41
41
42
42
43 import rhodecode
43 import rhodecode
44
44
45
45
46 log = logging.getLogger(__name__)
46 log = logging.getLogger(__name__)
47
47
48
48
49 def add_renderer_globals(event):
49 def add_renderer_globals(event):
50 from rhodecode.lib import helpers
50 from rhodecode.lib import helpers
51
51
52 # TODO: When executed in pyramid view context the request is not available
52 # TODO: When executed in pyramid view context the request is not available
53 # in the event. Find a better solution to get the request.
53 # in the event. Find a better solution to get the request.
54 request = event['request'] or get_current_request()
54 request = event['request'] or get_current_request()
55
55
56 # Add Pyramid translation as '_' to context
56 # Add Pyramid translation as '_' to context
57 event['_'] = request.translate
57 event['_'] = request.translate
58 event['_ungettext'] = request.plularize
58 event['_ungettext'] = request.plularize
59 event['h'] = helpers
59 event['h'] = helpers
60
60
61
61
62 def add_localizer(event):
62 def add_localizer(event):
63 request = event.request
63 request = event.request
64 localizer = request.localizer
64 localizer = request.localizer
65
65
66 def auto_translate(*args, **kwargs):
66 def auto_translate(*args, **kwargs):
67 return localizer.translate(tsf(*args, **kwargs))
67 return localizer.translate(tsf(*args, **kwargs))
68
68
69 request.translate = auto_translate
69 request.translate = auto_translate
70 request.plularize = localizer.pluralize
70 request.plularize = localizer.pluralize
71
71
72
72
73 def set_user_lang(event):
73 def set_user_lang(event):
74 request = event.request
74 request = event.request
75 cur_user = getattr(request, 'user', None)
75 cur_user = getattr(request, 'user', None)
76
76
77 if cur_user:
77 if cur_user:
78 user_lang = cur_user.get_instance().user_data.get('language')
78 user_lang = cur_user.get_instance().user_data.get('language')
79 if user_lang:
79 if user_lang:
80 log.debug('lang: setting current user:%s language to: %s', cur_user, user_lang)
80 log.debug('lang: setting current user:%s language to: %s', cur_user, user_lang)
81 event.request._LOCALE_ = user_lang
81 event.request._LOCALE_ = user_lang
82
82
83
83
84 def add_request_user_context(event):
84 def add_request_user_context(event):
85 """
85 """
86 Adds auth user into request context
86 Adds auth user into request context
87 """
87 """
88 request = event.request
88 request = event.request
89 # access req_id as soon as possible
89 # access req_id as soon as possible
90 req_id = request.req_id
90 req_id = request.req_id
91
91
92 if hasattr(request, 'vcs_call'):
92 if hasattr(request, 'vcs_call'):
93 # skip vcs calls
93 # skip vcs calls
94 return
94 return
95
95
96 if hasattr(request, 'rpc_method'):
96 if hasattr(request, 'rpc_method'):
97 # skip api calls
97 # skip api calls
98 return
98 return
99
99
100 auth_user = get_auth_user(request)
100 auth_user = get_auth_user(request)
101 request.user = auth_user
101 request.user = auth_user
102 request.environ['rc_auth_user'] = auth_user
102 request.environ['rc_auth_user'] = auth_user
103 request.environ['rc_req_id'] = req_id
103 request.environ['rc_req_id'] = req_id
104
104
105 def inject_app_settings(event):
105 def inject_app_settings(event):
106 settings = event.app.registry.settings
106 settings = event.app.registry.settings
107 # inject info about available permissions
107 # inject info about available permissions
108 auth.set_available_permissions(settings)
108 auth.set_available_permissions(settings)
109
109
110
110
111 def scan_repositories_if_enabled(event):
111 def scan_repositories_if_enabled(event):
112 """
112 """
113 This is subscribed to the `pyramid.events.ApplicationCreated` event. It
113 This is subscribed to the `pyramid.events.ApplicationCreated` event. It
114 does a repository scan if enabled in the settings.
114 does a repository scan if enabled in the settings.
115 """
115 """
116 settings = event.app.registry.settings
116 settings = event.app.registry.settings
117 vcs_server_enabled = settings['vcs.server.enable']
117 vcs_server_enabled = settings['vcs.server.enable']
118 import_on_startup = settings['startup.import_repos']
118 import_on_startup = settings['startup.import_repos']
119 if vcs_server_enabled and import_on_startup:
119 if vcs_server_enabled and import_on_startup:
120 from rhodecode.model.scm import ScmModel
120 from rhodecode.model.scm import ScmModel
121 from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_base_path
121 from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_base_path
122 repositories = ScmModel().repo_scan(get_rhodecode_base_path())
122 repositories = ScmModel().repo_scan(get_rhodecode_base_path())
123 repo2db_mapper(repositories, remove_obsolete=False)
123 repo2db_mapper(repositories, remove_obsolete=False)
124
124
125
125
126 def write_metadata_if_needed(event):
126 def write_metadata_if_needed(event):
127 """
127 """
128 Writes upgrade metadata
128 Writes upgrade metadata
129 """
129 """
130 import rhodecode
130 import rhodecode
131 from rhodecode.lib import system_info
131 from rhodecode.lib import system_info
132 from rhodecode.lib import ext_json
132 from rhodecode.lib import ext_json
133
133
134 fname = '.rcmetadata.json'
134 fname = '.rcmetadata.json'
135 ini_loc = os.path.dirname(rhodecode.CONFIG.get('__file__'))
135 ini_loc = os.path.dirname(rhodecode.CONFIG.get('__file__'))
136 metadata_destination = os.path.join(ini_loc, fname)
136 metadata_destination = os.path.join(ini_loc, fname)
137
137
138 def get_update_age():
138 def get_update_age():
139 now = datetime.datetime.utcnow()
139 now = datetime.datetime.utcnow()
140
140
141 with open(metadata_destination, 'rb') as f:
141 with open(metadata_destination, 'rb') as f:
142 data = ext_json.json.loads(f.read())
142 data = ext_json.json.loads(f.read())
143 if 'created_on' in data:
143 if 'created_on' in data:
144 update_date = parse(data['created_on'])
144 update_date = parse(data['created_on'])
145 diff = now - update_date
145 diff = now - update_date
146 return diff.total_seconds() / 60.0
146 return diff.total_seconds() / 60.0
147
147
148 return 0
148 return 0
149
149
150 def write():
150 def write():
151 configuration = system_info.SysInfo(
151 configuration = system_info.SysInfo(
152 system_info.rhodecode_config)()['value']
152 system_info.rhodecode_config)()['value']
153 license_token = configuration['config']['license_token']
153 license_token = configuration['config']['license_token']
154
154
155 setup = dict(
155 setup = dict(
156 workers=configuration['config']['server:main'].get(
156 workers=configuration['config']['server:main'].get(
157 'workers', '?'),
157 'workers', '?'),
158 worker_type=configuration['config']['server:main'].get(
158 worker_type=configuration['config']['server:main'].get(
159 'worker_class', 'sync'),
159 'worker_class', 'sync'),
160 )
160 )
161 dbinfo = system_info.SysInfo(system_info.database_info)()['value']
161 dbinfo = system_info.SysInfo(system_info.database_info)()['value']
162 del dbinfo['url']
162 del dbinfo['url']
163
163
164 metadata = dict(
164 metadata = dict(
165 desc='upgrade metadata info',
165 desc='upgrade metadata info',
166 license_token=license_token,
166 license_token=license_token,
167 created_on=datetime.datetime.utcnow().isoformat(),
167 created_on=datetime.datetime.utcnow().isoformat(),
168 usage=system_info.SysInfo(system_info.usage_info)()['value'],
168 usage=system_info.SysInfo(system_info.usage_info)()['value'],
169 platform=system_info.SysInfo(system_info.platform_type)()['value'],
169 platform=system_info.SysInfo(system_info.platform_type)()['value'],
170 database=dbinfo,
170 database=dbinfo,
171 cpu=system_info.SysInfo(system_info.cpu)()['value'],
171 cpu=system_info.SysInfo(system_info.cpu)()['value'],
172 memory=system_info.SysInfo(system_info.memory)()['value'],
172 memory=system_info.SysInfo(system_info.memory)()['value'],
173 setup=setup
173 setup=setup
174 )
174 )
175
175
176 with open(metadata_destination, 'wb') as f:
176 with open(metadata_destination, 'wb') as f:
177 f.write(ext_json.json.dumps(metadata))
177 f.write(ext_json.json.dumps(metadata))
178
178
179 settings = event.app.registry.settings
179 settings = event.app.registry.settings
180 if settings.get('metadata.skip'):
180 if settings.get('metadata.skip'):
181 return
181 return
182
182
183 # only write this every 24h, workers restart caused unwanted delays
183 # only write this every 24h, workers restart caused unwanted delays
184 try:
184 try:
185 age_in_min = get_update_age()
185 age_in_min = get_update_age()
186 except Exception:
186 except Exception:
187 age_in_min = 0
187 age_in_min = 0
188
188
189 if age_in_min > 60 * 60 * 24:
189 if age_in_min > 60 * 60 * 24:
190 return
190 return
191
191
192 try:
192 try:
193 write()
193 write()
194 except Exception:
194 except Exception:
195 pass
195 pass
196
196
197
197
198 def write_js_routes_if_enabled(event):
198 def write_js_routes_if_enabled(event):
199 registry = event.app.registry
199 registry = event.app.registry
200
200
201 mapper = registry.queryUtility(IRoutesMapper)
201 mapper = registry.queryUtility(IRoutesMapper)
202 _argument_prog = re.compile('\{(.*?)\}|:\((.*)\)')
202 _argument_prog = re.compile('\{(.*?)\}|:\((.*)\)')
203
203
204 def _extract_route_information(route):
204 def _extract_route_information(route):
205 """
205 """
206 Convert a route into tuple(name, path, args), eg:
206 Convert a route into tuple(name, path, args), eg:
207 ('show_user', '/profile/%(username)s', ['username'])
207 ('show_user', '/profile/%(username)s', ['username'])
208 """
208 """
209
209
210 routepath = route.pattern
210 routepath = route.pattern
211 pattern = route.pattern
211 pattern = route.pattern
212
212
213 def replace(matchobj):
213 def replace(matchobj):
214 if matchobj.group(1):
214 if matchobj.group(1):
215 return "%%(%s)s" % matchobj.group(1).split(':')[0]
215 return "%%(%s)s" % matchobj.group(1).split(':')[0]
216 else:
216 else:
217 return "%%(%s)s" % matchobj.group(2)
217 return "%%(%s)s" % matchobj.group(2)
218
218
219 routepath = _argument_prog.sub(replace, routepath)
219 routepath = _argument_prog.sub(replace, routepath)
220
220
221 if not routepath.startswith('/'):
221 if not routepath.startswith('/'):
222 routepath = '/'+routepath
222 routepath = '/'+routepath
223
223
224 return (
224 return (
225 route.name,
225 route.name,
226 routepath,
226 routepath,
227 [(arg[0].split(':')[0] if arg[0] != '' else arg[1])
227 [(arg[0].split(':')[0] if arg[0] != '' else arg[1])
228 for arg in _argument_prog.findall(pattern)]
228 for arg in _argument_prog.findall(pattern)]
229 )
229 )
230
230
231 def get_routes():
231 def get_routes():
232 # pyramid routes
232 # pyramid routes
233 for route in mapper.get_routes():
233 for route in mapper.get_routes():
234 if not route.name.startswith('__'):
234 if not route.name.startswith('__'):
235 yield _extract_route_information(route)
235 yield _extract_route_information(route)
236
236
237 if asbool(registry.settings.get('generate_js_files', 'false')):
237 if asbool(registry.settings.get('generate_js_files', 'false')):
238 static_path = AssetResolver().resolve('rhodecode:public').abspath()
238 static_path = AssetResolver().resolve('rhodecode:public').abspath()
239 jsroutes = get_routes()
239 jsroutes = get_routes()
240 jsroutes_file_content = generate_jsroutes_content(jsroutes)
240 jsroutes_file_content = generate_jsroutes_content(jsroutes)
241 jsroutes_file_path = os.path.join(
241 jsroutes_file_path = os.path.join(
242 static_path, 'js', 'rhodecode', 'routes.js')
242 static_path, 'js', 'rhodecode', 'routes.js')
243
243
244 try:
244 with io.open(jsroutes_file_path, 'w', encoding='utf-8') as f:
245 with io.open(jsroutes_file_path, 'w', encoding='utf-8') as f:
245 f.write(jsroutes_file_content)
246 f.write(jsroutes_file_content)
247 except Exception:
248 log.exception('Failed to write routes.js into %s', jsroutes_file_path)
246
249
247
250
248 class Subscriber(object):
251 class Subscriber(object):
249 """
252 """
250 Base class for subscribers to the pyramid event system.
253 Base class for subscribers to the pyramid event system.
251 """
254 """
252 def __call__(self, event):
255 def __call__(self, event):
253 self.run(event)
256 self.run(event)
254
257
255 def run(self, event):
258 def run(self, event):
256 raise NotImplementedError('Subclass has to implement this.')
259 raise NotImplementedError('Subclass has to implement this.')
257
260
258
261
259 class AsyncSubscriber(Subscriber):
262 class AsyncSubscriber(Subscriber):
260 """
263 """
261 Subscriber that handles the execution of events in a separate task to not
264 Subscriber that handles the execution of events in a separate task to not
262 block the execution of the code which triggers the event. It puts the
265 block the execution of the code which triggers the event. It puts the
263 received events into a queue from which the worker process takes them in
266 received events into a queue from which the worker process takes them in
264 order.
267 order.
265 """
268 """
266 def __init__(self):
269 def __init__(self):
267 self._stop = False
270 self._stop = False
268 self._eventq = Queue.Queue()
271 self._eventq = Queue.Queue()
269 self._worker = self.create_worker()
272 self._worker = self.create_worker()
270 self._worker.start()
273 self._worker.start()
271
274
272 def __call__(self, event):
275 def __call__(self, event):
273 self._eventq.put(event)
276 self._eventq.put(event)
274
277
275 def create_worker(self):
278 def create_worker(self):
276 worker = Thread(target=self.do_work)
279 worker = Thread(target=self.do_work)
277 worker.daemon = True
280 worker.daemon = True
278 return worker
281 return worker
279
282
280 def stop_worker(self):
283 def stop_worker(self):
281 self._stop = False
284 self._stop = False
282 self._eventq.put(None)
285 self._eventq.put(None)
283 self._worker.join()
286 self._worker.join()
284
287
285 def do_work(self):
288 def do_work(self):
286 while not self._stop:
289 while not self._stop:
287 event = self._eventq.get()
290 event = self._eventq.get()
288 if event is not None:
291 if event is not None:
289 self.run(event)
292 self.run(event)
290
293
291
294
292 class AsyncSubprocessSubscriber(AsyncSubscriber):
295 class AsyncSubprocessSubscriber(AsyncSubscriber):
293 """
296 """
294 Subscriber that uses the subprocess32 module to execute a command if an
297 Subscriber that uses the subprocess32 module to execute a command if an
295 event is received. Events are handled asynchronously.
298 event is received. Events are handled asynchronously.
296 """
299 """
297
300
298 def __init__(self, cmd, timeout=None):
301 def __init__(self, cmd, timeout=None):
299 super(AsyncSubprocessSubscriber, self).__init__()
302 super(AsyncSubprocessSubscriber, self).__init__()
300 self._cmd = cmd
303 self._cmd = cmd
301 self._timeout = timeout
304 self._timeout = timeout
302
305
303 def run(self, event):
306 def run(self, event):
304 cmd = self._cmd
307 cmd = self._cmd
305 timeout = self._timeout
308 timeout = self._timeout
306 log.debug('Executing command %s.', cmd)
309 log.debug('Executing command %s.', cmd)
307
310
308 try:
311 try:
309 output = subprocess32.check_output(
312 output = subprocess32.check_output(
310 cmd, timeout=timeout, stderr=subprocess32.STDOUT)
313 cmd, timeout=timeout, stderr=subprocess32.STDOUT)
311 log.debug('Command finished %s', cmd)
314 log.debug('Command finished %s', cmd)
312 if output:
315 if output:
313 log.debug('Command output: %s', output)
316 log.debug('Command output: %s', output)
314 except subprocess32.TimeoutExpired as e:
317 except subprocess32.TimeoutExpired as e:
315 log.exception('Timeout while executing command.')
318 log.exception('Timeout while executing command.')
316 if e.output:
319 if e.output:
317 log.error('Command output: %s', e.output)
320 log.error('Command output: %s', e.output)
318 except subprocess32.CalledProcessError as e:
321 except subprocess32.CalledProcessError as e:
319 log.exception('Error while executing command.')
322 log.exception('Error while executing command.')
320 if e.output:
323 if e.output:
321 log.error('Command output: %s', e.output)
324 log.error('Command output: %s', e.output)
322 except:
325 except:
323 log.exception(
326 log.exception(
324 'Exception while executing command %s.', cmd)
327 'Exception while executing command %s.', cmd)
General Comments 0
You need to be logged in to leave comments. Login now