##// END OF EJS Templates
meta: fix write condition for meta file.
marcink -
r2544:47600b03 stable
parent child Browse files
Show More
@@ -1,322 +1,322 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2010-2018 RhodeCode GmbH
3 # Copyright (C) 2010-2018 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 import io
20 import io
21 import re
21 import re
22 import datetime
22 import datetime
23 import logging
23 import logging
24 import Queue
24 import Queue
25 import subprocess32
25 import subprocess32
26 import os
26 import os
27
27
28
28
29 from dateutil.parser import parse
29 from dateutil.parser import parse
30 from pyramid.i18n import get_localizer
30 from pyramid.i18n import get_localizer
31 from pyramid.threadlocal import get_current_request
31 from pyramid.threadlocal import get_current_request
32 from pyramid.interfaces import IRoutesMapper
32 from pyramid.interfaces import IRoutesMapper
33 from pyramid.settings import asbool
33 from pyramid.settings import asbool
34 from pyramid.path import AssetResolver
34 from pyramid.path import AssetResolver
35 from threading import Thread
35 from threading import Thread
36
36
37 from rhodecode.translation import _ as tsf
37 from rhodecode.translation import _ as tsf
38 from rhodecode.config.jsroutes import generate_jsroutes_content
38 from rhodecode.config.jsroutes import generate_jsroutes_content
39 from rhodecode.lib import auth
39 from rhodecode.lib import auth
40 from rhodecode.lib.base import get_auth_user
40 from rhodecode.lib.base import get_auth_user
41
41
42
42
43 import rhodecode
43 import rhodecode
44
44
45
45
46 log = logging.getLogger(__name__)
46 log = logging.getLogger(__name__)
47
47
48
48
49 def add_renderer_globals(event):
49 def add_renderer_globals(event):
50 from rhodecode.lib import helpers
50 from rhodecode.lib import helpers
51
51
52 # TODO: When executed in pyramid view context the request is not available
52 # TODO: When executed in pyramid view context the request is not available
53 # in the event. Find a better solution to get the request.
53 # in the event. Find a better solution to get the request.
54 request = event['request'] or get_current_request()
54 request = event['request'] or get_current_request()
55
55
56 # Add Pyramid translation as '_' to context
56 # Add Pyramid translation as '_' to context
57 event['_'] = request.translate
57 event['_'] = request.translate
58 event['_ungettext'] = request.plularize
58 event['_ungettext'] = request.plularize
59 event['h'] = helpers
59 event['h'] = helpers
60
60
61
61
62 def add_localizer(event):
62 def add_localizer(event):
63 request = event.request
63 request = event.request
64 localizer = request.localizer
64 localizer = request.localizer
65
65
66 def auto_translate(*args, **kwargs):
66 def auto_translate(*args, **kwargs):
67 return localizer.translate(tsf(*args, **kwargs))
67 return localizer.translate(tsf(*args, **kwargs))
68
68
69 request.translate = auto_translate
69 request.translate = auto_translate
70 request.plularize = localizer.pluralize
70 request.plularize = localizer.pluralize
71
71
72
72
73 def set_user_lang(event):
73 def set_user_lang(event):
74 request = event.request
74 request = event.request
75 cur_user = getattr(request, 'user', None)
75 cur_user = getattr(request, 'user', None)
76
76
77 if cur_user:
77 if cur_user:
78 user_lang = cur_user.get_instance().user_data.get('language')
78 user_lang = cur_user.get_instance().user_data.get('language')
79 if user_lang:
79 if user_lang:
80 log.debug('lang: setting current user:%s language to: %s', cur_user, user_lang)
80 log.debug('lang: setting current user:%s language to: %s', cur_user, user_lang)
81 event.request._LOCALE_ = user_lang
81 event.request._LOCALE_ = user_lang
82
82
83
83
84 def add_request_user_context(event):
84 def add_request_user_context(event):
85 """
85 """
86 Adds auth user into request context
86 Adds auth user into request context
87 """
87 """
88 request = event.request
88 request = event.request
89
89
90 if hasattr(request, 'vcs_call'):
90 if hasattr(request, 'vcs_call'):
91 # skip vcs calls
91 # skip vcs calls
92 return
92 return
93
93
94 if hasattr(request, 'rpc_method'):
94 if hasattr(request, 'rpc_method'):
95 # skip api calls
95 # skip api calls
96 return
96 return
97
97
98 auth_user = get_auth_user(request)
98 auth_user = get_auth_user(request)
99 request.user = auth_user
99 request.user = auth_user
100 request.environ['rc_auth_user'] = auth_user
100 request.environ['rc_auth_user'] = auth_user
101
101
102
102
103 def inject_app_settings(event):
103 def inject_app_settings(event):
104 settings = event.app.registry.settings
104 settings = event.app.registry.settings
105 # inject info about available permissions
105 # inject info about available permissions
106 auth.set_available_permissions(settings)
106 auth.set_available_permissions(settings)
107
107
108
108
109 def scan_repositories_if_enabled(event):
109 def scan_repositories_if_enabled(event):
110 """
110 """
111 This is subscribed to the `pyramid.events.ApplicationCreated` event. It
111 This is subscribed to the `pyramid.events.ApplicationCreated` event. It
112 does a repository scan if enabled in the settings.
112 does a repository scan if enabled in the settings.
113 """
113 """
114 settings = event.app.registry.settings
114 settings = event.app.registry.settings
115 vcs_server_enabled = settings['vcs.server.enable']
115 vcs_server_enabled = settings['vcs.server.enable']
116 import_on_startup = settings['startup.import_repos']
116 import_on_startup = settings['startup.import_repos']
117 if vcs_server_enabled and import_on_startup:
117 if vcs_server_enabled and import_on_startup:
118 from rhodecode.model.scm import ScmModel
118 from rhodecode.model.scm import ScmModel
119 from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_base_path
119 from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_base_path
120 repositories = ScmModel().repo_scan(get_rhodecode_base_path())
120 repositories = ScmModel().repo_scan(get_rhodecode_base_path())
121 repo2db_mapper(repositories, remove_obsolete=False)
121 repo2db_mapper(repositories, remove_obsolete=False)
122
122
123
123
124 def write_metadata_if_needed(event):
124 def write_metadata_if_needed(event):
125 """
125 """
126 Writes upgrade metadata
126 Writes upgrade metadata
127 """
127 """
128 import rhodecode
128 import rhodecode
129 from rhodecode.lib import system_info
129 from rhodecode.lib import system_info
130 from rhodecode.lib import ext_json
130 from rhodecode.lib import ext_json
131
131
132 fname = '.rcmetadata.json'
132 fname = '.rcmetadata.json'
133 ini_loc = os.path.dirname(rhodecode.CONFIG.get('__file__'))
133 ini_loc = os.path.dirname(rhodecode.CONFIG.get('__file__'))
134 metadata_destination = os.path.join(ini_loc, fname)
134 metadata_destination = os.path.join(ini_loc, fname)
135
135
136 def get_update_age():
136 def get_update_age():
137 now = datetime.datetime.utcnow()
137 now = datetime.datetime.utcnow()
138
138
139 with open(metadata_destination, 'rb') as f:
139 with open(metadata_destination, 'rb') as f:
140 data = ext_json.json.loads(f.read())
140 data = ext_json.json.loads(f.read())
141 if 'created_on' in data:
141 if 'created_on' in data:
142 update_date = parse(data['created_on'])
142 update_date = parse(data['created_on'])
143 diff = now - update_date
143 diff = now - update_date
144 return diff.total_seconds() / 60.0
144 return diff.total_seconds() / 60.0
145
145
146 return 0
146 return 0
147
147
148 def write():
148 def write():
149 configuration = system_info.SysInfo(
149 configuration = system_info.SysInfo(
150 system_info.rhodecode_config)()['value']
150 system_info.rhodecode_config)()['value']
151 license_token = configuration['config']['license_token']
151 license_token = configuration['config']['license_token']
152
152
153 setup = dict(
153 setup = dict(
154 workers=configuration['config']['server:main'].get(
154 workers=configuration['config']['server:main'].get(
155 'workers', '?'),
155 'workers', '?'),
156 worker_type=configuration['config']['server:main'].get(
156 worker_type=configuration['config']['server:main'].get(
157 'worker_class', 'sync'),
157 'worker_class', 'sync'),
158 )
158 )
159 dbinfo = system_info.SysInfo(system_info.database_info)()['value']
159 dbinfo = system_info.SysInfo(system_info.database_info)()['value']
160 del dbinfo['url']
160 del dbinfo['url']
161
161
162 metadata = dict(
162 metadata = dict(
163 desc='upgrade metadata info',
163 desc='upgrade metadata info',
164 license_token=license_token,
164 license_token=license_token,
165 created_on=datetime.datetime.utcnow().isoformat(),
165 created_on=datetime.datetime.utcnow().isoformat(),
166 usage=system_info.SysInfo(system_info.usage_info)()['value'],
166 usage=system_info.SysInfo(system_info.usage_info)()['value'],
167 platform=system_info.SysInfo(system_info.platform_type)()['value'],
167 platform=system_info.SysInfo(system_info.platform_type)()['value'],
168 database=dbinfo,
168 database=dbinfo,
169 cpu=system_info.SysInfo(system_info.cpu)()['value'],
169 cpu=system_info.SysInfo(system_info.cpu)()['value'],
170 memory=system_info.SysInfo(system_info.memory)()['value'],
170 memory=system_info.SysInfo(system_info.memory)()['value'],
171 setup=setup
171 setup=setup
172 )
172 )
173
173
174 with open(metadata_destination, 'wb') as f:
174 with open(metadata_destination, 'wb') as f:
175 f.write(ext_json.json.dumps(metadata))
175 f.write(ext_json.json.dumps(metadata))
176
176
177 settings = event.app.registry.settings
177 settings = event.app.registry.settings
178 if settings.get('metadata.skip'):
178 if settings.get('metadata.skip'):
179 return
179 return
180
180
181 # only write this every 24h, workers restart caused unwanted delays
181 # only write this every 24h, workers restart caused unwanted delays
182 try:
182 try:
183 age_in_min = get_update_age()
183 age_in_min = get_update_age()
184 except Exception:
184 except Exception:
185 age_in_min = 0
185 age_in_min = 0
186
186
187 if age_in_min < 60 * 60 * 24:
187 if age_in_min > 60 * 60 * 24:
188 return
188 return
189
189
190 try:
190 try:
191 write()
191 write()
192 except Exception:
192 except Exception:
193 pass
193 pass
194
194
195
195
196 def write_js_routes_if_enabled(event):
196 def write_js_routes_if_enabled(event):
197 registry = event.app.registry
197 registry = event.app.registry
198
198
199 mapper = registry.queryUtility(IRoutesMapper)
199 mapper = registry.queryUtility(IRoutesMapper)
200 _argument_prog = re.compile('\{(.*?)\}|:\((.*)\)')
200 _argument_prog = re.compile('\{(.*?)\}|:\((.*)\)')
201
201
202 def _extract_route_information(route):
202 def _extract_route_information(route):
203 """
203 """
204 Convert a route into tuple(name, path, args), eg:
204 Convert a route into tuple(name, path, args), eg:
205 ('show_user', '/profile/%(username)s', ['username'])
205 ('show_user', '/profile/%(username)s', ['username'])
206 """
206 """
207
207
208 routepath = route.pattern
208 routepath = route.pattern
209 pattern = route.pattern
209 pattern = route.pattern
210
210
211 def replace(matchobj):
211 def replace(matchobj):
212 if matchobj.group(1):
212 if matchobj.group(1):
213 return "%%(%s)s" % matchobj.group(1).split(':')[0]
213 return "%%(%s)s" % matchobj.group(1).split(':')[0]
214 else:
214 else:
215 return "%%(%s)s" % matchobj.group(2)
215 return "%%(%s)s" % matchobj.group(2)
216
216
217 routepath = _argument_prog.sub(replace, routepath)
217 routepath = _argument_prog.sub(replace, routepath)
218
218
219 if not routepath.startswith('/'):
219 if not routepath.startswith('/'):
220 routepath = '/'+routepath
220 routepath = '/'+routepath
221
221
222 return (
222 return (
223 route.name,
223 route.name,
224 routepath,
224 routepath,
225 [(arg[0].split(':')[0] if arg[0] != '' else arg[1])
225 [(arg[0].split(':')[0] if arg[0] != '' else arg[1])
226 for arg in _argument_prog.findall(pattern)]
226 for arg in _argument_prog.findall(pattern)]
227 )
227 )
228
228
229 def get_routes():
229 def get_routes():
230 # pyramid routes
230 # pyramid routes
231 for route in mapper.get_routes():
231 for route in mapper.get_routes():
232 if not route.name.startswith('__'):
232 if not route.name.startswith('__'):
233 yield _extract_route_information(route)
233 yield _extract_route_information(route)
234
234
235 if asbool(registry.settings.get('generate_js_files', 'false')):
235 if asbool(registry.settings.get('generate_js_files', 'false')):
236 static_path = AssetResolver().resolve('rhodecode:public').abspath()
236 static_path = AssetResolver().resolve('rhodecode:public').abspath()
237 jsroutes = get_routes()
237 jsroutes = get_routes()
238 jsroutes_file_content = generate_jsroutes_content(jsroutes)
238 jsroutes_file_content = generate_jsroutes_content(jsroutes)
239 jsroutes_file_path = os.path.join(
239 jsroutes_file_path = os.path.join(
240 static_path, 'js', 'rhodecode', 'routes.js')
240 static_path, 'js', 'rhodecode', 'routes.js')
241
241
242 with io.open(jsroutes_file_path, 'w', encoding='utf-8') as f:
242 with io.open(jsroutes_file_path, 'w', encoding='utf-8') as f:
243 f.write(jsroutes_file_content)
243 f.write(jsroutes_file_content)
244
244
245
245
246 class Subscriber(object):
246 class Subscriber(object):
247 """
247 """
248 Base class for subscribers to the pyramid event system.
248 Base class for subscribers to the pyramid event system.
249 """
249 """
250 def __call__(self, event):
250 def __call__(self, event):
251 self.run(event)
251 self.run(event)
252
252
253 def run(self, event):
253 def run(self, event):
254 raise NotImplementedError('Subclass has to implement this.')
254 raise NotImplementedError('Subclass has to implement this.')
255
255
256
256
257 class AsyncSubscriber(Subscriber):
257 class AsyncSubscriber(Subscriber):
258 """
258 """
259 Subscriber that handles the execution of events in a separate task to not
259 Subscriber that handles the execution of events in a separate task to not
260 block the execution of the code which triggers the event. It puts the
260 block the execution of the code which triggers the event. It puts the
261 received events into a queue from which the worker process takes them in
261 received events into a queue from which the worker process takes them in
262 order.
262 order.
263 """
263 """
264 def __init__(self):
264 def __init__(self):
265 self._stop = False
265 self._stop = False
266 self._eventq = Queue.Queue()
266 self._eventq = Queue.Queue()
267 self._worker = self.create_worker()
267 self._worker = self.create_worker()
268 self._worker.start()
268 self._worker.start()
269
269
270 def __call__(self, event):
270 def __call__(self, event):
271 self._eventq.put(event)
271 self._eventq.put(event)
272
272
273 def create_worker(self):
273 def create_worker(self):
274 worker = Thread(target=self.do_work)
274 worker = Thread(target=self.do_work)
275 worker.daemon = True
275 worker.daemon = True
276 return worker
276 return worker
277
277
278 def stop_worker(self):
278 def stop_worker(self):
279 self._stop = False
279 self._stop = False
280 self._eventq.put(None)
280 self._eventq.put(None)
281 self._worker.join()
281 self._worker.join()
282
282
283 def do_work(self):
283 def do_work(self):
284 while not self._stop:
284 while not self._stop:
285 event = self._eventq.get()
285 event = self._eventq.get()
286 if event is not None:
286 if event is not None:
287 self.run(event)
287 self.run(event)
288
288
289
289
290 class AsyncSubprocessSubscriber(AsyncSubscriber):
290 class AsyncSubprocessSubscriber(AsyncSubscriber):
291 """
291 """
292 Subscriber that uses the subprocess32 module to execute a command if an
292 Subscriber that uses the subprocess32 module to execute a command if an
293 event is received. Events are handled asynchronously.
293 event is received. Events are handled asynchronously.
294 """
294 """
295
295
296 def __init__(self, cmd, timeout=None):
296 def __init__(self, cmd, timeout=None):
297 super(AsyncSubprocessSubscriber, self).__init__()
297 super(AsyncSubprocessSubscriber, self).__init__()
298 self._cmd = cmd
298 self._cmd = cmd
299 self._timeout = timeout
299 self._timeout = timeout
300
300
301 def run(self, event):
301 def run(self, event):
302 cmd = self._cmd
302 cmd = self._cmd
303 timeout = self._timeout
303 timeout = self._timeout
304 log.debug('Executing command %s.', cmd)
304 log.debug('Executing command %s.', cmd)
305
305
306 try:
306 try:
307 output = subprocess32.check_output(
307 output = subprocess32.check_output(
308 cmd, timeout=timeout, stderr=subprocess32.STDOUT)
308 cmd, timeout=timeout, stderr=subprocess32.STDOUT)
309 log.debug('Command finished %s', cmd)
309 log.debug('Command finished %s', cmd)
310 if output:
310 if output:
311 log.debug('Command output: %s', output)
311 log.debug('Command output: %s', output)
312 except subprocess32.TimeoutExpired as e:
312 except subprocess32.TimeoutExpired as e:
313 log.exception('Timeout while executing command.')
313 log.exception('Timeout while executing command.')
314 if e.output:
314 if e.output:
315 log.error('Command output: %s', e.output)
315 log.error('Command output: %s', e.output)
316 except subprocess32.CalledProcessError as e:
316 except subprocess32.CalledProcessError as e:
317 log.exception('Error while executing command.')
317 log.exception('Error while executing command.')
318 if e.output:
318 if e.output:
319 log.error('Command output: %s', e.output)
319 log.error('Command output: %s', e.output)
320 except:
320 except:
321 log.exception(
321 log.exception(
322 'Exception while executing command %s.', cmd)
322 'Exception while executing command %s.', cmd)
General Comments 0
You need to be logged in to leave comments. Login now