##// END OF EJS Templates
file-store: rename module from upload_store to file_store.
marcink -
r3453:c25a3725 default
parent child Browse files
Show More
@@ -1,481 +1,481 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2011-2019 RhodeCode GmbH
3 # Copyright (C) 2011-2019 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import inspect
21 import inspect
22 import logging
22 import logging
23 import itertools
23 import itertools
24 import base64
24 import base64
25
25
26 from pyramid import compat
26 from pyramid import compat
27
27
28 from rhodecode.api import (
28 from rhodecode.api import (
29 jsonrpc_method, JSONRPCError, JSONRPCForbidden, find_methods)
29 jsonrpc_method, JSONRPCError, JSONRPCForbidden, find_methods)
30
30
31 from rhodecode.api.utils import (
31 from rhodecode.api.utils import (
32 Optional, OAttr, has_superadmin_permission, get_user_or_error)
32 Optional, OAttr, has_superadmin_permission, get_user_or_error)
33 from rhodecode.lib.utils import repo2db_mapper
33 from rhodecode.lib.utils import repo2db_mapper
34 from rhodecode.lib import system_info
34 from rhodecode.lib import system_info
35 from rhodecode.lib import user_sessions
35 from rhodecode.lib import user_sessions
36 from rhodecode.lib import exc_tracking
36 from rhodecode.lib import exc_tracking
37 from rhodecode.lib.ext_json import json
37 from rhodecode.lib.ext_json import json
38 from rhodecode.lib.utils2 import safe_int
38 from rhodecode.lib.utils2 import safe_int
39 from rhodecode.model.db import UserIpMap
39 from rhodecode.model.db import UserIpMap
40 from rhodecode.model.scm import ScmModel
40 from rhodecode.model.scm import ScmModel
41 from rhodecode.model.settings import VcsSettingsModel
41 from rhodecode.model.settings import VcsSettingsModel
42 from rhodecode.apps.upload_store import utils
42 from rhodecode.apps.file_store import utils
43 from rhodecode.apps.upload_store.exceptions import FileNotAllowedException, \
43 from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \
44 FileOverSizeException
44 FileOverSizeException
45
45
46 log = logging.getLogger(__name__)
46 log = logging.getLogger(__name__)
47
47
48
48
49 @jsonrpc_method()
49 @jsonrpc_method()
50 def get_server_info(request, apiuser):
50 def get_server_info(request, apiuser):
51 """
51 """
52 Returns the |RCE| server information.
52 Returns the |RCE| server information.
53
53
54 This includes the running version of |RCE| and all installed
54 This includes the running version of |RCE| and all installed
55 packages. This command takes the following options:
55 packages. This command takes the following options:
56
56
57 :param apiuser: This is filled automatically from the |authtoken|.
57 :param apiuser: This is filled automatically from the |authtoken|.
58 :type apiuser: AuthUser
58 :type apiuser: AuthUser
59
59
60 Example output:
60 Example output:
61
61
62 .. code-block:: bash
62 .. code-block:: bash
63
63
64 id : <id_given_in_input>
64 id : <id_given_in_input>
65 result : {
65 result : {
66 'modules': [<module name>,...]
66 'modules': [<module name>,...]
67 'py_version': <python version>,
67 'py_version': <python version>,
68 'platform': <platform type>,
68 'platform': <platform type>,
69 'rhodecode_version': <rhodecode version>
69 'rhodecode_version': <rhodecode version>
70 }
70 }
71 error : null
71 error : null
72 """
72 """
73
73
74 if not has_superadmin_permission(apiuser):
74 if not has_superadmin_permission(apiuser):
75 raise JSONRPCForbidden()
75 raise JSONRPCForbidden()
76
76
77 server_info = ScmModel().get_server_info(request.environ)
77 server_info = ScmModel().get_server_info(request.environ)
78 # rhodecode-index requires those
78 # rhodecode-index requires those
79
79
80 server_info['index_storage'] = server_info['search']['value']['location']
80 server_info['index_storage'] = server_info['search']['value']['location']
81 server_info['storage'] = server_info['storage']['value']['path']
81 server_info['storage'] = server_info['storage']['value']['path']
82
82
83 return server_info
83 return server_info
84
84
85
85
86 @jsonrpc_method()
86 @jsonrpc_method()
87 def get_repo_store(request, apiuser):
87 def get_repo_store(request, apiuser):
88 """
88 """
89 Returns the |RCE| repository storage information.
89 Returns the |RCE| repository storage information.
90
90
91 :param apiuser: This is filled automatically from the |authtoken|.
91 :param apiuser: This is filled automatically from the |authtoken|.
92 :type apiuser: AuthUser
92 :type apiuser: AuthUser
93
93
94 Example output:
94 Example output:
95
95
96 .. code-block:: bash
96 .. code-block:: bash
97
97
98 id : <id_given_in_input>
98 id : <id_given_in_input>
99 result : {
99 result : {
100 'modules': [<module name>,...]
100 'modules': [<module name>,...]
101 'py_version': <python version>,
101 'py_version': <python version>,
102 'platform': <platform type>,
102 'platform': <platform type>,
103 'rhodecode_version': <rhodecode version>
103 'rhodecode_version': <rhodecode version>
104 }
104 }
105 error : null
105 error : null
106 """
106 """
107
107
108 if not has_superadmin_permission(apiuser):
108 if not has_superadmin_permission(apiuser):
109 raise JSONRPCForbidden()
109 raise JSONRPCForbidden()
110
110
111 path = VcsSettingsModel().get_repos_location()
111 path = VcsSettingsModel().get_repos_location()
112 return {"path": path}
112 return {"path": path}
113
113
114
114
115 @jsonrpc_method()
115 @jsonrpc_method()
116 def get_ip(request, apiuser, userid=Optional(OAttr('apiuser'))):
116 def get_ip(request, apiuser, userid=Optional(OAttr('apiuser'))):
117 """
117 """
118 Displays the IP Address as seen from the |RCE| server.
118 Displays the IP Address as seen from the |RCE| server.
119
119
120 * This command displays the IP Address, as well as all the defined IP
120 * This command displays the IP Address, as well as all the defined IP
121 addresses for the specified user. If the ``userid`` is not set, the
121 addresses for the specified user. If the ``userid`` is not set, the
122 data returned is for the user calling the method.
122 data returned is for the user calling the method.
123
123
124 This command can only be run using an |authtoken| with admin rights to
124 This command can only be run using an |authtoken| with admin rights to
125 the specified repository.
125 the specified repository.
126
126
127 This command takes the following options:
127 This command takes the following options:
128
128
129 :param apiuser: This is filled automatically from |authtoken|.
129 :param apiuser: This is filled automatically from |authtoken|.
130 :type apiuser: AuthUser
130 :type apiuser: AuthUser
131 :param userid: Sets the userid for which associated IP Address data
131 :param userid: Sets the userid for which associated IP Address data
132 is returned.
132 is returned.
133 :type userid: Optional(str or int)
133 :type userid: Optional(str or int)
134
134
135 Example output:
135 Example output:
136
136
137 .. code-block:: bash
137 .. code-block:: bash
138
138
139 id : <id_given_in_input>
139 id : <id_given_in_input>
140 result : {
140 result : {
141 "server_ip_addr": "<ip_from_clien>",
141 "server_ip_addr": "<ip_from_clien>",
142 "user_ips": [
142 "user_ips": [
143 {
143 {
144 "ip_addr": "<ip_with_mask>",
144 "ip_addr": "<ip_with_mask>",
145 "ip_range": ["<start_ip>", "<end_ip>"],
145 "ip_range": ["<start_ip>", "<end_ip>"],
146 },
146 },
147 ...
147 ...
148 ]
148 ]
149 }
149 }
150
150
151 """
151 """
152 if not has_superadmin_permission(apiuser):
152 if not has_superadmin_permission(apiuser):
153 raise JSONRPCForbidden()
153 raise JSONRPCForbidden()
154
154
155 userid = Optional.extract(userid, evaluate_locals=locals())
155 userid = Optional.extract(userid, evaluate_locals=locals())
156 userid = getattr(userid, 'user_id', userid)
156 userid = getattr(userid, 'user_id', userid)
157
157
158 user = get_user_or_error(userid)
158 user = get_user_or_error(userid)
159 ips = UserIpMap.query().filter(UserIpMap.user == user).all()
159 ips = UserIpMap.query().filter(UserIpMap.user == user).all()
160 return {
160 return {
161 'server_ip_addr': request.rpc_ip_addr,
161 'server_ip_addr': request.rpc_ip_addr,
162 'user_ips': ips
162 'user_ips': ips
163 }
163 }
164
164
165
165
166 @jsonrpc_method()
166 @jsonrpc_method()
167 def rescan_repos(request, apiuser, remove_obsolete=Optional(False)):
167 def rescan_repos(request, apiuser, remove_obsolete=Optional(False)):
168 """
168 """
169 Triggers a rescan of the specified repositories.
169 Triggers a rescan of the specified repositories.
170
170
171 * If the ``remove_obsolete`` option is set, it also deletes repositories
171 * If the ``remove_obsolete`` option is set, it also deletes repositories
172 that are found in the database but not on the file system, so called
172 that are found in the database but not on the file system, so called
173 "clean zombies".
173 "clean zombies".
174
174
175 This command can only be run using an |authtoken| with admin rights to
175 This command can only be run using an |authtoken| with admin rights to
176 the specified repository.
176 the specified repository.
177
177
178 This command takes the following options:
178 This command takes the following options:
179
179
180 :param apiuser: This is filled automatically from the |authtoken|.
180 :param apiuser: This is filled automatically from the |authtoken|.
181 :type apiuser: AuthUser
181 :type apiuser: AuthUser
182 :param remove_obsolete: Deletes repositories from the database that
182 :param remove_obsolete: Deletes repositories from the database that
183 are not found on the filesystem.
183 are not found on the filesystem.
184 :type remove_obsolete: Optional(``True`` | ``False``)
184 :type remove_obsolete: Optional(``True`` | ``False``)
185
185
186 Example output:
186 Example output:
187
187
188 .. code-block:: bash
188 .. code-block:: bash
189
189
190 id : <id_given_in_input>
190 id : <id_given_in_input>
191 result : {
191 result : {
192 'added': [<added repository name>,...]
192 'added': [<added repository name>,...]
193 'removed': [<removed repository name>,...]
193 'removed': [<removed repository name>,...]
194 }
194 }
195 error : null
195 error : null
196
196
197 Example error output:
197 Example error output:
198
198
199 .. code-block:: bash
199 .. code-block:: bash
200
200
201 id : <id_given_in_input>
201 id : <id_given_in_input>
202 result : null
202 result : null
203 error : {
203 error : {
204 'Error occurred during rescan repositories action'
204 'Error occurred during rescan repositories action'
205 }
205 }
206
206
207 """
207 """
208 if not has_superadmin_permission(apiuser):
208 if not has_superadmin_permission(apiuser):
209 raise JSONRPCForbidden()
209 raise JSONRPCForbidden()
210
210
211 try:
211 try:
212 rm_obsolete = Optional.extract(remove_obsolete)
212 rm_obsolete = Optional.extract(remove_obsolete)
213 added, removed = repo2db_mapper(ScmModel().repo_scan(),
213 added, removed = repo2db_mapper(ScmModel().repo_scan(),
214 remove_obsolete=rm_obsolete)
214 remove_obsolete=rm_obsolete)
215 return {'added': added, 'removed': removed}
215 return {'added': added, 'removed': removed}
216 except Exception:
216 except Exception:
217 log.exception('Failed to run repo rescann')
217 log.exception('Failed to run repo rescann')
218 raise JSONRPCError(
218 raise JSONRPCError(
219 'Error occurred during rescan repositories action'
219 'Error occurred during rescan repositories action'
220 )
220 )
221
221
222
222
223 @jsonrpc_method()
223 @jsonrpc_method()
224 def cleanup_sessions(request, apiuser, older_then=Optional(60)):
224 def cleanup_sessions(request, apiuser, older_then=Optional(60)):
225 """
225 """
226 Triggers a session cleanup action.
226 Triggers a session cleanup action.
227
227
228 If the ``older_then`` option is set, only sessions that hasn't been
228 If the ``older_then`` option is set, only sessions that hasn't been
229 accessed in the given number of days will be removed.
229 accessed in the given number of days will be removed.
230
230
231 This command can only be run using an |authtoken| with admin rights to
231 This command can only be run using an |authtoken| with admin rights to
232 the specified repository.
232 the specified repository.
233
233
234 This command takes the following options:
234 This command takes the following options:
235
235
236 :param apiuser: This is filled automatically from the |authtoken|.
236 :param apiuser: This is filled automatically from the |authtoken|.
237 :type apiuser: AuthUser
237 :type apiuser: AuthUser
238 :param older_then: Deletes session that hasn't been accessed
238 :param older_then: Deletes session that hasn't been accessed
239 in given number of days.
239 in given number of days.
240 :type older_then: Optional(int)
240 :type older_then: Optional(int)
241
241
242 Example output:
242 Example output:
243
243
244 .. code-block:: bash
244 .. code-block:: bash
245
245
246 id : <id_given_in_input>
246 id : <id_given_in_input>
247 result: {
247 result: {
248 "backend": "<type of backend>",
248 "backend": "<type of backend>",
249 "sessions_removed": <number_of_removed_sessions>
249 "sessions_removed": <number_of_removed_sessions>
250 }
250 }
251 error : null
251 error : null
252
252
253 Example error output:
253 Example error output:
254
254
255 .. code-block:: bash
255 .. code-block:: bash
256
256
257 id : <id_given_in_input>
257 id : <id_given_in_input>
258 result : null
258 result : null
259 error : {
259 error : {
260 'Error occurred during session cleanup'
260 'Error occurred during session cleanup'
261 }
261 }
262
262
263 """
263 """
264 if not has_superadmin_permission(apiuser):
264 if not has_superadmin_permission(apiuser):
265 raise JSONRPCForbidden()
265 raise JSONRPCForbidden()
266
266
267 older_then = safe_int(Optional.extract(older_then)) or 60
267 older_then = safe_int(Optional.extract(older_then)) or 60
268 older_than_seconds = 60 * 60 * 24 * older_then
268 older_than_seconds = 60 * 60 * 24 * older_then
269
269
270 config = system_info.rhodecode_config().get_value()['value']['config']
270 config = system_info.rhodecode_config().get_value()['value']['config']
271 session_model = user_sessions.get_session_handler(
271 session_model = user_sessions.get_session_handler(
272 config.get('beaker.session.type', 'memory'))(config)
272 config.get('beaker.session.type', 'memory'))(config)
273
273
274 backend = session_model.SESSION_TYPE
274 backend = session_model.SESSION_TYPE
275 try:
275 try:
276 cleaned = session_model.clean_sessions(
276 cleaned = session_model.clean_sessions(
277 older_than_seconds=older_than_seconds)
277 older_than_seconds=older_than_seconds)
278 return {'sessions_removed': cleaned, 'backend': backend}
278 return {'sessions_removed': cleaned, 'backend': backend}
279 except user_sessions.CleanupCommand as msg:
279 except user_sessions.CleanupCommand as msg:
280 return {'cleanup_command': msg.message, 'backend': backend}
280 return {'cleanup_command': msg.message, 'backend': backend}
281 except Exception as e:
281 except Exception as e:
282 log.exception('Failed session cleanup')
282 log.exception('Failed session cleanup')
283 raise JSONRPCError(
283 raise JSONRPCError(
284 'Error occurred during session cleanup'
284 'Error occurred during session cleanup'
285 )
285 )
286
286
287
287
288 @jsonrpc_method()
288 @jsonrpc_method()
289 def get_method(request, apiuser, pattern=Optional('*')):
289 def get_method(request, apiuser, pattern=Optional('*')):
290 """
290 """
291 Returns list of all available API methods. By default match pattern
291 Returns list of all available API methods. By default match pattern
292 os "*" but any other pattern can be specified. eg *comment* will return
292 os "*" but any other pattern can be specified. eg *comment* will return
293 all methods with comment inside them. If just single method is matched
293 all methods with comment inside them. If just single method is matched
294 returned data will also include method specification
294 returned data will also include method specification
295
295
296 This command can only be run using an |authtoken| with admin rights to
296 This command can only be run using an |authtoken| with admin rights to
297 the specified repository.
297 the specified repository.
298
298
299 This command takes the following options:
299 This command takes the following options:
300
300
301 :param apiuser: This is filled automatically from the |authtoken|.
301 :param apiuser: This is filled automatically from the |authtoken|.
302 :type apiuser: AuthUser
302 :type apiuser: AuthUser
303 :param pattern: pattern to match method names against
303 :param pattern: pattern to match method names against
304 :type pattern: Optional("*")
304 :type pattern: Optional("*")
305
305
306 Example output:
306 Example output:
307
307
308 .. code-block:: bash
308 .. code-block:: bash
309
309
310 id : <id_given_in_input>
310 id : <id_given_in_input>
311 "result": [
311 "result": [
312 "changeset_comment",
312 "changeset_comment",
313 "comment_pull_request",
313 "comment_pull_request",
314 "comment_commit"
314 "comment_commit"
315 ]
315 ]
316 error : null
316 error : null
317
317
318 .. code-block:: bash
318 .. code-block:: bash
319
319
320 id : <id_given_in_input>
320 id : <id_given_in_input>
321 "result": [
321 "result": [
322 "comment_commit",
322 "comment_commit",
323 {
323 {
324 "apiuser": "<RequiredType>",
324 "apiuser": "<RequiredType>",
325 "comment_type": "<Optional:u'note'>",
325 "comment_type": "<Optional:u'note'>",
326 "commit_id": "<RequiredType>",
326 "commit_id": "<RequiredType>",
327 "message": "<RequiredType>",
327 "message": "<RequiredType>",
328 "repoid": "<RequiredType>",
328 "repoid": "<RequiredType>",
329 "request": "<RequiredType>",
329 "request": "<RequiredType>",
330 "resolves_comment_id": "<Optional:None>",
330 "resolves_comment_id": "<Optional:None>",
331 "status": "<Optional:None>",
331 "status": "<Optional:None>",
332 "userid": "<Optional:<OptionalAttr:apiuser>>"
332 "userid": "<Optional:<OptionalAttr:apiuser>>"
333 }
333 }
334 ]
334 ]
335 error : null
335 error : null
336 """
336 """
337 if not has_superadmin_permission(apiuser):
337 if not has_superadmin_permission(apiuser):
338 raise JSONRPCForbidden()
338 raise JSONRPCForbidden()
339
339
340 pattern = Optional.extract(pattern)
340 pattern = Optional.extract(pattern)
341
341
342 matches = find_methods(request.registry.jsonrpc_methods, pattern)
342 matches = find_methods(request.registry.jsonrpc_methods, pattern)
343
343
344 args_desc = []
344 args_desc = []
345 if len(matches) == 1:
345 if len(matches) == 1:
346 func = matches[matches.keys()[0]]
346 func = matches[matches.keys()[0]]
347
347
348 argspec = inspect.getargspec(func)
348 argspec = inspect.getargspec(func)
349 arglist = argspec[0]
349 arglist = argspec[0]
350 defaults = map(repr, argspec[3] or [])
350 defaults = map(repr, argspec[3] or [])
351
351
352 default_empty = '<RequiredType>'
352 default_empty = '<RequiredType>'
353
353
354 # kw arguments required by this method
354 # kw arguments required by this method
355 func_kwargs = dict(itertools.izip_longest(
355 func_kwargs = dict(itertools.izip_longest(
356 reversed(arglist), reversed(defaults), fillvalue=default_empty))
356 reversed(arglist), reversed(defaults), fillvalue=default_empty))
357 args_desc.append(func_kwargs)
357 args_desc.append(func_kwargs)
358
358
359 return matches.keys() + args_desc
359 return matches.keys() + args_desc
360
360
361
361
362 @jsonrpc_method()
362 @jsonrpc_method()
363 def store_exception(request, apiuser, exc_data_json, prefix=Optional('rhodecode')):
363 def store_exception(request, apiuser, exc_data_json, prefix=Optional('rhodecode')):
364 """
364 """
365 Stores sent exception inside the built-in exception tracker in |RCE| server.
365 Stores sent exception inside the built-in exception tracker in |RCE| server.
366
366
367 This command can only be run using an |authtoken| with admin rights to
367 This command can only be run using an |authtoken| with admin rights to
368 the specified repository.
368 the specified repository.
369
369
370 This command takes the following options:
370 This command takes the following options:
371
371
372 :param apiuser: This is filled automatically from the |authtoken|.
372 :param apiuser: This is filled automatically from the |authtoken|.
373 :type apiuser: AuthUser
373 :type apiuser: AuthUser
374
374
375 :param exc_data_json: JSON data with exception e.g
375 :param exc_data_json: JSON data with exception e.g
376 {"exc_traceback": "Value `1` is not allowed", "exc_type_name": "ValueError"}
376 {"exc_traceback": "Value `1` is not allowed", "exc_type_name": "ValueError"}
377 :type exc_data_json: JSON data
377 :type exc_data_json: JSON data
378
378
379 :param prefix: prefix for error type, e.g 'rhodecode', 'vcsserver', 'rhodecode-tools'
379 :param prefix: prefix for error type, e.g 'rhodecode', 'vcsserver', 'rhodecode-tools'
380 :type prefix: Optional("rhodecode")
380 :type prefix: Optional("rhodecode")
381
381
382 Example output:
382 Example output:
383
383
384 .. code-block:: bash
384 .. code-block:: bash
385
385
386 id : <id_given_in_input>
386 id : <id_given_in_input>
387 "result": {
387 "result": {
388 "exc_id": 139718459226384,
388 "exc_id": 139718459226384,
389 "exc_url": "http://localhost:8080/_admin/settings/exceptions/139718459226384"
389 "exc_url": "http://localhost:8080/_admin/settings/exceptions/139718459226384"
390 }
390 }
391 error : null
391 error : null
392 """
392 """
393 if not has_superadmin_permission(apiuser):
393 if not has_superadmin_permission(apiuser):
394 raise JSONRPCForbidden()
394 raise JSONRPCForbidden()
395
395
396 prefix = Optional.extract(prefix)
396 prefix = Optional.extract(prefix)
397 exc_id = exc_tracking.generate_id()
397 exc_id = exc_tracking.generate_id()
398
398
399 try:
399 try:
400 exc_data = json.loads(exc_data_json)
400 exc_data = json.loads(exc_data_json)
401 except Exception:
401 except Exception:
402 log.error('Failed to parse JSON: %r', exc_data_json)
402 log.error('Failed to parse JSON: %r', exc_data_json)
403 raise JSONRPCError('Failed to parse JSON data from exc_data_json field. '
403 raise JSONRPCError('Failed to parse JSON data from exc_data_json field. '
404 'Please make sure it contains a valid JSON.')
404 'Please make sure it contains a valid JSON.')
405
405
406 try:
406 try:
407 exc_traceback = exc_data['exc_traceback']
407 exc_traceback = exc_data['exc_traceback']
408 exc_type_name = exc_data['exc_type_name']
408 exc_type_name = exc_data['exc_type_name']
409 except KeyError as err:
409 except KeyError as err:
410 raise JSONRPCError('Missing exc_traceback, or exc_type_name '
410 raise JSONRPCError('Missing exc_traceback, or exc_type_name '
411 'in exc_data_json field. Missing: {}'.format(err))
411 'in exc_data_json field. Missing: {}'.format(err))
412
412
413 exc_tracking._store_exception(
413 exc_tracking._store_exception(
414 exc_id=exc_id, exc_traceback=exc_traceback,
414 exc_id=exc_id, exc_traceback=exc_traceback,
415 exc_type_name=exc_type_name, prefix=prefix)
415 exc_type_name=exc_type_name, prefix=prefix)
416
416
417 exc_url = request.route_url(
417 exc_url = request.route_url(
418 'admin_settings_exception_tracker_show', exception_id=exc_id)
418 'admin_settings_exception_tracker_show', exception_id=exc_id)
419 return {'exc_id': exc_id, 'exc_url': exc_url}
419 return {'exc_id': exc_id, 'exc_url': exc_url}
420
420
421
421
422 @jsonrpc_method()
422 @jsonrpc_method()
423 def upload_file(request, apiuser, filename, content):
423 def upload_file(request, apiuser, filename, content):
424 """
424 """
425 Upload API for the file_store
425 Upload API for the file_store
426
426
427 Example usage from CLI::
427 Example usage from CLI::
428 rhodecode-api --instance-name=enterprise-1 upload_file "{\"content\": \"$(cat image.jpg | base64)\", \"filename\":\"image.jpg\"}"
428 rhodecode-api --instance-name=enterprise-1 upload_file "{\"content\": \"$(cat image.jpg | base64)\", \"filename\":\"image.jpg\"}"
429
429
430
430
431 This command can only be run using an |authtoken| with admin rights to
431 This command can only be run using an |authtoken| with admin rights to
432 the specified repository.
432 the specified repository.
433
433
434 This command takes the following options:
434 This command takes the following options:
435
435
436 :param apiuser: This is filled automatically from the |authtoken|.
436 :param apiuser: This is filled automatically from the |authtoken|.
437 :type apiuser: AuthUser
437 :type apiuser: AuthUser
438 :param filename: name of the file uploaded
438 :param filename: name of the file uploaded
439 :type filename: str
439 :type filename: str
440 :param content: base64 encoded content of the uploaded file
440 :param content: base64 encoded content of the uploaded file
441 :type prefix: str
441 :type prefix: str
442
442
443 Example output:
443 Example output:
444
444
445 .. code-block:: bash
445 .. code-block:: bash
446
446
447 id : <id_given_in_input>
447 id : <id_given_in_input>
448 result: {
448 result: {
449 "access_path": "/_file_store/download/84d156f7-8323-4ad3-9fce-4a8e88e1deaf-0.jpg",
449 "access_path": "/_file_store/download/84d156f7-8323-4ad3-9fce-4a8e88e1deaf-0.jpg",
450 "access_path_fqn": "http://server.domain.com/_file_store/download/84d156f7-8323-4ad3-9fce-4a8e88e1deaf-0.jpg",
450 "access_path_fqn": "http://server.domain.com/_file_store/download/84d156f7-8323-4ad3-9fce-4a8e88e1deaf-0.jpg",
451 "store_fid": "84d156f7-8323-4ad3-9fce-4a8e88e1deaf-0.jpg"
451 "store_fid": "84d156f7-8323-4ad3-9fce-4a8e88e1deaf-0.jpg"
452 }
452 }
453 error : null
453 error : null
454 """
454 """
455 if not has_superadmin_permission(apiuser):
455 if not has_superadmin_permission(apiuser):
456 raise JSONRPCForbidden()
456 raise JSONRPCForbidden()
457
457
458 storage = utils.get_file_storage(request.registry.settings)
458 storage = utils.get_file_storage(request.registry.settings)
459
459
460 try:
460 try:
461 file_obj = compat.NativeIO(base64.decodestring(content))
461 file_obj = compat.NativeIO(base64.decodestring(content))
462 except Exception as exc:
462 except Exception as exc:
463 raise JSONRPCError('File `{}` content decoding error: {}.'.format(filename, exc))
463 raise JSONRPCError('File `{}` content decoding error: {}.'.format(filename, exc))
464
464
465 metadata = {
465 metadata = {
466 'filename': filename,
466 'filename': filename,
467 'size': '', # filled by save_file
467 'size': '', # filled by save_file
468 'user_uploaded': {'username': apiuser.username,
468 'user_uploaded': {'username': apiuser.username,
469 'user_id': apiuser.user_id,
469 'user_id': apiuser.user_id,
470 'ip': apiuser.ip_addr}}
470 'ip': apiuser.ip_addr}}
471 try:
471 try:
472 store_fid = storage.save_file(file_obj, filename, metadata=metadata)
472 store_fid, metadata = storage.save_file(file_obj, filename, metadata=metadata)
473 except FileNotAllowedException:
473 except FileNotAllowedException:
474 raise JSONRPCError('File `{}` is not allowed.'.format(filename))
474 raise JSONRPCError('File `{}` is not allowed.'.format(filename))
475
475
476 except FileOverSizeException:
476 except FileOverSizeException:
477 raise JSONRPCError('File `{}` is exceeding allowed limit.'.format(filename))
477 raise JSONRPCError('File `{}` is exceeding allowed limit.'.format(filename))
478
478
479 return {'store_fid': store_fid,
479 return {'store_fid': store_fid,
480 'access_path_fqn': request.route_url('download_file', fid=store_fid),
480 'access_path_fqn': request.route_url('download_file', fid=store_fid),
481 'access_path': request.route_path('download_file', fid=store_fid)}
481 'access_path': request.route_path('download_file', fid=store_fid)}
@@ -1,49 +1,49 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2016-2019 RhodeCode GmbH
3 # Copyright (C) 2016-2019 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 import os
20 import os
21 from rhodecode.apps.upload_store import config_keys
21 from rhodecode.apps.file_store import config_keys
22 from rhodecode.config.middleware import _bool_setting, _string_setting
22 from rhodecode.config.middleware import _bool_setting, _string_setting
23
23
24
24
25 def _sanitize_settings_and_apply_defaults(settings):
25 def _sanitize_settings_and_apply_defaults(settings):
26 """
26 """
27 Set defaults, convert to python types and validate settings.
27 Set defaults, convert to python types and validate settings.
28 """
28 """
29 _bool_setting(settings, config_keys.enabled, 'true')
29 _bool_setting(settings, config_keys.enabled, 'true')
30
30
31 _string_setting(settings, config_keys.backend, 'local')
31 _string_setting(settings, config_keys.backend, 'local')
32
32
33 default_store = os.path.join(os.path.dirname(settings['__file__']), 'upload_store')
33 default_store = os.path.join(os.path.dirname(settings['__file__']), 'upload_store')
34 _string_setting(settings, config_keys.store_path, default_store)
34 _string_setting(settings, config_keys.store_path, default_store)
35
35
36
36
37 def includeme(config):
37 def includeme(config):
38 settings = config.registry.settings
38 settings = config.registry.settings
39 _sanitize_settings_and_apply_defaults(settings)
39 _sanitize_settings_and_apply_defaults(settings)
40
40
41 config.add_route(
41 config.add_route(
42 name='upload_file',
42 name='upload_file',
43 pattern='/_file_store/upload')
43 pattern='/_file_store/upload')
44 config.add_route(
44 config.add_route(
45 name='download_file',
45 name='download_file',
46 pattern='/_file_store/download/{fid}')
46 pattern='/_file_store/download/{fid}')
47
47
48 # Scan module for configuration decorators.
48 # Scan module for configuration decorators.
49 config.scan('.views', ignore='.tests')
49 config.scan('.views', ignore='.tests')
1 NO CONTENT: file renamed from rhodecode/apps/upload_store/config_keys.py to rhodecode/apps/file_store/config_keys.py
NO CONTENT: file renamed from rhodecode/apps/upload_store/config_keys.py to rhodecode/apps/file_store/config_keys.py
1 NO CONTENT: file renamed from rhodecode/apps/upload_store/exceptions.py to rhodecode/apps/file_store/exceptions.py
NO CONTENT: file renamed from rhodecode/apps/upload_store/exceptions.py to rhodecode/apps/file_store/exceptions.py
1 NO CONTENT: file renamed from rhodecode/apps/upload_store/extensions.py to rhodecode/apps/file_store/extensions.py
NO CONTENT: file renamed from rhodecode/apps/upload_store/extensions.py to rhodecode/apps/file_store/extensions.py
@@ -1,171 +1,175 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2016-2019 RhodeCode GmbH
3 # Copyright (C) 2016-2019 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import os
21 import os
22 import time
22 import time
23 import shutil
23 import shutil
24
24
25 from rhodecode.lib.ext_json import json
25 from rhodecode.lib.ext_json import json
26 from rhodecode.apps.upload_store import utils
26 from rhodecode.apps.file_store import utils
27 from rhodecode.apps.upload_store.extensions import resolve_extensions
27 from rhodecode.apps.file_store.extensions import resolve_extensions
28 from rhodecode.apps.upload_store.exceptions import FileNotAllowedException
28 from rhodecode.apps.file_store.exceptions import FileNotAllowedException
29
29
30 METADATA_VER = 'v1'
30 METADATA_VER = 'v1'
31
31
32
32
33 class LocalFileStorage(object):
33 class LocalFileStorage(object):
34
34
35 @classmethod
35 @classmethod
36 def resolve_name(cls, name, directory):
36 def resolve_name(cls, name, directory):
37 """
37 """
38 Resolves a unique name and the correct path. If a filename
38 Resolves a unique name and the correct path. If a filename
39 for that path already exists then a numeric prefix with values > 0 will be
39 for that path already exists then a numeric prefix with values > 0 will be
40 added, for example test.jpg -> test-1.jpg etc. initially file would have 0 prefix.
40 added, for example test.jpg -> test-1.jpg etc. initially file would have 0 prefix.
41
41
42 :param name: base name of file
42 :param name: base name of file
43 :param directory: absolute directory path
43 :param directory: absolute directory path
44 """
44 """
45
45
46 basename, ext = os.path.splitext(name)
46 basename, ext = os.path.splitext(name)
47 counter = 0
47 counter = 0
48 while True:
48 while True:
49 name = '%s-%d%s' % (basename, counter, ext)
49 name = '%s-%d%s' % (basename, counter, ext)
50 path = os.path.join(directory, name)
50 path = os.path.join(directory, name)
51 if not os.path.exists(path):
51 if not os.path.exists(path):
52 return name, path
52 return name, path
53 counter += 1
53 counter += 1
54
54
55 def __init__(self, base_path, extension_groups=None):
55 def __init__(self, base_path, extension_groups=None):
56
56
57 """
57 """
58 Local file storage
58 Local file storage
59
59
60 :param base_path: the absolute base path where uploads are stored
60 :param base_path: the absolute base path where uploads are stored
61 :param extension_groups: extensions string
61 :param extension_groups: extensions string
62 """
62 """
63
63
64 extension_groups = extension_groups or ['any']
64 extension_groups = extension_groups or ['any']
65 self.base_path = base_path
65 self.base_path = base_path
66 self.extensions = resolve_extensions([], groups=extension_groups)
66 self.extensions = resolve_extensions([], groups=extension_groups)
67
67
68 def store_path(self, filename):
68 def store_path(self, filename):
69 """
69 """
70 Returns absolute file path of the filename, joined to the
70 Returns absolute file path of the filename, joined to the
71 base_path.
71 base_path.
72
72
73 :param filename: base name of file
73 :param filename: base name of file
74 """
74 """
75 return os.path.join(self.base_path, filename)
75 return os.path.join(self.base_path, filename)
76
76
77 def delete(self, filename):
77 def delete(self, filename):
78 """
78 """
79 Deletes the filename. Filename is resolved with the
79 Deletes the filename. Filename is resolved with the
80 absolute path based on base_path. If file does not exist,
80 absolute path based on base_path. If file does not exist,
81 returns **False**, otherwise **True**
81 returns **False**, otherwise **True**
82
82
83 :param filename: base name of file
83 :param filename: base name of file
84 """
84 """
85 if self.exists(filename):
85 if self.exists(filename):
86 os.remove(self.store_path(filename))
86 os.remove(self.store_path(filename))
87 return True
87 return True
88 return False
88 return False
89
89
90 def exists(self, filename):
90 def exists(self, filename):
91 """
91 """
92 Checks if file exists. Resolves filename's absolute
92 Checks if file exists. Resolves filename's absolute
93 path based on base_path.
93 path based on base_path.
94
94
95 :param filename: base name of file
95 :param filename: base name of file
96 """
96 """
97 return os.path.exists(self.store_path(filename))
97 return os.path.exists(self.store_path(filename))
98
98
99 def filename_allowed(self, filename, extensions=None):
99 def filename_allowed(self, filename, extensions=None):
100 """Checks if a filename has an allowed extension
100 """Checks if a filename has an allowed extension
101
101
102 :param filename: base name of file
102 :param filename: base name of file
103 :param extensions: iterable of extensions (or self.extensions)
103 :param extensions: iterable of extensions (or self.extensions)
104 """
104 """
105 _, ext = os.path.splitext(filename)
105 _, ext = os.path.splitext(filename)
106 return self.extension_allowed(ext, extensions)
106 return self.extension_allowed(ext, extensions)
107
107
108 def extension_allowed(self, ext, extensions=None):
108 def extension_allowed(self, ext, extensions=None):
109 """
109 """
110 Checks if an extension is permitted. Both e.g. ".jpg" and
110 Checks if an extension is permitted. Both e.g. ".jpg" and
111 "jpg" can be passed in. Extension lookup is case-insensitive.
111 "jpg" can be passed in. Extension lookup is case-insensitive.
112
112
113 :param extensions: iterable of extensions (or self.extensions)
113 :param ext: extension to check
114 :param extensions: iterable of extensions to validate against (or self.extensions)
114 """
115 """
115
116
116 extensions = extensions or self.extensions
117 extensions = extensions or self.extensions
117 if not extensions:
118 if not extensions:
118 return True
119 return True
119 if ext.startswith('.'):
120 if ext.startswith('.'):
120 ext = ext[1:]
121 ext = ext[1:]
121 return ext.lower() in extensions
122 return ext.lower() in extensions
122
123
123 def save_file(self, file_obj, filename, directory=None, extensions=None,
124 def save_file(self, file_obj, filename, directory=None, extensions=None,
124 metadata=None, **kwargs):
125 metadata=None, **kwargs):
125 """
126 """
126 Saves a file object to the uploads location.
127 Saves a file object to the uploads location.
127 Returns the resolved filename, i.e. the directory +
128 Returns the resolved filename, i.e. the directory +
128 the (randomized/incremented) base name.
129 the (randomized/incremented) base name.
129
130
130 :param file_obj: **cgi.FieldStorage** object (or similar)
131 :param file_obj: **cgi.FieldStorage** object (or similar)
131 :param filename: original filename
132 :param filename: original filename
132 :param directory: relative path of sub-directory
133 :param directory: relative path of sub-directory
133 :param extensions: iterable of allowed extensions, if not default
134 :param extensions: iterable of allowed extensions, if not default
134 :param metadata: JSON metadata to store next to the file with .meta suffix
135 :param metadata: JSON metadata to store next to the file with .meta suffix
135 :returns: modified filename
136 :returns: modified filename
136 """
137 """
137
138
138 extensions = extensions or self.extensions
139 extensions = extensions or self.extensions
139
140
140 if not self.filename_allowed(filename, extensions):
141 if not self.filename_allowed(filename, extensions):
141 raise FileNotAllowedException()
142 raise FileNotAllowedException()
142
143
143 if directory:
144 if directory:
144 dest_directory = os.path.join(self.base_path, directory)
145 dest_directory = os.path.join(self.base_path, directory)
145 else:
146 else:
146 dest_directory = self.base_path
147 dest_directory = self.base_path
147
148
148 if not os.path.exists(dest_directory):
149 if not os.path.exists(dest_directory):
149 os.makedirs(dest_directory)
150 os.makedirs(dest_directory)
150
151
151 filename = utils.uid_filename(filename)
152 filename = utils.uid_filename(filename)
152
153
153 filename, path = self.resolve_name(filename, dest_directory)
154 filename, path = self.resolve_name(filename, dest_directory)
154 filename_meta = filename + '.meta'
155 filename_meta = filename + '.meta'
155
156
156 file_obj.seek(0)
157 file_obj.seek(0)
157
158
158 with open(path, "wb") as dest:
159 with open(path, "wb") as dest:
159 shutil.copyfileobj(file_obj, dest)
160 shutil.copyfileobj(file_obj, dest)
160
161
161 if metadata:
162 if metadata:
162 size = os.stat(path).st_size
163 size = os.stat(path).st_size
163 metadata.update({'size': size, "time": time.time(),
164 metadata.update(
165 {"size": size,
166 "time": time.time(),
164 "meta_ver": METADATA_VER})
167 "meta_ver": METADATA_VER})
168
165 with open(os.path.join(dest_directory, filename_meta), "wb") as dest_meta:
169 with open(os.path.join(dest_directory, filename_meta), "wb") as dest_meta:
166 dest_meta.write(json.dumps(metadata))
170 dest_meta.write(json.dumps(metadata))
167
171
168 if directory:
172 if directory:
169 filename = os.path.join(directory, filename)
173 filename = os.path.join(directory, filename)
170
174
171 return filename
175 return filename, metadata
1 NO CONTENT: file renamed from rhodecode/apps/upload_store/tests/__init__.py to rhodecode/apps/file_store/tests/__init__.py
NO CONTENT: file renamed from rhodecode/apps/upload_store/tests/__init__.py to rhodecode/apps/file_store/tests/__init__.py
@@ -1,110 +1,110 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2010-2019 RhodeCode GmbH
3 # Copyright (C) 2010-2019 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 import os
20 import os
21 import pytest
21 import pytest
22
22
23 from rhodecode.lib.ext_json import json
23 from rhodecode.lib.ext_json import json
24 from rhodecode.tests import TestController
24 from rhodecode.tests import TestController
25 from rhodecode.apps.upload_store import utils, config_keys
25 from rhodecode.apps.file_store import utils, config_keys
26
26
27
27
28 def route_path(name, params=None, **kwargs):
28 def route_path(name, params=None, **kwargs):
29 import urllib
29 import urllib
30
30
31 base_url = {
31 base_url = {
32 'upload_file': '/_file_store/upload',
32 'upload_file': '/_file_store/upload',
33 'download_file': '/_file_store/download/{fid}',
33 'download_file': '/_file_store/download/{fid}',
34
34
35 }[name].format(**kwargs)
35 }[name].format(**kwargs)
36
36
37 if params:
37 if params:
38 base_url = '{}?{}'.format(base_url, urllib.urlencode(params))
38 base_url = '{}?{}'.format(base_url, urllib.urlencode(params))
39 return base_url
39 return base_url
40
40
41
41
42 class TestFileStoreViews(TestController):
42 class TestFileStoreViews(TestController):
43
43
44 @pytest.mark.parametrize("fid, content, exists", [
44 @pytest.mark.parametrize("fid, content, exists", [
45 ('abcde-0.jpg', "xxxxx", True),
45 ('abcde-0.jpg', "xxxxx", True),
46 ('abcde-0.exe', "1234567", True),
46 ('abcde-0.exe', "1234567", True),
47 ('abcde-0.jpg', "xxxxx", False),
47 ('abcde-0.jpg', "xxxxx", False),
48 ])
48 ])
49 def test_get_files_from_store(self, fid, content, exists, tmpdir):
49 def test_get_files_from_store(self, fid, content, exists, tmpdir):
50 self.log_user()
50 self.log_user()
51 store_path = self.app._pyramid_settings[config_keys.store_path]
51 store_path = self.app._pyramid_settings[config_keys.store_path]
52
52
53 if exists:
53 if exists:
54 status = 200
54 status = 200
55 store = utils.get_file_storage({config_keys.store_path: store_path})
55 store = utils.get_file_storage({config_keys.store_path: store_path})
56 filesystem_file = os.path.join(str(tmpdir), fid)
56 filesystem_file = os.path.join(str(tmpdir), fid)
57 with open(filesystem_file, 'wb') as f:
57 with open(filesystem_file, 'wb') as f:
58 f.write(content)
58 f.write(content)
59
59
60 with open(filesystem_file, 'rb') as f:
60 with open(filesystem_file, 'rb') as f:
61 fid = store.save_file(f, fid, metadata={'filename': fid})
61 fid, metadata = store.save_file(f, fid, metadata={'filename': fid})
62
62
63 else:
63 else:
64 status = 404
64 status = 404
65
65
66 response = self.app.get(route_path('download_file', fid=fid), status=status)
66 response = self.app.get(route_path('download_file', fid=fid), status=status)
67
67
68 if exists:
68 if exists:
69 assert response.text == content
69 assert response.text == content
70 metadata = os.path.join(store_path, fid + '.meta')
70 metadata = os.path.join(store_path, fid + '.meta')
71 assert os.path.exists(metadata)
71 assert os.path.exists(metadata)
72 with open(metadata, 'rb') as f:
72 with open(metadata, 'rb') as f:
73 json_data = json.loads(f.read())
73 json_data = json.loads(f.read())
74
74
75 assert json_data
75 assert json_data
76 assert 'size' in json_data
76 assert 'size' in json_data
77
77
78 def test_upload_files_without_content_to_store(self):
78 def test_upload_files_without_content_to_store(self):
79 self.log_user()
79 self.log_user()
80 response = self.app.post(
80 response = self.app.post(
81 route_path('upload_file'),
81 route_path('upload_file'),
82 params={'csrf_token': self.csrf_token},
82 params={'csrf_token': self.csrf_token},
83 status=200)
83 status=200)
84
84
85 assert response.json == {
85 assert response.json == {
86 u'error': u'store_file data field is missing',
86 u'error': u'store_file data field is missing',
87 u'access_path': None,
87 u'access_path': None,
88 u'store_fid': None}
88 u'store_fid': None}
89
89
90 def test_upload_files_bogus_content_to_store(self):
90 def test_upload_files_bogus_content_to_store(self):
91 self.log_user()
91 self.log_user()
92 response = self.app.post(
92 response = self.app.post(
93 route_path('upload_file'),
93 route_path('upload_file'),
94 params={'csrf_token': self.csrf_token, 'store_file': 'bogus'},
94 params={'csrf_token': self.csrf_token, 'store_file': 'bogus'},
95 status=200)
95 status=200)
96
96
97 assert response.json == {
97 assert response.json == {
98 u'error': u'filename cannot be read from the data field',
98 u'error': u'filename cannot be read from the data field',
99 u'access_path': None,
99 u'access_path': None,
100 u'store_fid': None}
100 u'store_fid': None}
101
101
102 def test_upload_content_to_store(self):
102 def test_upload_content_to_store(self):
103 self.log_user()
103 self.log_user()
104 response = self.app.post(
104 response = self.app.post(
105 route_path('upload_file'),
105 route_path('upload_file'),
106 upload_files=[('store_file', 'myfile.txt', 'SOME CONTENT')],
106 upload_files=[('store_file', 'myfile.txt', 'SOME CONTENT')],
107 params={'csrf_token': self.csrf_token},
107 params={'csrf_token': self.csrf_token},
108 status=200)
108 status=200)
109
109
110 assert response.json['store_fid']
110 assert response.json['store_fid']
@@ -1,47 +1,47 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2016-2019 RhodeCode GmbH
3 # Copyright (C) 2016-2019 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21
21
22 import os
22 import os
23 import uuid
23 import uuid
24
24
25
25
26 def get_file_storage(settings):
26 def get_file_storage(settings):
27 from rhodecode.apps.upload_store.local_store import LocalFileStorage
27 from rhodecode.apps.file_store.local_store import LocalFileStorage
28 from rhodecode.apps.upload_store import config_keys
28 from rhodecode.apps.file_store import config_keys
29 store_path = settings.get(config_keys.store_path)
29 store_path = settings.get(config_keys.store_path)
30 return LocalFileStorage(base_path=store_path)
30 return LocalFileStorage(base_path=store_path)
31
31
32
32
33 def uid_filename(filename, randomized=True):
33 def uid_filename(filename, randomized=True):
34 """
34 """
35 Generates a randomized or stable (uuid) filename,
35 Generates a randomized or stable (uuid) filename,
36 preserving the original extension.
36 preserving the original extension.
37
37
38 :param filename: the original filename
38 :param filename: the original filename
39 :param randomized: define if filename should be stable (sha1 based) or randomized
39 :param randomized: define if filename should be stable (sha1 based) or randomized
40 """
40 """
41 _, ext = os.path.splitext(filename)
41 _, ext = os.path.splitext(filename)
42 if randomized:
42 if randomized:
43 uid = uuid.uuid4()
43 uid = uuid.uuid4()
44 else:
44 else:
45 hash_key = '{}.{}'.format(filename, 'store')
45 hash_key = '{}.{}'.format(filename, 'store')
46 uid = uuid.uuid5(uuid.NAMESPACE_URL, hash_key)
46 uid = uuid.uuid5(uuid.NAMESPACE_URL, hash_key)
47 return str(uid) + ext.lower()
47 return str(uid) + ext.lower()
@@ -1,97 +1,96 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2016-2019 RhodeCode GmbH
3 # Copyright (C) 2016-2019 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 import logging
20 import logging
21
21
22 from pyramid.view import view_config
22 from pyramid.view import view_config
23 from pyramid.response import FileResponse
23 from pyramid.response import FileResponse
24 from pyramid.httpexceptions import HTTPFound, HTTPNotFound
24 from pyramid.httpexceptions import HTTPFound, HTTPNotFound
25
25
26 from rhodecode.apps._base import BaseAppView
26 from rhodecode.apps._base import BaseAppView
27 from rhodecode.apps.upload_store import utils
27 from rhodecode.apps.file_store import utils
28 from rhodecode.apps.upload_store.exceptions import (
28 from rhodecode.apps.file_store.exceptions import (
29 FileNotAllowedException,FileOverSizeException)
29 FileNotAllowedException,FileOverSizeException)
30
30
31 from rhodecode.lib import helpers as h
31 from rhodecode.lib import helpers as h
32 from rhodecode.lib import audit_logger
32 from rhodecode.lib import audit_logger
33 from rhodecode.lib.auth import (CSRFRequired, NotAnonymous)
33 from rhodecode.lib.auth import (CSRFRequired, NotAnonymous)
34
34
35 log = logging.getLogger(__name__)
35 log = logging.getLogger(__name__)
36
36
37
37
38 class FileStoreView(BaseAppView):
38 class FileStoreView(BaseAppView):
39 upload_key = 'store_file'
39 upload_key = 'store_file'
40
40
41 def load_default_context(self):
41 def load_default_context(self):
42 c = self._get_local_tmpl_context()
42 c = self._get_local_tmpl_context()
43 self.storage = utils.get_file_storage(self.request.registry.settings)
43 self.storage = utils.get_file_storage(self.request.registry.settings)
44 return c
44 return c
45
45
46 @NotAnonymous()
46 @NotAnonymous()
47 @CSRFRequired()
47 @CSRFRequired()
48 @view_config(route_name='upload_file', request_method='POST', renderer='json_ext')
48 @view_config(route_name='upload_file', request_method='POST', renderer='json_ext')
49 def upload_file(self):
49 def upload_file(self):
50 self.load_default_context()
50 self.load_default_context()
51 file_obj = self.request.POST.get(self.upload_key)
51 file_obj = self.request.POST.get(self.upload_key)
52
52
53 if file_obj is None:
53 if file_obj is None:
54 return {'store_fid': None,
54 return {'store_fid': None,
55 'access_path': None,
55 'access_path': None,
56 'error': '{} data field is missing'.format(self.upload_key)}
56 'error': '{} data field is missing'.format(self.upload_key)}
57
57
58 if not hasattr(file_obj, 'filename'):
58 if not hasattr(file_obj, 'filename'):
59 return {'store_fid': None,
59 return {'store_fid': None,
60 'access_path': None,
60 'access_path': None,
61 'error': 'filename cannot be read from the data field'}
61 'error': 'filename cannot be read from the data field'}
62
62
63 filename = file_obj.filename
63 filename = file_obj.filename
64
64
65 metadata = {
65 metadata = {
66 'filename': filename,
66 'filename': filename,
67 'size': '', # filled by save_file
67 'size': '', # filled by save_file
68 'user_uploaded': {'username': self._rhodecode_user.username,
68 'user_uploaded': {'username': self._rhodecode_user.username,
69 'user_id': self._rhodecode_user.user_id,
69 'user_id': self._rhodecode_user.user_id,
70 'ip': self._rhodecode_user.ip_addr}}
70 'ip': self._rhodecode_user.ip_addr}}
71 try:
71 try:
72 store_fid = self.storage.save_file(file_obj.file, filename,
72 store_fid, metadata = self.storage.save_file(file_obj.file, filename, metadata=metadata)
73 metadata=metadata)
74 except FileNotAllowedException:
73 except FileNotAllowedException:
75 return {'store_fid': None,
74 return {'store_fid': None,
76 'access_path': None,
75 'access_path': None,
77 'error': 'File {} is not allowed.'.format(filename)}
76 'error': 'File {} is not allowed.'.format(filename)}
78
77
79 except FileOverSizeException:
78 except FileOverSizeException:
80 return {'store_fid': None,
79 return {'store_fid': None,
81 'access_path': None,
80 'access_path': None,
82 'error': 'File {} is exceeding allowed limit.'.format(filename)}
81 'error': 'File {} is exceeding allowed limit.'.format(filename)}
83
82
84 return {'store_fid': store_fid,
83 return {'store_fid': store_fid,
85 'access_path': h.route_path('download_file', fid=store_fid)}
84 'access_path': h.route_path('download_file', fid=store_fid)}
86
85
87 @view_config(route_name='download_file')
86 @view_config(route_name='download_file')
88 def download_file(self):
87 def download_file(self):
89 self.load_default_context()
88 self.load_default_context()
90 file_uid = self.request.matchdict['fid']
89 file_uid = self.request.matchdict['fid']
91 log.debug('Requesting FID:%s from store %s', file_uid, self.storage)
90 log.debug('Requesting FID:%s from store %s', file_uid, self.storage)
92 if not self.storage.exists(file_uid):
91 if not self.storage.exists(file_uid):
93 log.debug('File with FID:%s not found in the store', file_uid)
92 log.debug('File with FID:%s not found in the store', file_uid)
94 raise HTTPNotFound()
93 raise HTTPNotFound()
95
94
96 file_path = self.storage.store_path(file_uid)
95 file_path = self.storage.store_path(file_uid)
97 return FileResponse(file_path)
96 return FileResponse(file_path)
@@ -1,740 +1,740 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2010-2019 RhodeCode GmbH
3 # Copyright (C) 2010-2019 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import os
21 import os
22 import sys
22 import sys
23 import logging
23 import logging
24 import collections
24 import collections
25 import tempfile
25 import tempfile
26 import time
26 import time
27
27
28 from paste.gzipper import make_gzip_middleware
28 from paste.gzipper import make_gzip_middleware
29 import pyramid.events
29 import pyramid.events
30 from pyramid.wsgi import wsgiapp
30 from pyramid.wsgi import wsgiapp
31 from pyramid.authorization import ACLAuthorizationPolicy
31 from pyramid.authorization import ACLAuthorizationPolicy
32 from pyramid.config import Configurator
32 from pyramid.config import Configurator
33 from pyramid.settings import asbool, aslist
33 from pyramid.settings import asbool, aslist
34 from pyramid.httpexceptions import (
34 from pyramid.httpexceptions import (
35 HTTPException, HTTPError, HTTPInternalServerError, HTTPFound, HTTPNotFound)
35 HTTPException, HTTPError, HTTPInternalServerError, HTTPFound, HTTPNotFound)
36 from pyramid.renderers import render_to_response
36 from pyramid.renderers import render_to_response
37
37
38 from rhodecode.model import meta
38 from rhodecode.model import meta
39 from rhodecode.config import patches
39 from rhodecode.config import patches
40 from rhodecode.config import utils as config_utils
40 from rhodecode.config import utils as config_utils
41 from rhodecode.config.environment import load_pyramid_environment
41 from rhodecode.config.environment import load_pyramid_environment
42
42
43 import rhodecode.events
43 import rhodecode.events
44 from rhodecode.lib.middleware.vcs import VCSMiddleware
44 from rhodecode.lib.middleware.vcs import VCSMiddleware
45 from rhodecode.lib.request import Request
45 from rhodecode.lib.request import Request
46 from rhodecode.lib.vcs import VCSCommunicationError
46 from rhodecode.lib.vcs import VCSCommunicationError
47 from rhodecode.lib.exceptions import VCSServerUnavailable
47 from rhodecode.lib.exceptions import VCSServerUnavailable
48 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
48 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
49 from rhodecode.lib.middleware.https_fixup import HttpsFixup
49 from rhodecode.lib.middleware.https_fixup import HttpsFixup
50 from rhodecode.lib.celerylib.loader import configure_celery
50 from rhodecode.lib.celerylib.loader import configure_celery
51 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
51 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
52 from rhodecode.lib.utils2 import aslist as rhodecode_aslist, AttributeDict
52 from rhodecode.lib.utils2 import aslist as rhodecode_aslist, AttributeDict
53 from rhodecode.lib.exc_tracking import store_exception
53 from rhodecode.lib.exc_tracking import store_exception
54 from rhodecode.subscribers import (
54 from rhodecode.subscribers import (
55 scan_repositories_if_enabled, write_js_routes_if_enabled,
55 scan_repositories_if_enabled, write_js_routes_if_enabled,
56 write_metadata_if_needed, inject_app_settings)
56 write_metadata_if_needed, inject_app_settings)
57
57
58
58
59 log = logging.getLogger(__name__)
59 log = logging.getLogger(__name__)
60
60
61
61
62 def is_http_error(response):
62 def is_http_error(response):
63 # error which should have traceback
63 # error which should have traceback
64 return response.status_code > 499
64 return response.status_code > 499
65
65
66
66
67 def should_load_all():
67 def should_load_all():
68 """
68 """
69 Returns if all application components should be loaded. In some cases it's
69 Returns if all application components should be loaded. In some cases it's
70 desired to skip apps loading for faster shell script execution
70 desired to skip apps loading for faster shell script execution
71 """
71 """
72 return True
72 return True
73
73
74
74
75 def make_pyramid_app(global_config, **settings):
75 def make_pyramid_app(global_config, **settings):
76 """
76 """
77 Constructs the WSGI application based on Pyramid.
77 Constructs the WSGI application based on Pyramid.
78
78
79 Specials:
79 Specials:
80
80
81 * The application can also be integrated like a plugin via the call to
81 * The application can also be integrated like a plugin via the call to
82 `includeme`. This is accompanied with the other utility functions which
82 `includeme`. This is accompanied with the other utility functions which
83 are called. Changing this should be done with great care to not break
83 are called. Changing this should be done with great care to not break
84 cases when these fragments are assembled from another place.
84 cases when these fragments are assembled from another place.
85
85
86 """
86 """
87
87
88 # Allows to use format style "{ENV_NAME}" placeholders in the configuration. It
88 # Allows to use format style "{ENV_NAME}" placeholders in the configuration. It
89 # will be replaced by the value of the environment variable "NAME" in this case.
89 # will be replaced by the value of the environment variable "NAME" in this case.
90 start_time = time.time()
90 start_time = time.time()
91
91
92 debug = asbool(global_config.get('debug'))
92 debug = asbool(global_config.get('debug'))
93 if debug:
93 if debug:
94 enable_debug()
94 enable_debug()
95
95
96 environ = {'ENV_{}'.format(key): value for key, value in os.environ.items()}
96 environ = {'ENV_{}'.format(key): value for key, value in os.environ.items()}
97
97
98 global_config = _substitute_values(global_config, environ)
98 global_config = _substitute_values(global_config, environ)
99 settings = _substitute_values(settings, environ)
99 settings = _substitute_values(settings, environ)
100
100
101 sanitize_settings_and_apply_defaults(global_config, settings)
101 sanitize_settings_and_apply_defaults(global_config, settings)
102
102
103 config = Configurator(settings=settings)
103 config = Configurator(settings=settings)
104
104
105 # Apply compatibility patches
105 # Apply compatibility patches
106 patches.inspect_getargspec()
106 patches.inspect_getargspec()
107
107
108 load_pyramid_environment(global_config, settings)
108 load_pyramid_environment(global_config, settings)
109
109
110 # Static file view comes first
110 # Static file view comes first
111 includeme_first(config)
111 includeme_first(config)
112
112
113 includeme(config)
113 includeme(config)
114
114
115 pyramid_app = config.make_wsgi_app()
115 pyramid_app = config.make_wsgi_app()
116 pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
116 pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
117 pyramid_app.config = config
117 pyramid_app.config = config
118
118
119 config.configure_celery(global_config['__file__'])
119 config.configure_celery(global_config['__file__'])
120 # creating the app uses a connection - return it after we are done
120 # creating the app uses a connection - return it after we are done
121 meta.Session.remove()
121 meta.Session.remove()
122 total_time = time.time() - start_time
122 total_time = time.time() - start_time
123 log.info('Pyramid app `%s` created and configured in %.2fs',
123 log.info('Pyramid app `%s` created and configured in %.2fs',
124 pyramid_app.func_name, total_time)
124 pyramid_app.func_name, total_time)
125
125
126 return pyramid_app
126 return pyramid_app
127
127
128
128
129 def not_found_view(request):
129 def not_found_view(request):
130 """
130 """
131 This creates the view which should be registered as not-found-view to
131 This creates the view which should be registered as not-found-view to
132 pyramid.
132 pyramid.
133 """
133 """
134
134
135 if not getattr(request, 'vcs_call', None):
135 if not getattr(request, 'vcs_call', None):
136 # handle like regular case with our error_handler
136 # handle like regular case with our error_handler
137 return error_handler(HTTPNotFound(), request)
137 return error_handler(HTTPNotFound(), request)
138
138
139 # handle not found view as a vcs call
139 # handle not found view as a vcs call
140 settings = request.registry.settings
140 settings = request.registry.settings
141 ae_client = getattr(request, 'ae_client', None)
141 ae_client = getattr(request, 'ae_client', None)
142 vcs_app = VCSMiddleware(
142 vcs_app = VCSMiddleware(
143 HTTPNotFound(), request.registry, settings,
143 HTTPNotFound(), request.registry, settings,
144 appenlight_client=ae_client)
144 appenlight_client=ae_client)
145
145
146 return wsgiapp(vcs_app)(None, request)
146 return wsgiapp(vcs_app)(None, request)
147
147
148
148
149 def error_handler(exception, request):
149 def error_handler(exception, request):
150 import rhodecode
150 import rhodecode
151 from rhodecode.lib import helpers
151 from rhodecode.lib import helpers
152
152
153 rhodecode_title = rhodecode.CONFIG.get('rhodecode_title') or 'RhodeCode'
153 rhodecode_title = rhodecode.CONFIG.get('rhodecode_title') or 'RhodeCode'
154
154
155 base_response = HTTPInternalServerError()
155 base_response = HTTPInternalServerError()
156 # prefer original exception for the response since it may have headers set
156 # prefer original exception for the response since it may have headers set
157 if isinstance(exception, HTTPException):
157 if isinstance(exception, HTTPException):
158 base_response = exception
158 base_response = exception
159 elif isinstance(exception, VCSCommunicationError):
159 elif isinstance(exception, VCSCommunicationError):
160 base_response = VCSServerUnavailable()
160 base_response = VCSServerUnavailable()
161
161
162 if is_http_error(base_response):
162 if is_http_error(base_response):
163 log.exception(
163 log.exception(
164 'error occurred handling this request for path: %s', request.path)
164 'error occurred handling this request for path: %s', request.path)
165
165
166 error_explanation = base_response.explanation or str(base_response)
166 error_explanation = base_response.explanation or str(base_response)
167 if base_response.status_code == 404:
167 if base_response.status_code == 404:
168 error_explanation += " Optionally you don't have permission to access this page."
168 error_explanation += " Optionally you don't have permission to access this page."
169 c = AttributeDict()
169 c = AttributeDict()
170 c.error_message = base_response.status
170 c.error_message = base_response.status
171 c.error_explanation = error_explanation
171 c.error_explanation = error_explanation
172 c.visual = AttributeDict()
172 c.visual = AttributeDict()
173
173
174 c.visual.rhodecode_support_url = (
174 c.visual.rhodecode_support_url = (
175 request.registry.settings.get('rhodecode_support_url') or
175 request.registry.settings.get('rhodecode_support_url') or
176 request.route_url('rhodecode_support')
176 request.route_url('rhodecode_support')
177 )
177 )
178 c.redirect_time = 0
178 c.redirect_time = 0
179 c.rhodecode_name = rhodecode_title
179 c.rhodecode_name = rhodecode_title
180 if not c.rhodecode_name:
180 if not c.rhodecode_name:
181 c.rhodecode_name = 'Rhodecode'
181 c.rhodecode_name = 'Rhodecode'
182
182
183 c.causes = []
183 c.causes = []
184 if is_http_error(base_response):
184 if is_http_error(base_response):
185 c.causes.append('Server is overloaded.')
185 c.causes.append('Server is overloaded.')
186 c.causes.append('Server database connection is lost.')
186 c.causes.append('Server database connection is lost.')
187 c.causes.append('Server expected unhandled error.')
187 c.causes.append('Server expected unhandled error.')
188
188
189 if hasattr(base_response, 'causes'):
189 if hasattr(base_response, 'causes'):
190 c.causes = base_response.causes
190 c.causes = base_response.causes
191
191
192 c.messages = helpers.flash.pop_messages(request=request)
192 c.messages = helpers.flash.pop_messages(request=request)
193
193
194 exc_info = sys.exc_info()
194 exc_info = sys.exc_info()
195 c.exception_id = id(exc_info)
195 c.exception_id = id(exc_info)
196 c.show_exception_id = isinstance(base_response, VCSServerUnavailable) \
196 c.show_exception_id = isinstance(base_response, VCSServerUnavailable) \
197 or base_response.status_code > 499
197 or base_response.status_code > 499
198 c.exception_id_url = request.route_url(
198 c.exception_id_url = request.route_url(
199 'admin_settings_exception_tracker_show', exception_id=c.exception_id)
199 'admin_settings_exception_tracker_show', exception_id=c.exception_id)
200
200
201 if c.show_exception_id:
201 if c.show_exception_id:
202 store_exception(c.exception_id, exc_info)
202 store_exception(c.exception_id, exc_info)
203
203
204 response = render_to_response(
204 response = render_to_response(
205 '/errors/error_document.mako', {'c': c, 'h': helpers}, request=request,
205 '/errors/error_document.mako', {'c': c, 'h': helpers}, request=request,
206 response=base_response)
206 response=base_response)
207
207
208 return response
208 return response
209
209
210
210
211 def includeme_first(config):
211 def includeme_first(config):
212 # redirect automatic browser favicon.ico requests to correct place
212 # redirect automatic browser favicon.ico requests to correct place
213 def favicon_redirect(context, request):
213 def favicon_redirect(context, request):
214 return HTTPFound(
214 return HTTPFound(
215 request.static_path('rhodecode:public/images/favicon.ico'))
215 request.static_path('rhodecode:public/images/favicon.ico'))
216
216
217 config.add_view(favicon_redirect, route_name='favicon')
217 config.add_view(favicon_redirect, route_name='favicon')
218 config.add_route('favicon', '/favicon.ico')
218 config.add_route('favicon', '/favicon.ico')
219
219
220 def robots_redirect(context, request):
220 def robots_redirect(context, request):
221 return HTTPFound(
221 return HTTPFound(
222 request.static_path('rhodecode:public/robots.txt'))
222 request.static_path('rhodecode:public/robots.txt'))
223
223
224 config.add_view(robots_redirect, route_name='robots')
224 config.add_view(robots_redirect, route_name='robots')
225 config.add_route('robots', '/robots.txt')
225 config.add_route('robots', '/robots.txt')
226
226
227 config.add_static_view(
227 config.add_static_view(
228 '_static/deform', 'deform:static')
228 '_static/deform', 'deform:static')
229 config.add_static_view(
229 config.add_static_view(
230 '_static/rhodecode', path='rhodecode:public', cache_max_age=3600 * 24)
230 '_static/rhodecode', path='rhodecode:public', cache_max_age=3600 * 24)
231
231
232
232
233 def includeme(config):
233 def includeme(config):
234 log.debug('Initializing main includeme from %s', os.path.basename(__file__))
234 log.debug('Initializing main includeme from %s', os.path.basename(__file__))
235 settings = config.registry.settings
235 settings = config.registry.settings
236 config.set_request_factory(Request)
236 config.set_request_factory(Request)
237
237
238 # plugin information
238 # plugin information
239 config.registry.rhodecode_plugins = collections.OrderedDict()
239 config.registry.rhodecode_plugins = collections.OrderedDict()
240
240
241 config.add_directive(
241 config.add_directive(
242 'register_rhodecode_plugin', register_rhodecode_plugin)
242 'register_rhodecode_plugin', register_rhodecode_plugin)
243
243
244 config.add_directive('configure_celery', configure_celery)
244 config.add_directive('configure_celery', configure_celery)
245
245
246 if asbool(settings.get('appenlight', 'false')):
246 if asbool(settings.get('appenlight', 'false')):
247 config.include('appenlight_client.ext.pyramid_tween')
247 config.include('appenlight_client.ext.pyramid_tween')
248
248
249 load_all = should_load_all()
249 load_all = should_load_all()
250
250
251 # Includes which are required. The application would fail without them.
251 # Includes which are required. The application would fail without them.
252 config.include('pyramid_mako')
252 config.include('pyramid_mako')
253 config.include('pyramid_beaker')
253 config.include('pyramid_beaker')
254 config.include('rhodecode.lib.rc_cache')
254 config.include('rhodecode.lib.rc_cache')
255
255
256 config.include('rhodecode.apps._base.navigation')
256 config.include('rhodecode.apps._base.navigation')
257 config.include('rhodecode.apps._base.subscribers')
257 config.include('rhodecode.apps._base.subscribers')
258 config.include('rhodecode.tweens')
258 config.include('rhodecode.tweens')
259
259
260 config.include('rhodecode.integrations')
260 config.include('rhodecode.integrations')
261 config.include('rhodecode.authentication')
261 config.include('rhodecode.authentication')
262
262
263 if load_all:
263 if load_all:
264 from rhodecode.authentication import discover_legacy_plugins
264 from rhodecode.authentication import discover_legacy_plugins
265 # load CE authentication plugins
265 # load CE authentication plugins
266 config.include('rhodecode.authentication.plugins.auth_crowd')
266 config.include('rhodecode.authentication.plugins.auth_crowd')
267 config.include('rhodecode.authentication.plugins.auth_headers')
267 config.include('rhodecode.authentication.plugins.auth_headers')
268 config.include('rhodecode.authentication.plugins.auth_jasig_cas')
268 config.include('rhodecode.authentication.plugins.auth_jasig_cas')
269 config.include('rhodecode.authentication.plugins.auth_ldap')
269 config.include('rhodecode.authentication.plugins.auth_ldap')
270 config.include('rhodecode.authentication.plugins.auth_pam')
270 config.include('rhodecode.authentication.plugins.auth_pam')
271 config.include('rhodecode.authentication.plugins.auth_rhodecode')
271 config.include('rhodecode.authentication.plugins.auth_rhodecode')
272 config.include('rhodecode.authentication.plugins.auth_token')
272 config.include('rhodecode.authentication.plugins.auth_token')
273
273
274 # Auto discover authentication plugins and include their configuration.
274 # Auto discover authentication plugins and include their configuration.
275 discover_legacy_plugins(config)
275 discover_legacy_plugins(config)
276
276
277 # apps
277 # apps
278 config.include('rhodecode.apps._base')
278 config.include('rhodecode.apps._base')
279
279
280 if load_all:
280 if load_all:
281 config.include('rhodecode.apps.ops')
281 config.include('rhodecode.apps.ops')
282 config.include('rhodecode.apps.admin')
282 config.include('rhodecode.apps.admin')
283 config.include('rhodecode.apps.channelstream')
283 config.include('rhodecode.apps.channelstream')
284 config.include('rhodecode.apps.upload_store')
284 config.include('rhodecode.apps.file_store')
285 config.include('rhodecode.apps.login')
285 config.include('rhodecode.apps.login')
286 config.include('rhodecode.apps.home')
286 config.include('rhodecode.apps.home')
287 config.include('rhodecode.apps.journal')
287 config.include('rhodecode.apps.journal')
288 config.include('rhodecode.apps.repository')
288 config.include('rhodecode.apps.repository')
289 config.include('rhodecode.apps.repo_group')
289 config.include('rhodecode.apps.repo_group')
290 config.include('rhodecode.apps.user_group')
290 config.include('rhodecode.apps.user_group')
291 config.include('rhodecode.apps.search')
291 config.include('rhodecode.apps.search')
292 config.include('rhodecode.apps.user_profile')
292 config.include('rhodecode.apps.user_profile')
293 config.include('rhodecode.apps.user_group_profile')
293 config.include('rhodecode.apps.user_group_profile')
294 config.include('rhodecode.apps.my_account')
294 config.include('rhodecode.apps.my_account')
295 config.include('rhodecode.apps.svn_support')
295 config.include('rhodecode.apps.svn_support')
296 config.include('rhodecode.apps.ssh_support')
296 config.include('rhodecode.apps.ssh_support')
297 config.include('rhodecode.apps.gist')
297 config.include('rhodecode.apps.gist')
298 config.include('rhodecode.apps.debug_style')
298 config.include('rhodecode.apps.debug_style')
299 config.include('rhodecode.api')
299 config.include('rhodecode.api')
300
300
301 config.add_route('rhodecode_support', 'https://rhodecode.com/help/', static=True)
301 config.add_route('rhodecode_support', 'https://rhodecode.com/help/', static=True)
302 config.add_translation_dirs('rhodecode:i18n/')
302 config.add_translation_dirs('rhodecode:i18n/')
303 settings['default_locale_name'] = settings.get('lang', 'en')
303 settings['default_locale_name'] = settings.get('lang', 'en')
304
304
305 # Add subscribers.
305 # Add subscribers.
306 config.add_subscriber(inject_app_settings,
306 config.add_subscriber(inject_app_settings,
307 pyramid.events.ApplicationCreated)
307 pyramid.events.ApplicationCreated)
308 config.add_subscriber(scan_repositories_if_enabled,
308 config.add_subscriber(scan_repositories_if_enabled,
309 pyramid.events.ApplicationCreated)
309 pyramid.events.ApplicationCreated)
310 config.add_subscriber(write_metadata_if_needed,
310 config.add_subscriber(write_metadata_if_needed,
311 pyramid.events.ApplicationCreated)
311 pyramid.events.ApplicationCreated)
312 config.add_subscriber(write_js_routes_if_enabled,
312 config.add_subscriber(write_js_routes_if_enabled,
313 pyramid.events.ApplicationCreated)
313 pyramid.events.ApplicationCreated)
314
314
315 # request custom methods
315 # request custom methods
316 config.add_request_method(
316 config.add_request_method(
317 'rhodecode.lib.partial_renderer.get_partial_renderer',
317 'rhodecode.lib.partial_renderer.get_partial_renderer',
318 'get_partial_renderer')
318 'get_partial_renderer')
319
319
320 # Set the authorization policy.
320 # Set the authorization policy.
321 authz_policy = ACLAuthorizationPolicy()
321 authz_policy = ACLAuthorizationPolicy()
322 config.set_authorization_policy(authz_policy)
322 config.set_authorization_policy(authz_policy)
323
323
324 # Set the default renderer for HTML templates to mako.
324 # Set the default renderer for HTML templates to mako.
325 config.add_mako_renderer('.html')
325 config.add_mako_renderer('.html')
326
326
327 config.add_renderer(
327 config.add_renderer(
328 name='json_ext',
328 name='json_ext',
329 factory='rhodecode.lib.ext_json_renderer.pyramid_ext_json')
329 factory='rhodecode.lib.ext_json_renderer.pyramid_ext_json')
330
330
331 # include RhodeCode plugins
331 # include RhodeCode plugins
332 includes = aslist(settings.get('rhodecode.includes', []))
332 includes = aslist(settings.get('rhodecode.includes', []))
333 for inc in includes:
333 for inc in includes:
334 config.include(inc)
334 config.include(inc)
335
335
336 # custom not found view, if our pyramid app doesn't know how to handle
336 # custom not found view, if our pyramid app doesn't know how to handle
337 # the request pass it to potential VCS handling ap
337 # the request pass it to potential VCS handling ap
338 config.add_notfound_view(not_found_view)
338 config.add_notfound_view(not_found_view)
339 if not settings.get('debugtoolbar.enabled', False):
339 if not settings.get('debugtoolbar.enabled', False):
340 # disabled debugtoolbar handle all exceptions via the error_handlers
340 # disabled debugtoolbar handle all exceptions via the error_handlers
341 config.add_view(error_handler, context=Exception)
341 config.add_view(error_handler, context=Exception)
342
342
343 # all errors including 403/404/50X
343 # all errors including 403/404/50X
344 config.add_view(error_handler, context=HTTPError)
344 config.add_view(error_handler, context=HTTPError)
345
345
346
346
347 def wrap_app_in_wsgi_middlewares(pyramid_app, config):
347 def wrap_app_in_wsgi_middlewares(pyramid_app, config):
348 """
348 """
349 Apply outer WSGI middlewares around the application.
349 Apply outer WSGI middlewares around the application.
350 """
350 """
351 registry = config.registry
351 registry = config.registry
352 settings = registry.settings
352 settings = registry.settings
353
353
354 # enable https redirects based on HTTP_X_URL_SCHEME set by proxy
354 # enable https redirects based on HTTP_X_URL_SCHEME set by proxy
355 pyramid_app = HttpsFixup(pyramid_app, settings)
355 pyramid_app = HttpsFixup(pyramid_app, settings)
356
356
357 pyramid_app, _ae_client = wrap_in_appenlight_if_enabled(
357 pyramid_app, _ae_client = wrap_in_appenlight_if_enabled(
358 pyramid_app, settings)
358 pyramid_app, settings)
359 registry.ae_client = _ae_client
359 registry.ae_client = _ae_client
360
360
361 if settings['gzip_responses']:
361 if settings['gzip_responses']:
362 pyramid_app = make_gzip_middleware(
362 pyramid_app = make_gzip_middleware(
363 pyramid_app, settings, compress_level=1)
363 pyramid_app, settings, compress_level=1)
364
364
365 # this should be the outer most middleware in the wsgi stack since
365 # this should be the outer most middleware in the wsgi stack since
366 # middleware like Routes make database calls
366 # middleware like Routes make database calls
367 def pyramid_app_with_cleanup(environ, start_response):
367 def pyramid_app_with_cleanup(environ, start_response):
368 try:
368 try:
369 return pyramid_app(environ, start_response)
369 return pyramid_app(environ, start_response)
370 finally:
370 finally:
371 # Dispose current database session and rollback uncommitted
371 # Dispose current database session and rollback uncommitted
372 # transactions.
372 # transactions.
373 meta.Session.remove()
373 meta.Session.remove()
374
374
375 # In a single threaded mode server, on non sqlite db we should have
375 # In a single threaded mode server, on non sqlite db we should have
376 # '0 Current Checked out connections' at the end of a request,
376 # '0 Current Checked out connections' at the end of a request,
377 # if not, then something, somewhere is leaving a connection open
377 # if not, then something, somewhere is leaving a connection open
378 pool = meta.Base.metadata.bind.engine.pool
378 pool = meta.Base.metadata.bind.engine.pool
379 log.debug('sa pool status: %s', pool.status())
379 log.debug('sa pool status: %s', pool.status())
380 log.debug('Request processing finalized')
380 log.debug('Request processing finalized')
381
381
382 return pyramid_app_with_cleanup
382 return pyramid_app_with_cleanup
383
383
384
384
385 def sanitize_settings_and_apply_defaults(global_config, settings):
385 def sanitize_settings_and_apply_defaults(global_config, settings):
386 """
386 """
387 Applies settings defaults and does all type conversion.
387 Applies settings defaults and does all type conversion.
388
388
389 We would move all settings parsing and preparation into this place, so that
389 We would move all settings parsing and preparation into this place, so that
390 we have only one place left which deals with this part. The remaining parts
390 we have only one place left which deals with this part. The remaining parts
391 of the application would start to rely fully on well prepared settings.
391 of the application would start to rely fully on well prepared settings.
392
392
393 This piece would later be split up per topic to avoid a big fat monster
393 This piece would later be split up per topic to avoid a big fat monster
394 function.
394 function.
395 """
395 """
396
396
397 settings.setdefault('rhodecode.edition', 'Community Edition')
397 settings.setdefault('rhodecode.edition', 'Community Edition')
398
398
399 if 'mako.default_filters' not in settings:
399 if 'mako.default_filters' not in settings:
400 # set custom default filters if we don't have it defined
400 # set custom default filters if we don't have it defined
401 settings['mako.imports'] = 'from rhodecode.lib.base import h_filter'
401 settings['mako.imports'] = 'from rhodecode.lib.base import h_filter'
402 settings['mako.default_filters'] = 'h_filter'
402 settings['mako.default_filters'] = 'h_filter'
403
403
404 if 'mako.directories' not in settings:
404 if 'mako.directories' not in settings:
405 mako_directories = settings.setdefault('mako.directories', [
405 mako_directories = settings.setdefault('mako.directories', [
406 # Base templates of the original application
406 # Base templates of the original application
407 'rhodecode:templates',
407 'rhodecode:templates',
408 ])
408 ])
409 log.debug(
409 log.debug(
410 "Using the following Mako template directories: %s",
410 "Using the following Mako template directories: %s",
411 mako_directories)
411 mako_directories)
412
412
413 # Default includes, possible to change as a user
413 # Default includes, possible to change as a user
414 pyramid_includes = settings.setdefault('pyramid.includes', [
414 pyramid_includes = settings.setdefault('pyramid.includes', [
415 'rhodecode.lib.middleware.request_wrapper',
415 'rhodecode.lib.middleware.request_wrapper',
416 ])
416 ])
417 log.debug(
417 log.debug(
418 "Using the following pyramid.includes: %s",
418 "Using the following pyramid.includes: %s",
419 pyramid_includes)
419 pyramid_includes)
420
420
421 # TODO: johbo: Re-think this, usually the call to config.include
421 # TODO: johbo: Re-think this, usually the call to config.include
422 # should allow to pass in a prefix.
422 # should allow to pass in a prefix.
423 settings.setdefault('rhodecode.api.url', '/_admin/api')
423 settings.setdefault('rhodecode.api.url', '/_admin/api')
424 settings.setdefault('__file__', global_config.get('__file__'))
424 settings.setdefault('__file__', global_config.get('__file__'))
425
425
426 # Sanitize generic settings.
426 # Sanitize generic settings.
427 _list_setting(settings, 'default_encoding', 'UTF-8')
427 _list_setting(settings, 'default_encoding', 'UTF-8')
428 _bool_setting(settings, 'is_test', 'false')
428 _bool_setting(settings, 'is_test', 'false')
429 _bool_setting(settings, 'gzip_responses', 'false')
429 _bool_setting(settings, 'gzip_responses', 'false')
430
430
431 # Call split out functions that sanitize settings for each topic.
431 # Call split out functions that sanitize settings for each topic.
432 _sanitize_appenlight_settings(settings)
432 _sanitize_appenlight_settings(settings)
433 _sanitize_vcs_settings(settings)
433 _sanitize_vcs_settings(settings)
434 _sanitize_cache_settings(settings)
434 _sanitize_cache_settings(settings)
435
435
436 # configure instance id
436 # configure instance id
437 config_utils.set_instance_id(settings)
437 config_utils.set_instance_id(settings)
438
438
439 return settings
439 return settings
440
440
441
441
442 def enable_debug():
442 def enable_debug():
443 """
443 """
444 Helper to enable debug on running instance
444 Helper to enable debug on running instance
445 :return:
445 :return:
446 """
446 """
447 import tempfile
447 import tempfile
448 import textwrap
448 import textwrap
449 import logging.config
449 import logging.config
450
450
451 ini_template = textwrap.dedent("""
451 ini_template = textwrap.dedent("""
452 #####################################
452 #####################################
453 ### DEBUG LOGGING CONFIGURATION ####
453 ### DEBUG LOGGING CONFIGURATION ####
454 #####################################
454 #####################################
455 [loggers]
455 [loggers]
456 keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
456 keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
457
457
458 [handlers]
458 [handlers]
459 keys = console, console_sql
459 keys = console, console_sql
460
460
461 [formatters]
461 [formatters]
462 keys = generic, color_formatter, color_formatter_sql
462 keys = generic, color_formatter, color_formatter_sql
463
463
464 #############
464 #############
465 ## LOGGERS ##
465 ## LOGGERS ##
466 #############
466 #############
467 [logger_root]
467 [logger_root]
468 level = NOTSET
468 level = NOTSET
469 handlers = console
469 handlers = console
470
470
471 [logger_sqlalchemy]
471 [logger_sqlalchemy]
472 level = INFO
472 level = INFO
473 handlers = console_sql
473 handlers = console_sql
474 qualname = sqlalchemy.engine
474 qualname = sqlalchemy.engine
475 propagate = 0
475 propagate = 0
476
476
477 [logger_beaker]
477 [logger_beaker]
478 level = DEBUG
478 level = DEBUG
479 handlers =
479 handlers =
480 qualname = beaker.container
480 qualname = beaker.container
481 propagate = 1
481 propagate = 1
482
482
483 [logger_rhodecode]
483 [logger_rhodecode]
484 level = DEBUG
484 level = DEBUG
485 handlers =
485 handlers =
486 qualname = rhodecode
486 qualname = rhodecode
487 propagate = 1
487 propagate = 1
488
488
489 [logger_ssh_wrapper]
489 [logger_ssh_wrapper]
490 level = DEBUG
490 level = DEBUG
491 handlers =
491 handlers =
492 qualname = ssh_wrapper
492 qualname = ssh_wrapper
493 propagate = 1
493 propagate = 1
494
494
495 [logger_celery]
495 [logger_celery]
496 level = DEBUG
496 level = DEBUG
497 handlers =
497 handlers =
498 qualname = celery
498 qualname = celery
499
499
500
500
501 ##############
501 ##############
502 ## HANDLERS ##
502 ## HANDLERS ##
503 ##############
503 ##############
504
504
505 [handler_console]
505 [handler_console]
506 class = StreamHandler
506 class = StreamHandler
507 args = (sys.stderr, )
507 args = (sys.stderr, )
508 level = DEBUG
508 level = DEBUG
509 formatter = color_formatter
509 formatter = color_formatter
510
510
511 [handler_console_sql]
511 [handler_console_sql]
512 # "level = DEBUG" logs SQL queries and results.
512 # "level = DEBUG" logs SQL queries and results.
513 # "level = INFO" logs SQL queries.
513 # "level = INFO" logs SQL queries.
514 # "level = WARN" logs neither. (Recommended for production systems.)
514 # "level = WARN" logs neither. (Recommended for production systems.)
515 class = StreamHandler
515 class = StreamHandler
516 args = (sys.stderr, )
516 args = (sys.stderr, )
517 level = WARN
517 level = WARN
518 formatter = color_formatter_sql
518 formatter = color_formatter_sql
519
519
520 ################
520 ################
521 ## FORMATTERS ##
521 ## FORMATTERS ##
522 ################
522 ################
523
523
524 [formatter_generic]
524 [formatter_generic]
525 class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
525 class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
526 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s | %(req_id)s
526 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s | %(req_id)s
527 datefmt = %Y-%m-%d %H:%M:%S
527 datefmt = %Y-%m-%d %H:%M:%S
528
528
529 [formatter_color_formatter]
529 [formatter_color_formatter]
530 class = rhodecode.lib.logging_formatter.ColorRequestTrackingFormatter
530 class = rhodecode.lib.logging_formatter.ColorRequestTrackingFormatter
531 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s | %(req_id)s
531 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s | %(req_id)s
532 datefmt = %Y-%m-%d %H:%M:%S
532 datefmt = %Y-%m-%d %H:%M:%S
533
533
534 [formatter_color_formatter_sql]
534 [formatter_color_formatter_sql]
535 class = rhodecode.lib.logging_formatter.ColorFormatterSql
535 class = rhodecode.lib.logging_formatter.ColorFormatterSql
536 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
536 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
537 datefmt = %Y-%m-%d %H:%M:%S
537 datefmt = %Y-%m-%d %H:%M:%S
538 """)
538 """)
539
539
540 with tempfile.NamedTemporaryFile(prefix='rc_debug_logging_', suffix='.ini',
540 with tempfile.NamedTemporaryFile(prefix='rc_debug_logging_', suffix='.ini',
541 delete=False) as f:
541 delete=False) as f:
542 log.info('Saved Temporary DEBUG config at %s', f.name)
542 log.info('Saved Temporary DEBUG config at %s', f.name)
543 f.write(ini_template)
543 f.write(ini_template)
544
544
545 logging.config.fileConfig(f.name)
545 logging.config.fileConfig(f.name)
546 log.debug('DEBUG MODE ON')
546 log.debug('DEBUG MODE ON')
547 os.remove(f.name)
547 os.remove(f.name)
548
548
549
549
550 def _sanitize_appenlight_settings(settings):
550 def _sanitize_appenlight_settings(settings):
551 _bool_setting(settings, 'appenlight', 'false')
551 _bool_setting(settings, 'appenlight', 'false')
552
552
553
553
554 def _sanitize_vcs_settings(settings):
554 def _sanitize_vcs_settings(settings):
555 """
555 """
556 Applies settings defaults and does type conversion for all VCS related
556 Applies settings defaults and does type conversion for all VCS related
557 settings.
557 settings.
558 """
558 """
559 _string_setting(settings, 'vcs.svn.compatible_version', '')
559 _string_setting(settings, 'vcs.svn.compatible_version', '')
560 _string_setting(settings, 'git_rev_filter', '--all')
560 _string_setting(settings, 'git_rev_filter', '--all')
561 _string_setting(settings, 'vcs.hooks.protocol', 'http')
561 _string_setting(settings, 'vcs.hooks.protocol', 'http')
562 _string_setting(settings, 'vcs.hooks.host', '127.0.0.1')
562 _string_setting(settings, 'vcs.hooks.host', '127.0.0.1')
563 _string_setting(settings, 'vcs.scm_app_implementation', 'http')
563 _string_setting(settings, 'vcs.scm_app_implementation', 'http')
564 _string_setting(settings, 'vcs.server', '')
564 _string_setting(settings, 'vcs.server', '')
565 _string_setting(settings, 'vcs.server.log_level', 'debug')
565 _string_setting(settings, 'vcs.server.log_level', 'debug')
566 _string_setting(settings, 'vcs.server.protocol', 'http')
566 _string_setting(settings, 'vcs.server.protocol', 'http')
567 _bool_setting(settings, 'startup.import_repos', 'false')
567 _bool_setting(settings, 'startup.import_repos', 'false')
568 _bool_setting(settings, 'vcs.hooks.direct_calls', 'false')
568 _bool_setting(settings, 'vcs.hooks.direct_calls', 'false')
569 _bool_setting(settings, 'vcs.server.enable', 'true')
569 _bool_setting(settings, 'vcs.server.enable', 'true')
570 _bool_setting(settings, 'vcs.start_server', 'false')
570 _bool_setting(settings, 'vcs.start_server', 'false')
571 _list_setting(settings, 'vcs.backends', 'hg, git, svn')
571 _list_setting(settings, 'vcs.backends', 'hg, git, svn')
572 _int_setting(settings, 'vcs.connection_timeout', 3600)
572 _int_setting(settings, 'vcs.connection_timeout', 3600)
573
573
574 # Support legacy values of vcs.scm_app_implementation. Legacy
574 # Support legacy values of vcs.scm_app_implementation. Legacy
575 # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
575 # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
576 # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
576 # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
577 scm_app_impl = settings['vcs.scm_app_implementation']
577 scm_app_impl = settings['vcs.scm_app_implementation']
578 if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']:
578 if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']:
579 settings['vcs.scm_app_implementation'] = 'http'
579 settings['vcs.scm_app_implementation'] = 'http'
580
580
581
581
582 def _sanitize_cache_settings(settings):
582 def _sanitize_cache_settings(settings):
583 temp_store = tempfile.gettempdir()
583 temp_store = tempfile.gettempdir()
584 default_cache_dir = os.path.join(temp_store, 'rc_cache')
584 default_cache_dir = os.path.join(temp_store, 'rc_cache')
585
585
586 # save default, cache dir, and use it for all backends later.
586 # save default, cache dir, and use it for all backends later.
587 default_cache_dir = _string_setting(
587 default_cache_dir = _string_setting(
588 settings,
588 settings,
589 'cache_dir',
589 'cache_dir',
590 default_cache_dir, lower=False, default_when_empty=True)
590 default_cache_dir, lower=False, default_when_empty=True)
591
591
592 # ensure we have our dir created
592 # ensure we have our dir created
593 if not os.path.isdir(default_cache_dir):
593 if not os.path.isdir(default_cache_dir):
594 os.makedirs(default_cache_dir, mode=0o755)
594 os.makedirs(default_cache_dir, mode=0o755)
595
595
596 # exception store cache
596 # exception store cache
597 _string_setting(
597 _string_setting(
598 settings,
598 settings,
599 'exception_tracker.store_path',
599 'exception_tracker.store_path',
600 temp_store, lower=False, default_when_empty=True)
600 temp_store, lower=False, default_when_empty=True)
601
601
602 # cache_perms
602 # cache_perms
603 _string_setting(
603 _string_setting(
604 settings,
604 settings,
605 'rc_cache.cache_perms.backend',
605 'rc_cache.cache_perms.backend',
606 'dogpile.cache.rc.file_namespace', lower=False)
606 'dogpile.cache.rc.file_namespace', lower=False)
607 _int_setting(
607 _int_setting(
608 settings,
608 settings,
609 'rc_cache.cache_perms.expiration_time',
609 'rc_cache.cache_perms.expiration_time',
610 60)
610 60)
611 _string_setting(
611 _string_setting(
612 settings,
612 settings,
613 'rc_cache.cache_perms.arguments.filename',
613 'rc_cache.cache_perms.arguments.filename',
614 os.path.join(default_cache_dir, 'rc_cache_1'), lower=False)
614 os.path.join(default_cache_dir, 'rc_cache_1'), lower=False)
615
615
616 # cache_repo
616 # cache_repo
617 _string_setting(
617 _string_setting(
618 settings,
618 settings,
619 'rc_cache.cache_repo.backend',
619 'rc_cache.cache_repo.backend',
620 'dogpile.cache.rc.file_namespace', lower=False)
620 'dogpile.cache.rc.file_namespace', lower=False)
621 _int_setting(
621 _int_setting(
622 settings,
622 settings,
623 'rc_cache.cache_repo.expiration_time',
623 'rc_cache.cache_repo.expiration_time',
624 60)
624 60)
625 _string_setting(
625 _string_setting(
626 settings,
626 settings,
627 'rc_cache.cache_repo.arguments.filename',
627 'rc_cache.cache_repo.arguments.filename',
628 os.path.join(default_cache_dir, 'rc_cache_2'), lower=False)
628 os.path.join(default_cache_dir, 'rc_cache_2'), lower=False)
629
629
630 # cache_license
630 # cache_license
631 _string_setting(
631 _string_setting(
632 settings,
632 settings,
633 'rc_cache.cache_license.backend',
633 'rc_cache.cache_license.backend',
634 'dogpile.cache.rc.file_namespace', lower=False)
634 'dogpile.cache.rc.file_namespace', lower=False)
635 _int_setting(
635 _int_setting(
636 settings,
636 settings,
637 'rc_cache.cache_license.expiration_time',
637 'rc_cache.cache_license.expiration_time',
638 5*60)
638 5*60)
639 _string_setting(
639 _string_setting(
640 settings,
640 settings,
641 'rc_cache.cache_license.arguments.filename',
641 'rc_cache.cache_license.arguments.filename',
642 os.path.join(default_cache_dir, 'rc_cache_3'), lower=False)
642 os.path.join(default_cache_dir, 'rc_cache_3'), lower=False)
643
643
644 # cache_repo_longterm memory, 96H
644 # cache_repo_longterm memory, 96H
645 _string_setting(
645 _string_setting(
646 settings,
646 settings,
647 'rc_cache.cache_repo_longterm.backend',
647 'rc_cache.cache_repo_longterm.backend',
648 'dogpile.cache.rc.memory_lru', lower=False)
648 'dogpile.cache.rc.memory_lru', lower=False)
649 _int_setting(
649 _int_setting(
650 settings,
650 settings,
651 'rc_cache.cache_repo_longterm.expiration_time',
651 'rc_cache.cache_repo_longterm.expiration_time',
652 345600)
652 345600)
653 _int_setting(
653 _int_setting(
654 settings,
654 settings,
655 'rc_cache.cache_repo_longterm.max_size',
655 'rc_cache.cache_repo_longterm.max_size',
656 10000)
656 10000)
657
657
658 # sql_cache_short
658 # sql_cache_short
659 _string_setting(
659 _string_setting(
660 settings,
660 settings,
661 'rc_cache.sql_cache_short.backend',
661 'rc_cache.sql_cache_short.backend',
662 'dogpile.cache.rc.memory_lru', lower=False)
662 'dogpile.cache.rc.memory_lru', lower=False)
663 _int_setting(
663 _int_setting(
664 settings,
664 settings,
665 'rc_cache.sql_cache_short.expiration_time',
665 'rc_cache.sql_cache_short.expiration_time',
666 30)
666 30)
667 _int_setting(
667 _int_setting(
668 settings,
668 settings,
669 'rc_cache.sql_cache_short.max_size',
669 'rc_cache.sql_cache_short.max_size',
670 10000)
670 10000)
671
671
672
672
673 def _int_setting(settings, name, default):
673 def _int_setting(settings, name, default):
674 settings[name] = int(settings.get(name, default))
674 settings[name] = int(settings.get(name, default))
675 return settings[name]
675 return settings[name]
676
676
677
677
678 def _bool_setting(settings, name, default):
678 def _bool_setting(settings, name, default):
679 input_val = settings.get(name, default)
679 input_val = settings.get(name, default)
680 if isinstance(input_val, unicode):
680 if isinstance(input_val, unicode):
681 input_val = input_val.encode('utf8')
681 input_val = input_val.encode('utf8')
682 settings[name] = asbool(input_val)
682 settings[name] = asbool(input_val)
683 return settings[name]
683 return settings[name]
684
684
685
685
686 def _list_setting(settings, name, default):
686 def _list_setting(settings, name, default):
687 raw_value = settings.get(name, default)
687 raw_value = settings.get(name, default)
688
688
689 old_separator = ','
689 old_separator = ','
690 if old_separator in raw_value:
690 if old_separator in raw_value:
691 # If we get a comma separated list, pass it to our own function.
691 # If we get a comma separated list, pass it to our own function.
692 settings[name] = rhodecode_aslist(raw_value, sep=old_separator)
692 settings[name] = rhodecode_aslist(raw_value, sep=old_separator)
693 else:
693 else:
694 # Otherwise we assume it uses pyramids space/newline separation.
694 # Otherwise we assume it uses pyramids space/newline separation.
695 settings[name] = aslist(raw_value)
695 settings[name] = aslist(raw_value)
696 return settings[name]
696 return settings[name]
697
697
698
698
699 def _string_setting(settings, name, default, lower=True, default_when_empty=False):
699 def _string_setting(settings, name, default, lower=True, default_when_empty=False):
700 value = settings.get(name, default)
700 value = settings.get(name, default)
701
701
702 if default_when_empty and not value:
702 if default_when_empty and not value:
703 # use default value when value is empty
703 # use default value when value is empty
704 value = default
704 value = default
705
705
706 if lower:
706 if lower:
707 value = value.lower()
707 value = value.lower()
708 settings[name] = value
708 settings[name] = value
709 return settings[name]
709 return settings[name]
710
710
711
711
712 def _substitute_values(mapping, substitutions):
712 def _substitute_values(mapping, substitutions):
713 result = {}
713 result = {}
714
714
715 try:
715 try:
716 for key, value in mapping.items():
716 for key, value in mapping.items():
717 # initialize without substitution first
717 # initialize without substitution first
718 result[key] = value
718 result[key] = value
719
719
720 # Note: Cannot use regular replacements, since they would clash
720 # Note: Cannot use regular replacements, since they would clash
721 # with the implementation of ConfigParser. Using "format" instead.
721 # with the implementation of ConfigParser. Using "format" instead.
722 try:
722 try:
723 result[key] = value.format(**substitutions)
723 result[key] = value.format(**substitutions)
724 except KeyError as e:
724 except KeyError as e:
725 env_var = '{}'.format(e.args[0])
725 env_var = '{}'.format(e.args[0])
726
726
727 msg = 'Failed to substitute: `{key}={{{var}}}` with environment entry. ' \
727 msg = 'Failed to substitute: `{key}={{{var}}}` with environment entry. ' \
728 'Make sure your environment has {var} set, or remove this ' \
728 'Make sure your environment has {var} set, or remove this ' \
729 'variable from config file'.format(key=key, var=env_var)
729 'variable from config file'.format(key=key, var=env_var)
730
730
731 if env_var.startswith('ENV_'):
731 if env_var.startswith('ENV_'):
732 raise ValueError(msg)
732 raise ValueError(msg)
733 else:
733 else:
734 log.warning(msg)
734 log.warning(msg)
735
735
736 except ValueError as e:
736 except ValueError as e:
737 log.warning('Failed to substitute ENV variable: %s', e)
737 log.warning('Failed to substitute ENV variable: %s', e)
738 result = mapping
738 result = mapping
739
739
740 return result
740 return result
General Comments 0
You need to be logged in to leave comments. Login now