##// END OF EJS Templates
vcs: Move code for parsing the repository names (url, vcs, acl) from vcs to simplevcs.
Martin Bornhold -
r889:4fa21815 default
parent child Browse files
Show More
@@ -1,453 +1,479 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2014-2016 RhodeCode GmbH
3 # Copyright (C) 2014-2016 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 """
21 """
22 SimpleVCS middleware for handling protocol request (push/clone etc.)
22 SimpleVCS middleware for handling protocol request (push/clone etc.)
23 It's implemented with basic auth function
23 It's implemented with basic auth function
24 """
24 """
25
25
26 import os
26 import os
27 import logging
27 import logging
28 import importlib
28 import importlib
29 import re
29 from functools import wraps
30 from functools import wraps
30
31
31 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
32 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
32 from webob.exc import (
33 from webob.exc import (
33 HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
34 HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
34
35
35 import rhodecode
36 import rhodecode
36 from rhodecode.authentication.base import authenticate, VCS_TYPE
37 from rhodecode.authentication.base import authenticate, VCS_TYPE
37 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
38 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
38 from rhodecode.lib.base import BasicAuth, get_ip_addr, vcs_operation_context
39 from rhodecode.lib.base import BasicAuth, get_ip_addr, vcs_operation_context
39 from rhodecode.lib.exceptions import (
40 from rhodecode.lib.exceptions import (
40 HTTPLockedRC, HTTPRequirementError, UserCreationError,
41 HTTPLockedRC, HTTPRequirementError, UserCreationError,
41 NotAllowedToCreateUserError)
42 NotAllowedToCreateUserError)
42 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
43 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
43 from rhodecode.lib.middleware import appenlight
44 from rhodecode.lib.middleware import appenlight
44 from rhodecode.lib.middleware.utils import scm_app
45 from rhodecode.lib.middleware.utils import scm_app
45 from rhodecode.lib.utils import (
46 from rhodecode.lib.utils import (
46 is_valid_repo, get_rhodecode_realm, get_rhodecode_base_path)
47 is_valid_repo, get_rhodecode_realm, get_rhodecode_base_path)
47 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool
48 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool
48 from rhodecode.lib.vcs.conf import settings as vcs_settings
49 from rhodecode.lib.vcs.conf import settings as vcs_settings
49 from rhodecode.lib.vcs.backends import base
50 from rhodecode.lib.vcs.backends import base
50 from rhodecode.model import meta
51 from rhodecode.model import meta
51 from rhodecode.model.db import User, Repository
52 from rhodecode.model.db import User, Repository
52 from rhodecode.model.scm import ScmModel
53 from rhodecode.model.scm import ScmModel
53
54
54
55
55 log = logging.getLogger(__name__)
56 log = logging.getLogger(__name__)
56
57
57
58
58 def initialize_generator(factory):
59 def initialize_generator(factory):
59 """
60 """
60 Initializes the returned generator by draining its first element.
61 Initializes the returned generator by draining its first element.
61
62
62 This can be used to give a generator an initializer, which is the code
63 This can be used to give a generator an initializer, which is the code
63 up to the first yield statement. This decorator enforces that the first
64 up to the first yield statement. This decorator enforces that the first
64 produced element has the value ``"__init__"`` to make its special
65 produced element has the value ``"__init__"`` to make its special
65 purpose very explicit in the using code.
66 purpose very explicit in the using code.
66 """
67 """
67
68
68 @wraps(factory)
69 @wraps(factory)
69 def wrapper(*args, **kwargs):
70 def wrapper(*args, **kwargs):
70 gen = factory(*args, **kwargs)
71 gen = factory(*args, **kwargs)
71 try:
72 try:
72 init = gen.next()
73 init = gen.next()
73 except StopIteration:
74 except StopIteration:
74 raise ValueError('Generator must yield at least one element.')
75 raise ValueError('Generator must yield at least one element.')
75 if init != "__init__":
76 if init != "__init__":
76 raise ValueError('First yielded element must be "__init__".')
77 raise ValueError('First yielded element must be "__init__".')
77 return gen
78 return gen
78 return wrapper
79 return wrapper
79
80
80
81
81 class SimpleVCS(object):
82 class SimpleVCS(object):
82 """Common functionality for SCM HTTP handlers."""
83 """Common functionality for SCM HTTP handlers."""
83
84
84 SCM = 'unknown'
85 SCM = 'unknown'
85
86
86 acl_repo_name = None
87 acl_repo_name = None
87 url_repo_name = None
88 url_repo_name = None
88 vcs_repo_name = None
89 vcs_repo_name = None
89
90
90 def __init__(self, application, config, registry):
91 def __init__(self, application, config, registry):
91 self.registry = registry
92 self.registry = registry
92 self.application = application
93 self.application = application
93 self.config = config
94 self.config = config
94 # re-populated by specialized middleware
95 # re-populated by specialized middleware
95 self.repo_vcs_config = base.Config()
96 self.repo_vcs_config = base.Config()
96
97
97 # base path of repo locations
98 # base path of repo locations
98 self.basepath = get_rhodecode_base_path()
99 self.basepath = get_rhodecode_base_path()
99 # authenticate this VCS request using authfunc
100 # authenticate this VCS request using authfunc
100 auth_ret_code_detection = \
101 auth_ret_code_detection = \
101 str2bool(self.config.get('auth_ret_code_detection', False))
102 str2bool(self.config.get('auth_ret_code_detection', False))
102 self.authenticate = BasicAuth(
103 self.authenticate = BasicAuth(
103 '', authenticate, registry, config.get('auth_ret_code'),
104 '', authenticate, registry, config.get('auth_ret_code'),
104 auth_ret_code_detection)
105 auth_ret_code_detection)
105 self.ip_addr = '0.0.0.0'
106 self.ip_addr = '0.0.0.0'
106
107
108 def set_repo_names(self, environ):
109 """
110 This will populate the attributes acl_repo_name, url_repo_name and
111 vcs_repo_name on the current instance.
112 """
113 # TODO: martinb: Move to class or module scope.
114 pr_regex = re.compile(
115 '(?P<base_name>(?:[\w-]+)(?:/[\w-]+)*)/'
116 '(?P<repo_name>[\w-]+)'
117 '/pull-request/(?P<pr_id>\d+)/repository')
118
119 self.url_repo_name = self._get_repository_name(environ)
120 match = pr_regex.match(self.url_repo_name)
121
122 if match:
123 match_dict = match.groupdict()
124 self.acl_repo_name = '{base_name}/{repo_name}'.format(**match_dict)
125 self.vcs_repo_name = '{base_name}/.__shadow_{repo_name}_pr-{pr_id}'.format(
126 **match_dict)
127 self.pr_id = match_dict.get('pr_id')
128 else:
129 self.acl_repo_name = self.url_repo_name
130 self.vcs_repo_name = self.url_repo_name
131 self.pr_id = None
132
107 @property
133 @property
108 def repo_name(self):
134 def repo_name(self):
109 # TODO: johbo: Remove, switch to correct repo name attribute
135 # TODO: johbo: Remove, switch to correct repo name attribute
110 return self.acl_repo_name
136 return self.acl_repo_name
111
137
112 @property
138 @property
113 def scm_app(self):
139 def scm_app(self):
114 custom_implementation = self.config.get('vcs.scm_app_implementation')
140 custom_implementation = self.config.get('vcs.scm_app_implementation')
115 if custom_implementation and custom_implementation != 'pyro4':
141 if custom_implementation and custom_implementation != 'pyro4':
116 log.info(
142 log.info(
117 "Using custom implementation of scm_app: %s",
143 "Using custom implementation of scm_app: %s",
118 custom_implementation)
144 custom_implementation)
119 scm_app_impl = importlib.import_module(custom_implementation)
145 scm_app_impl = importlib.import_module(custom_implementation)
120 else:
146 else:
121 scm_app_impl = scm_app
147 scm_app_impl = scm_app
122 return scm_app_impl
148 return scm_app_impl
123
149
124 def _get_by_id(self, repo_name):
150 def _get_by_id(self, repo_name):
125 """
151 """
126 Gets a special pattern _<ID> from clone url and tries to replace it
152 Gets a special pattern _<ID> from clone url and tries to replace it
127 with a repository_name for support of _<ID> non changeable urls
153 with a repository_name for support of _<ID> non changeable urls
128 """
154 """
129
155
130 data = repo_name.split('/')
156 data = repo_name.split('/')
131 if len(data) >= 2:
157 if len(data) >= 2:
132 from rhodecode.model.repo import RepoModel
158 from rhodecode.model.repo import RepoModel
133 by_id_match = RepoModel().get_repo_by_id(repo_name)
159 by_id_match = RepoModel().get_repo_by_id(repo_name)
134 if by_id_match:
160 if by_id_match:
135 data[1] = by_id_match.repo_name
161 data[1] = by_id_match.repo_name
136
162
137 return safe_str('/'.join(data))
163 return safe_str('/'.join(data))
138
164
139 def _invalidate_cache(self, repo_name):
165 def _invalidate_cache(self, repo_name):
140 """
166 """
141 Set's cache for this repository for invalidation on next access
167 Set's cache for this repository for invalidation on next access
142
168
143 :param repo_name: full repo name, also a cache key
169 :param repo_name: full repo name, also a cache key
144 """
170 """
145 ScmModel().mark_for_invalidation(repo_name)
171 ScmModel().mark_for_invalidation(repo_name)
146
172
147 def is_valid_and_existing_repo(self, repo_name, base_path, scm_type):
173 def is_valid_and_existing_repo(self, repo_name, base_path, scm_type):
148 db_repo = Repository.get_by_repo_name(repo_name)
174 db_repo = Repository.get_by_repo_name(repo_name)
149 if not db_repo:
175 if not db_repo:
150 log.debug('Repository `%s` not found inside the database.',
176 log.debug('Repository `%s` not found inside the database.',
151 repo_name)
177 repo_name)
152 return False
178 return False
153
179
154 if db_repo.repo_type != scm_type:
180 if db_repo.repo_type != scm_type:
155 log.warning(
181 log.warning(
156 'Repository `%s` have incorrect scm_type, expected %s got %s',
182 'Repository `%s` have incorrect scm_type, expected %s got %s',
157 repo_name, db_repo.repo_type, scm_type)
183 repo_name, db_repo.repo_type, scm_type)
158 return False
184 return False
159
185
160 return is_valid_repo(repo_name, base_path, expect_scm=scm_type)
186 return is_valid_repo(repo_name, base_path, expect_scm=scm_type)
161
187
162 def valid_and_active_user(self, user):
188 def valid_and_active_user(self, user):
163 """
189 """
164 Checks if that user is not empty, and if it's actually object it checks
190 Checks if that user is not empty, and if it's actually object it checks
165 if he's active.
191 if he's active.
166
192
167 :param user: user object or None
193 :param user: user object or None
168 :return: boolean
194 :return: boolean
169 """
195 """
170 if user is None:
196 if user is None:
171 return False
197 return False
172
198
173 elif user.active:
199 elif user.active:
174 return True
200 return True
175
201
176 return False
202 return False
177
203
178 def _check_permission(self, action, user, repo_name, ip_addr=None):
204 def _check_permission(self, action, user, repo_name, ip_addr=None):
179 """
205 """
180 Checks permissions using action (push/pull) user and repository
206 Checks permissions using action (push/pull) user and repository
181 name
207 name
182
208
183 :param action: push or pull action
209 :param action: push or pull action
184 :param user: user instance
210 :param user: user instance
185 :param repo_name: repository name
211 :param repo_name: repository name
186 """
212 """
187 # check IP
213 # check IP
188 inherit = user.inherit_default_permissions
214 inherit = user.inherit_default_permissions
189 ip_allowed = AuthUser.check_ip_allowed(user.user_id, ip_addr,
215 ip_allowed = AuthUser.check_ip_allowed(user.user_id, ip_addr,
190 inherit_from_default=inherit)
216 inherit_from_default=inherit)
191 if ip_allowed:
217 if ip_allowed:
192 log.info('Access for IP:%s allowed', ip_addr)
218 log.info('Access for IP:%s allowed', ip_addr)
193 else:
219 else:
194 return False
220 return False
195
221
196 if action == 'push':
222 if action == 'push':
197 if not HasPermissionAnyMiddleware('repository.write',
223 if not HasPermissionAnyMiddleware('repository.write',
198 'repository.admin')(user,
224 'repository.admin')(user,
199 repo_name):
225 repo_name):
200 return False
226 return False
201
227
202 else:
228 else:
203 # any other action need at least read permission
229 # any other action need at least read permission
204 if not HasPermissionAnyMiddleware('repository.read',
230 if not HasPermissionAnyMiddleware('repository.read',
205 'repository.write',
231 'repository.write',
206 'repository.admin')(user,
232 'repository.admin')(user,
207 repo_name):
233 repo_name):
208 return False
234 return False
209
235
210 return True
236 return True
211
237
212 def _check_ssl(self, environ, start_response):
238 def _check_ssl(self, environ, start_response):
213 """
239 """
214 Checks the SSL check flag and returns False if SSL is not present
240 Checks the SSL check flag and returns False if SSL is not present
215 and required True otherwise
241 and required True otherwise
216 """
242 """
217 org_proto = environ['wsgi._org_proto']
243 org_proto = environ['wsgi._org_proto']
218 # check if we have SSL required ! if not it's a bad request !
244 # check if we have SSL required ! if not it's a bad request !
219 require_ssl = str2bool(self.repo_vcs_config.get('web', 'push_ssl'))
245 require_ssl = str2bool(self.repo_vcs_config.get('web', 'push_ssl'))
220 if require_ssl and org_proto == 'http':
246 if require_ssl and org_proto == 'http':
221 log.debug('proto is %s and SSL is required BAD REQUEST !',
247 log.debug('proto is %s and SSL is required BAD REQUEST !',
222 org_proto)
248 org_proto)
223 return False
249 return False
224 return True
250 return True
225
251
226 def __call__(self, environ, start_response):
252 def __call__(self, environ, start_response):
227 try:
253 try:
228 return self._handle_request(environ, start_response)
254 return self._handle_request(environ, start_response)
229 except Exception:
255 except Exception:
230 log.exception("Exception while handling request")
256 log.exception("Exception while handling request")
231 appenlight.track_exception(environ)
257 appenlight.track_exception(environ)
232 return HTTPInternalServerError()(environ, start_response)
258 return HTTPInternalServerError()(environ, start_response)
233 finally:
259 finally:
234 meta.Session.remove()
260 meta.Session.remove()
235
261
236 def _handle_request(self, environ, start_response):
262 def _handle_request(self, environ, start_response):
237
263
238 if not self._check_ssl(environ, start_response):
264 if not self._check_ssl(environ, start_response):
239 reason = ('SSL required, while RhodeCode was unable '
265 reason = ('SSL required, while RhodeCode was unable '
240 'to detect this as SSL request')
266 'to detect this as SSL request')
241 log.debug('User not allowed to proceed, %s', reason)
267 log.debug('User not allowed to proceed, %s', reason)
242 return HTTPNotAcceptable(reason)(environ, start_response)
268 return HTTPNotAcceptable(reason)(environ, start_response)
243
269
244 if not self.repo_name:
270 if not self.repo_name:
245 log.warning('Repository name is empty: %s', self.repo_name)
271 log.warning('Repository name is empty: %s', self.repo_name)
246 # failed to get repo name, we fail now
272 # failed to get repo name, we fail now
247 return HTTPNotFound()(environ, start_response)
273 return HTTPNotFound()(environ, start_response)
248 log.debug('Extracted repo name is %s', self.repo_name)
274 log.debug('Extracted repo name is %s', self.repo_name)
249
275
250 ip_addr = get_ip_addr(environ)
276 ip_addr = get_ip_addr(environ)
251 username = None
277 username = None
252
278
253 # skip passing error to error controller
279 # skip passing error to error controller
254 environ['pylons.status_code_redirect'] = True
280 environ['pylons.status_code_redirect'] = True
255
281
256 # ======================================================================
282 # ======================================================================
257 # GET ACTION PULL or PUSH
283 # GET ACTION PULL or PUSH
258 # ======================================================================
284 # ======================================================================
259 action = self._get_action(environ)
285 action = self._get_action(environ)
260
286
261 # ======================================================================
287 # ======================================================================
262 # CHECK ANONYMOUS PERMISSION
288 # CHECK ANONYMOUS PERMISSION
263 # ======================================================================
289 # ======================================================================
264 if action in ['pull', 'push']:
290 if action in ['pull', 'push']:
265 anonymous_user = User.get_default_user()
291 anonymous_user = User.get_default_user()
266 username = anonymous_user.username
292 username = anonymous_user.username
267 if anonymous_user.active:
293 if anonymous_user.active:
268 # ONLY check permissions if the user is activated
294 # ONLY check permissions if the user is activated
269 anonymous_perm = self._check_permission(
295 anonymous_perm = self._check_permission(
270 action, anonymous_user, self.repo_name, ip_addr)
296 action, anonymous_user, self.repo_name, ip_addr)
271 else:
297 else:
272 anonymous_perm = False
298 anonymous_perm = False
273
299
274 if not anonymous_user.active or not anonymous_perm:
300 if not anonymous_user.active or not anonymous_perm:
275 if not anonymous_user.active:
301 if not anonymous_user.active:
276 log.debug('Anonymous access is disabled, running '
302 log.debug('Anonymous access is disabled, running '
277 'authentication')
303 'authentication')
278
304
279 if not anonymous_perm:
305 if not anonymous_perm:
280 log.debug('Not enough credentials to access this '
306 log.debug('Not enough credentials to access this '
281 'repository as anonymous user')
307 'repository as anonymous user')
282
308
283 username = None
309 username = None
284 # ==============================================================
310 # ==============================================================
285 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
311 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
286 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
312 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
287 # ==============================================================
313 # ==============================================================
288
314
289 # try to auth based on environ, container auth methods
315 # try to auth based on environ, container auth methods
290 log.debug('Running PRE-AUTH for container based authentication')
316 log.debug('Running PRE-AUTH for container based authentication')
291 pre_auth = authenticate(
317 pre_auth = authenticate(
292 '', '', environ, VCS_TYPE, registry=self.registry)
318 '', '', environ, VCS_TYPE, registry=self.registry)
293 if pre_auth and pre_auth.get('username'):
319 if pre_auth and pre_auth.get('username'):
294 username = pre_auth['username']
320 username = pre_auth['username']
295 log.debug('PRE-AUTH got %s as username', username)
321 log.debug('PRE-AUTH got %s as username', username)
296
322
297 # If not authenticated by the container, running basic auth
323 # If not authenticated by the container, running basic auth
298 if not username:
324 if not username:
299 self.authenticate.realm = get_rhodecode_realm()
325 self.authenticate.realm = get_rhodecode_realm()
300
326
301 try:
327 try:
302 result = self.authenticate(environ)
328 result = self.authenticate(environ)
303 except (UserCreationError, NotAllowedToCreateUserError) as e:
329 except (UserCreationError, NotAllowedToCreateUserError) as e:
304 log.error(e)
330 log.error(e)
305 reason = safe_str(e)
331 reason = safe_str(e)
306 return HTTPNotAcceptable(reason)(environ, start_response)
332 return HTTPNotAcceptable(reason)(environ, start_response)
307
333
308 if isinstance(result, str):
334 if isinstance(result, str):
309 AUTH_TYPE.update(environ, 'basic')
335 AUTH_TYPE.update(environ, 'basic')
310 REMOTE_USER.update(environ, result)
336 REMOTE_USER.update(environ, result)
311 username = result
337 username = result
312 else:
338 else:
313 return result.wsgi_application(environ, start_response)
339 return result.wsgi_application(environ, start_response)
314
340
315 # ==============================================================
341 # ==============================================================
316 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
342 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
317 # ==============================================================
343 # ==============================================================
318 user = User.get_by_username(username)
344 user = User.get_by_username(username)
319 if not self.valid_and_active_user(user):
345 if not self.valid_and_active_user(user):
320 return HTTPForbidden()(environ, start_response)
346 return HTTPForbidden()(environ, start_response)
321 username = user.username
347 username = user.username
322 user.update_lastactivity()
348 user.update_lastactivity()
323 meta.Session().commit()
349 meta.Session().commit()
324
350
325 # check user attributes for password change flag
351 # check user attributes for password change flag
326 user_obj = user
352 user_obj = user
327 if user_obj and user_obj.username != User.DEFAULT_USER and \
353 if user_obj and user_obj.username != User.DEFAULT_USER and \
328 user_obj.user_data.get('force_password_change'):
354 user_obj.user_data.get('force_password_change'):
329 reason = 'password change required'
355 reason = 'password change required'
330 log.debug('User not allowed to authenticate, %s', reason)
356 log.debug('User not allowed to authenticate, %s', reason)
331 return HTTPNotAcceptable(reason)(environ, start_response)
357 return HTTPNotAcceptable(reason)(environ, start_response)
332
358
333 # check permissions for this repository
359 # check permissions for this repository
334 perm = self._check_permission(
360 perm = self._check_permission(
335 action, user, self.repo_name, ip_addr)
361 action, user, self.repo_name, ip_addr)
336 if not perm:
362 if not perm:
337 return HTTPForbidden()(environ, start_response)
363 return HTTPForbidden()(environ, start_response)
338
364
339 # extras are injected into UI object and later available
365 # extras are injected into UI object and later available
340 # in hooks executed by rhodecode
366 # in hooks executed by rhodecode
341 check_locking = _should_check_locking(environ.get('QUERY_STRING'))
367 check_locking = _should_check_locking(environ.get('QUERY_STRING'))
342 extras = vcs_operation_context(
368 extras = vcs_operation_context(
343 environ, repo_name=self.repo_name, username=username,
369 environ, repo_name=self.repo_name, username=username,
344 action=action, scm=self.SCM,
370 action=action, scm=self.SCM,
345 check_locking=check_locking)
371 check_locking=check_locking)
346
372
347 # ======================================================================
373 # ======================================================================
348 # REQUEST HANDLING
374 # REQUEST HANDLING
349 # ======================================================================
375 # ======================================================================
350 str_repo_name = safe_str(self.repo_name)
376 str_repo_name = safe_str(self.repo_name)
351 repo_path = os.path.join(
377 repo_path = os.path.join(
352 safe_str(self.basepath), safe_str(self.vcs_repo_name))
378 safe_str(self.basepath), safe_str(self.vcs_repo_name))
353 log.debug('Repository path is %s', repo_path)
379 log.debug('Repository path is %s', repo_path)
354
380
355 fix_PATH()
381 fix_PATH()
356
382
357 log.info(
383 log.info(
358 '%s action on %s repo "%s" by "%s" from %s',
384 '%s action on %s repo "%s" by "%s" from %s',
359 action, self.SCM, str_repo_name, safe_str(username), ip_addr)
385 action, self.SCM, str_repo_name, safe_str(username), ip_addr)
360
386
361 return self._generate_vcs_response(
387 return self._generate_vcs_response(
362 environ, start_response, repo_path, self.url_repo_name, extras, action)
388 environ, start_response, repo_path, self.url_repo_name, extras, action)
363
389
364 @initialize_generator
390 @initialize_generator
365 def _generate_vcs_response(
391 def _generate_vcs_response(
366 self, environ, start_response, repo_path, repo_name, extras,
392 self, environ, start_response, repo_path, repo_name, extras,
367 action):
393 action):
368 """
394 """
369 Returns a generator for the response content.
395 Returns a generator for the response content.
370
396
371 This method is implemented as a generator, so that it can trigger
397 This method is implemented as a generator, so that it can trigger
372 the cache validation after all content sent back to the client. It
398 the cache validation after all content sent back to the client. It
373 also handles the locking exceptions which will be triggered when
399 also handles the locking exceptions which will be triggered when
374 the first chunk is produced by the underlying WSGI application.
400 the first chunk is produced by the underlying WSGI application.
375 """
401 """
376 callback_daemon, extras = self._prepare_callback_daemon(extras)
402 callback_daemon, extras = self._prepare_callback_daemon(extras)
377 config = self._create_config(extras, self.acl_repo_name)
403 config = self._create_config(extras, self.acl_repo_name)
378 log.debug('HOOKS extras is %s', extras)
404 log.debug('HOOKS extras is %s', extras)
379 app = self._create_wsgi_app(repo_path, repo_name, config)
405 app = self._create_wsgi_app(repo_path, repo_name, config)
380
406
381 try:
407 try:
382 with callback_daemon:
408 with callback_daemon:
383 try:
409 try:
384 response = app(environ, start_response)
410 response = app(environ, start_response)
385 finally:
411 finally:
386 # This statement works together with the decorator
412 # This statement works together with the decorator
387 # "initialize_generator" above. The decorator ensures that
413 # "initialize_generator" above. The decorator ensures that
388 # we hit the first yield statement before the generator is
414 # we hit the first yield statement before the generator is
389 # returned back to the WSGI server. This is needed to
415 # returned back to the WSGI server. This is needed to
390 # ensure that the call to "app" above triggers the
416 # ensure that the call to "app" above triggers the
391 # needed callback to "start_response" before the
417 # needed callback to "start_response" before the
392 # generator is actually used.
418 # generator is actually used.
393 yield "__init__"
419 yield "__init__"
394
420
395 for chunk in response:
421 for chunk in response:
396 yield chunk
422 yield chunk
397 except Exception as exc:
423 except Exception as exc:
398 # TODO: johbo: Improve "translating" back the exception.
424 # TODO: johbo: Improve "translating" back the exception.
399 if getattr(exc, '_vcs_kind', None) == 'repo_locked':
425 if getattr(exc, '_vcs_kind', None) == 'repo_locked':
400 exc = HTTPLockedRC(*exc.args)
426 exc = HTTPLockedRC(*exc.args)
401 _code = rhodecode.CONFIG.get('lock_ret_code')
427 _code = rhodecode.CONFIG.get('lock_ret_code')
402 log.debug('Repository LOCKED ret code %s!', (_code,))
428 log.debug('Repository LOCKED ret code %s!', (_code,))
403 elif getattr(exc, '_vcs_kind', None) == 'requirement':
429 elif getattr(exc, '_vcs_kind', None) == 'requirement':
404 log.debug(
430 log.debug(
405 'Repository requires features unknown to this Mercurial')
431 'Repository requires features unknown to this Mercurial')
406 exc = HTTPRequirementError(*exc.args)
432 exc = HTTPRequirementError(*exc.args)
407 else:
433 else:
408 raise
434 raise
409
435
410 for chunk in exc(environ, start_response):
436 for chunk in exc(environ, start_response):
411 yield chunk
437 yield chunk
412 finally:
438 finally:
413 # invalidate cache on push
439 # invalidate cache on push
414 try:
440 try:
415 if action == 'push':
441 if action == 'push':
416 self._invalidate_cache(repo_name)
442 self._invalidate_cache(repo_name)
417 finally:
443 finally:
418 meta.Session.remove()
444 meta.Session.remove()
419
445
420 def _get_repository_name(self, environ):
446 def _get_repository_name(self, environ):
421 """Get repository name out of the environmnent
447 """Get repository name out of the environmnent
422
448
423 :param environ: WSGI environment
449 :param environ: WSGI environment
424 """
450 """
425 raise NotImplementedError()
451 raise NotImplementedError()
426
452
427 def _get_action(self, environ):
453 def _get_action(self, environ):
428 """Map request commands into a pull or push command.
454 """Map request commands into a pull or push command.
429
455
430 :param environ: WSGI environment
456 :param environ: WSGI environment
431 """
457 """
432 raise NotImplementedError()
458 raise NotImplementedError()
433
459
434 def _create_wsgi_app(self, repo_path, repo_name, config):
460 def _create_wsgi_app(self, repo_path, repo_name, config):
435 """Return the WSGI app that will finally handle the request."""
461 """Return the WSGI app that will finally handle the request."""
436 raise NotImplementedError()
462 raise NotImplementedError()
437
463
438 def _create_config(self, extras, repo_name):
464 def _create_config(self, extras, repo_name):
439 """Create a Pyro safe config representation."""
465 """Create a Pyro safe config representation."""
440 raise NotImplementedError()
466 raise NotImplementedError()
441
467
442 def _prepare_callback_daemon(self, extras):
468 def _prepare_callback_daemon(self, extras):
443 return prepare_callback_daemon(
469 return prepare_callback_daemon(
444 extras, protocol=vcs_settings.HOOKS_PROTOCOL,
470 extras, protocol=vcs_settings.HOOKS_PROTOCOL,
445 use_direct_calls=vcs_settings.HOOKS_DIRECT_CALLS)
471 use_direct_calls=vcs_settings.HOOKS_DIRECT_CALLS)
446
472
447
473
448 def _should_check_locking(query_string):
474 def _should_check_locking(query_string):
449 # this is kind of hacky, but due to how mercurial handles client-server
475 # this is kind of hacky, but due to how mercurial handles client-server
450 # server see all operation on commit; bookmarks, phases and
476 # server see all operation on commit; bookmarks, phases and
451 # obsolescence marker in different transaction, we don't want to check
477 # obsolescence marker in different transaction, we don't want to check
452 # locking on those
478 # locking on those
453 return query_string not in ['cmd=listkeys']
479 return query_string not in ['cmd=listkeys']
@@ -1,226 +1,211 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2010-2016 RhodeCode GmbH
3 # Copyright (C) 2010-2016 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import gzip
21 import gzip
22 import re
22 import re
23 import shutil
23 import shutil
24 import logging
24 import logging
25 import tempfile
25 import tempfile
26 import urlparse
26 import urlparse
27
27
28 from webob.exc import HTTPNotFound
28 from webob.exc import HTTPNotFound
29
29
30 import rhodecode
30 import rhodecode
31 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
31 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
32 from rhodecode.lib.middleware.simplegit import SimpleGit, GIT_PROTO_PAT
32 from rhodecode.lib.middleware.simplegit import SimpleGit, GIT_PROTO_PAT
33 from rhodecode.lib.middleware.simplehg import SimpleHg
33 from rhodecode.lib.middleware.simplehg import SimpleHg
34 from rhodecode.lib.middleware.simplesvn import SimpleSvn
34 from rhodecode.lib.middleware.simplesvn import SimpleSvn
35 from rhodecode.model.settings import VcsSettingsModel
35 from rhodecode.model.settings import VcsSettingsModel
36
36
37 log = logging.getLogger(__name__)
37 log = logging.getLogger(__name__)
38
38
39
39
40 def is_git(environ):
40 def is_git(environ):
41 """
41 """
42 Returns True if requests should be handled by GIT wsgi middleware
42 Returns True if requests should be handled by GIT wsgi middleware
43 """
43 """
44 is_git_path = GIT_PROTO_PAT.match(environ['PATH_INFO'])
44 is_git_path = GIT_PROTO_PAT.match(environ['PATH_INFO'])
45 log.debug(
45 log.debug(
46 'request path: `%s` detected as GIT PROTOCOL %s', environ['PATH_INFO'],
46 'request path: `%s` detected as GIT PROTOCOL %s', environ['PATH_INFO'],
47 is_git_path is not None)
47 is_git_path is not None)
48
48
49 return is_git_path
49 return is_git_path
50
50
51
51
52 def is_hg(environ):
52 def is_hg(environ):
53 """
53 """
54 Returns True if requests target is mercurial server - header
54 Returns True if requests target is mercurial server - header
55 ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
55 ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
56 """
56 """
57 is_hg_path = False
57 is_hg_path = False
58
58
59 http_accept = environ.get('HTTP_ACCEPT')
59 http_accept = environ.get('HTTP_ACCEPT')
60
60
61 if http_accept and http_accept.startswith('application/mercurial'):
61 if http_accept and http_accept.startswith('application/mercurial'):
62 query = urlparse.parse_qs(environ['QUERY_STRING'])
62 query = urlparse.parse_qs(environ['QUERY_STRING'])
63 if 'cmd' in query:
63 if 'cmd' in query:
64 is_hg_path = True
64 is_hg_path = True
65
65
66 log.debug(
66 log.debug(
67 'request path: `%s` detected as HG PROTOCOL %s', environ['PATH_INFO'],
67 'request path: `%s` detected as HG PROTOCOL %s', environ['PATH_INFO'],
68 is_hg_path)
68 is_hg_path)
69
69
70 return is_hg_path
70 return is_hg_path
71
71
72
72
73 def is_svn(environ):
73 def is_svn(environ):
74 """
74 """
75 Returns True if requests target is Subversion server
75 Returns True if requests target is Subversion server
76 """
76 """
77 http_dav = environ.get('HTTP_DAV', '')
77 http_dav = environ.get('HTTP_DAV', '')
78 magic_path_segment = rhodecode.CONFIG.get(
78 magic_path_segment = rhodecode.CONFIG.get(
79 'rhodecode_subversion_magic_path', '/!svn')
79 'rhodecode_subversion_magic_path', '/!svn')
80 is_svn_path = (
80 is_svn_path = (
81 'subversion' in http_dav or
81 'subversion' in http_dav or
82 magic_path_segment in environ['PATH_INFO'])
82 magic_path_segment in environ['PATH_INFO'])
83 log.debug(
83 log.debug(
84 'request path: `%s` detected as SVN PROTOCOL %s', environ['PATH_INFO'],
84 'request path: `%s` detected as SVN PROTOCOL %s', environ['PATH_INFO'],
85 is_svn_path)
85 is_svn_path)
86
86
87 return is_svn_path
87 return is_svn_path
88
88
89
89
90 class GunzipMiddleware(object):
90 class GunzipMiddleware(object):
91 """
91 """
92 WSGI middleware that unzips gzip-encoded requests before
92 WSGI middleware that unzips gzip-encoded requests before
93 passing on to the underlying application.
93 passing on to the underlying application.
94 """
94 """
95
95
96 def __init__(self, application):
96 def __init__(self, application):
97 self.app = application
97 self.app = application
98
98
99 def __call__(self, environ, start_response):
99 def __call__(self, environ, start_response):
100 accepts_encoding_header = environ.get('HTTP_CONTENT_ENCODING', b'')
100 accepts_encoding_header = environ.get('HTTP_CONTENT_ENCODING', b'')
101
101
102 if b'gzip' in accepts_encoding_header:
102 if b'gzip' in accepts_encoding_header:
103 log.debug('gzip detected, now running gunzip wrapper')
103 log.debug('gzip detected, now running gunzip wrapper')
104 wsgi_input = environ['wsgi.input']
104 wsgi_input = environ['wsgi.input']
105
105
106 if not hasattr(environ['wsgi.input'], 'seek'):
106 if not hasattr(environ['wsgi.input'], 'seek'):
107 # The gzip implementation in the standard library of Python 2.x
107 # The gzip implementation in the standard library of Python 2.x
108 # requires the '.seek()' and '.tell()' methods to be available
108 # requires the '.seek()' and '.tell()' methods to be available
109 # on the input stream. Read the data into a temporary file to
109 # on the input stream. Read the data into a temporary file to
110 # work around this limitation.
110 # work around this limitation.
111
111
112 wsgi_input = tempfile.SpooledTemporaryFile(64 * 1024 * 1024)
112 wsgi_input = tempfile.SpooledTemporaryFile(64 * 1024 * 1024)
113 shutil.copyfileobj(environ['wsgi.input'], wsgi_input)
113 shutil.copyfileobj(environ['wsgi.input'], wsgi_input)
114 wsgi_input.seek(0)
114 wsgi_input.seek(0)
115
115
116 environ['wsgi.input'] = gzip.GzipFile(fileobj=wsgi_input, mode='r')
116 environ['wsgi.input'] = gzip.GzipFile(fileobj=wsgi_input, mode='r')
117 # since we "Ungzipped" the content we say now it's no longer gzip
117 # since we "Ungzipped" the content we say now it's no longer gzip
118 # content encoding
118 # content encoding
119 del environ['HTTP_CONTENT_ENCODING']
119 del environ['HTTP_CONTENT_ENCODING']
120
120
121 # content length has changes ? or i'm not sure
121 # content length has changes ? or i'm not sure
122 if 'CONTENT_LENGTH' in environ:
122 if 'CONTENT_LENGTH' in environ:
123 del environ['CONTENT_LENGTH']
123 del environ['CONTENT_LENGTH']
124 else:
124 else:
125 log.debug('content not gzipped, gzipMiddleware passing '
125 log.debug('content not gzipped, gzipMiddleware passing '
126 'request further')
126 'request further')
127 return self.app(environ, start_response)
127 return self.app(environ, start_response)
128
128
129
129
130 class VCSMiddleware(object):
130 class VCSMiddleware(object):
131
131
132 def __init__(self, app, config, appenlight_client, registry):
132 def __init__(self, app, config, appenlight_client, registry):
133 self.application = app
133 self.application = app
134 self.config = config
134 self.config = config
135 self.appenlight_client = appenlight_client
135 self.appenlight_client = appenlight_client
136 self.registry = registry
136 self.registry = registry
137 self.use_gzip = True
137 self.use_gzip = True
138 # order in which we check the middlewares, based on vcs.backends config
138 # order in which we check the middlewares, based on vcs.backends config
139 self.check_middlewares = config['vcs.backends']
139 self.check_middlewares = config['vcs.backends']
140 self.checks = {
140 self.checks = {
141 'hg': (is_hg, SimpleHg),
141 'hg': (is_hg, SimpleHg),
142 'git': (is_git, SimpleGit),
142 'git': (is_git, SimpleGit),
143 'svn': (is_svn, SimpleSvn),
143 'svn': (is_svn, SimpleSvn),
144 }
144 }
145
145
146 def vcs_config(self, repo_name=None):
146 def vcs_config(self, repo_name=None):
147 """
147 """
148 returns serialized VcsSettings
148 returns serialized VcsSettings
149 """
149 """
150 return VcsSettingsModel(repo=repo_name).get_ui_settings_as_config_obj()
150 return VcsSettingsModel(repo=repo_name).get_ui_settings_as_config_obj()
151
151
152 def wrap_in_gzip_if_enabled(self, app, config):
152 def wrap_in_gzip_if_enabled(self, app, config):
153 if self.use_gzip:
153 if self.use_gzip:
154 app = GunzipMiddleware(app)
154 app = GunzipMiddleware(app)
155 return app
155 return app
156
156
157 def _get_handler_app(self, environ):
157 def _get_handler_app(self, environ):
158 app = None
158 app = None
159 log.debug('Checking vcs types in order: %r', self.check_middlewares)
159 log.debug('Checking vcs types in order: %r', self.check_middlewares)
160 for vcs_type in self.check_middlewares:
160 for vcs_type in self.check_middlewares:
161 vcs_check, handler = self.checks[vcs_type]
161 vcs_check, handler = self.checks[vcs_type]
162 if vcs_check(environ):
162 if vcs_check(environ):
163 log.debug(
163 log.debug(
164 'Found VCS Middleware to handle the request %s', handler)
164 'Found VCS Middleware to handle the request %s', handler)
165 app = handler(self.application, self.config, self.registry)
165 app = handler(self.application, self.config, self.registry)
166 break
166 break
167
167
168 return app
168 return app
169
169
170 def __call__(self, environ, start_response):
170 def __call__(self, environ, start_response):
171 # check if we handle one of interesting protocols, optionally extract
171 # check if we handle one of interesting protocols, optionally extract
172 # specific vcsSettings and allow changes of how things are wrapped
172 # specific vcsSettings and allow changes of how things are wrapped
173 vcs_handler = self._get_handler_app(environ)
173 vcs_handler = self._get_handler_app(environ)
174 if vcs_handler:
174 if vcs_handler:
175 # translate the _REPO_ID into real repo NAME for usage
175 # translate the _REPO_ID into real repo NAME for usage
176 # in middleware
176 # in middleware
177 environ['PATH_INFO'] = vcs_handler._get_by_id(environ['PATH_INFO'])
177 environ['PATH_INFO'] = vcs_handler._get_by_id(environ['PATH_INFO'])
178 repo_name = vcs_handler._get_repository_name(environ)
179
178
180 acl_repo_name = repo_name
179 # Set repo names for permission checks, vcs and web interaction.
181 vcs_repo_name = repo_name
180 vcs_handler.set_repo_names(environ)
182 url_repo_name = repo_name
183 pr_id = None
184
185 pr_regex = re.compile(
186 '(?P<base_name>(?:[\w-]+)(?:/[\w-]+)*)/'
187 '(?P<repo_name>[\w-]+)'
188 '/pull-request/(?P<pr_id>\d+)/repository')
189 match = pr_regex.match(repo_name)
190 if match:
191 match_dict = match.groupdict()
192 pr_id = match_dict.get('pr_id')
193 acl_repo_name = '{base_name}/{repo_name}'.format(**match_dict)
194 vcs_repo_name = '{base_name}/.__shadow_{repo_name}_pr-{pr_id}'.format(
195 **match_dict)
196
197 log.debug('repo_names %s', {
181 log.debug('repo_names %s', {
198 'acl_repo_name': acl_repo_name,
182 'acl_repo_name': vcs_handler.acl_repo_name,
199 'vcs_repo_name': vcs_repo_name,
183 'vcs_repo_name': vcs_handler.vcs_repo_name,
200 'url_repo_name': url_repo_name,
184 'url_repo_name': vcs_handler.url_repo_name,
201 })
185 })
202 log.debug('pull_request %s', pr_id)
186 log.debug('pull_request %s', vcs_handler.pr_id)
203
187
204 # check for type, presence in database and on filesystem
188 # check for type, presence in database and on filesystem
205 if not vcs_handler.is_valid_and_existing_repo(
189 if not vcs_handler.is_valid_and_existing_repo(
206 acl_repo_name, vcs_handler.basepath, vcs_handler.SCM):
190 vcs_handler.acl_repo_name,
191 vcs_handler.basepath,
192 vcs_handler.SCM):
207 return HTTPNotFound()(environ, start_response)
193 return HTTPNotFound()(environ, start_response)
208
194
209 # TODO: johbo: Needed for the Pyro4 backend and Mercurial only.
195 # TODO: johbo: Needed for the Pyro4 backend and Mercurial only.
210 # Remove once we fully switched to the HTTP backend.
196 # Remove once we fully switched to the HTTP backend.
211 environ['REPO_NAME'] = url_repo_name
197 environ['REPO_NAME'] = vcs_handler.url_repo_name
212
198
213 # register repo_name and it's config back to the handler
199 # register repo config back to the handler
214 vcs_handler.acl_repo_name = acl_repo_name
200 vcs_handler.repo_vcs_config = self.vcs_config(
215 vcs_handler.url_repo_name = url_repo_name
201 vcs_handler.acl_repo_name)
216 vcs_handler.vcs_repo_name = vcs_repo_name
217 vcs_handler.pr_id = pr_id
218 vcs_handler.repo_vcs_config = self.vcs_config(acl_repo_name)
219
202
203 # Wrap handler in middlewares if they are enabled.
220 vcs_handler = self.wrap_in_gzip_if_enabled(
204 vcs_handler = self.wrap_in_gzip_if_enabled(
221 vcs_handler, self.config)
205 vcs_handler, self.config)
222 vcs_handler, _ = wrap_in_appenlight_if_enabled(
206 vcs_handler, _ = wrap_in_appenlight_if_enabled(
223 vcs_handler, self.config, self.appenlight_client)
207 vcs_handler, self.config, self.appenlight_client)
208
224 return vcs_handler(environ, start_response)
209 return vcs_handler(environ, start_response)
225
210
226 return self.application(environ, start_response)
211 return self.application(environ, start_response)
General Comments 0
You need to be logged in to leave comments. Login now