##// END OF EJS Templates
vcs: Clean up the shadow-repo-expose code and make some nicer comments.
Martin Bornhold -
r904:8e8700f6 default
parent child Browse files
Show More
@@ -1,511 +1,523 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2014-2016 RhodeCode GmbH
3 # Copyright (C) 2014-2016 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 """
21 """
22 SimpleVCS middleware for handling protocol request (push/clone etc.)
22 SimpleVCS middleware for handling protocol request (push/clone etc.)
23 It's implemented with basic auth function
23 It's implemented with basic auth function
24 """
24 """
25
25
26 import os
26 import os
27 import logging
27 import logging
28 import importlib
28 import importlib
29 import re
29 import re
30 from functools import wraps
30 from functools import wraps
31
31
32 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
32 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
33 from webob.exc import (
33 from webob.exc import (
34 HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
34 HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
35
35
36 import rhodecode
36 import rhodecode
37 from rhodecode.authentication.base import authenticate, VCS_TYPE
37 from rhodecode.authentication.base import authenticate, VCS_TYPE
38 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
38 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
39 from rhodecode.lib.base import BasicAuth, get_ip_addr, vcs_operation_context
39 from rhodecode.lib.base import BasicAuth, get_ip_addr, vcs_operation_context
40 from rhodecode.lib.exceptions import (
40 from rhodecode.lib.exceptions import (
41 HTTPLockedRC, HTTPRequirementError, UserCreationError,
41 HTTPLockedRC, HTTPRequirementError, UserCreationError,
42 NotAllowedToCreateUserError)
42 NotAllowedToCreateUserError)
43 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
43 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
44 from rhodecode.lib.middleware import appenlight
44 from rhodecode.lib.middleware import appenlight
45 from rhodecode.lib.middleware.utils import scm_app
45 from rhodecode.lib.middleware.utils import scm_app
46 from rhodecode.lib.utils import (
46 from rhodecode.lib.utils import (
47 is_valid_repo, get_rhodecode_realm, get_rhodecode_base_path, SLUG_RE)
47 is_valid_repo, get_rhodecode_realm, get_rhodecode_base_path, SLUG_RE)
48 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool
48 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool, safe_unicode
49 from rhodecode.lib.vcs.conf import settings as vcs_settings
49 from rhodecode.lib.vcs.conf import settings as vcs_settings
50 from rhodecode.lib.vcs.backends import base
50 from rhodecode.lib.vcs.backends import base
51 from rhodecode.model import meta
51 from rhodecode.model import meta
52 from rhodecode.model.db import User, Repository, PullRequest
52 from rhodecode.model.db import User, Repository, PullRequest
53 from rhodecode.model.scm import ScmModel
53 from rhodecode.model.scm import ScmModel
54 from rhodecode.model.pull_request import PullRequestModel
54 from rhodecode.model.pull_request import PullRequestModel
55
55
56
56
57 log = logging.getLogger(__name__)
57 log = logging.getLogger(__name__)
58
58
59
59
60 def initialize_generator(factory):
60 def initialize_generator(factory):
61 """
61 """
62 Initializes the returned generator by draining its first element.
62 Initializes the returned generator by draining its first element.
63
63
64 This can be used to give a generator an initializer, which is the code
64 This can be used to give a generator an initializer, which is the code
65 up to the first yield statement. This decorator enforces that the first
65 up to the first yield statement. This decorator enforces that the first
66 produced element has the value ``"__init__"`` to make its special
66 produced element has the value ``"__init__"`` to make its special
67 purpose very explicit in the using code.
67 purpose very explicit in the using code.
68 """
68 """
69
69
70 @wraps(factory)
70 @wraps(factory)
71 def wrapper(*args, **kwargs):
71 def wrapper(*args, **kwargs):
72 gen = factory(*args, **kwargs)
72 gen = factory(*args, **kwargs)
73 try:
73 try:
74 init = gen.next()
74 init = gen.next()
75 except StopIteration:
75 except StopIteration:
76 raise ValueError('Generator must yield at least one element.')
76 raise ValueError('Generator must yield at least one element.')
77 if init != "__init__":
77 if init != "__init__":
78 raise ValueError('First yielded element must be "__init__".')
78 raise ValueError('First yielded element must be "__init__".')
79 return gen
79 return gen
80 return wrapper
80 return wrapper
81
81
82
82
83 class SimpleVCS(object):
83 class SimpleVCS(object):
84 """Common functionality for SCM HTTP handlers."""
84 """Common functionality for SCM HTTP handlers."""
85
85
86 SCM = 'unknown'
86 SCM = 'unknown'
87
87
88 acl_repo_name = None
88 acl_repo_name = None
89 url_repo_name = None
89 url_repo_name = None
90 vcs_repo_name = None
90 vcs_repo_name = None
91
91
92 # We have to handle requests to shadow repositories different than requests
92 # We have to handle requests to shadow repositories different than requests
93 # to normal repositories. Therefore we have to distinguish them. To do this
93 # to normal repositories. Therefore we have to distinguish them. To do this
94 # we use this regex which will match only on URLs pointing to shadow
94 # we use this regex which will match only on URLs pointing to shadow
95 # repositories.
95 # repositories.
96 shadow_repo_re = re.compile(
96 shadow_repo_re = re.compile(
97 '(?P<groups>(?:{slug_pat})(?:/{slug_pat})*)' # repo groups
97 '(?P<groups>(?:{slug_pat})(?:/{slug_pat})*)' # repo groups
98 '/(?P<target>{slug_pat})' # target repo
98 '/(?P<target>{slug_pat})' # target repo
99 '/pull-request/(?P<pr_id>\d+)' # pull request
99 '/pull-request/(?P<pr_id>\d+)' # pull request
100 '/repository$' # shadow repo
100 '/repository$' # shadow repo
101 .format(slug_pat=SLUG_RE.pattern))
101 .format(slug_pat=SLUG_RE.pattern))
102
102
103 def __init__(self, application, config, registry):
103 def __init__(self, application, config, registry):
104 self.registry = registry
104 self.registry = registry
105 self.application = application
105 self.application = application
106 self.config = config
106 self.config = config
107 # re-populated by specialized middleware
107 # re-populated by specialized middleware
108 self.repo_vcs_config = base.Config()
108 self.repo_vcs_config = base.Config()
109
109
110 # base path of repo locations
110 # base path of repo locations
111 self.basepath = get_rhodecode_base_path()
111 self.basepath = get_rhodecode_base_path()
112 # authenticate this VCS request using authfunc
112 # authenticate this VCS request using authfunc
113 auth_ret_code_detection = \
113 auth_ret_code_detection = \
114 str2bool(self.config.get('auth_ret_code_detection', False))
114 str2bool(self.config.get('auth_ret_code_detection', False))
115 self.authenticate = BasicAuth(
115 self.authenticate = BasicAuth(
116 '', authenticate, registry, config.get('auth_ret_code'),
116 '', authenticate, registry, config.get('auth_ret_code'),
117 auth_ret_code_detection)
117 auth_ret_code_detection)
118 self.ip_addr = '0.0.0.0'
118 self.ip_addr = '0.0.0.0'
119
119
120 def set_repo_names(self, environ):
120 def set_repo_names(self, environ):
121 """
121 """
122 This will populate the attributes acl_repo_name, url_repo_name,
122 This will populate the attributes acl_repo_name, url_repo_name,
123 vcs_repo_name and is_shadow_repo on the current instance.
123 vcs_repo_name and is_shadow_repo. In case of requests to normal (non
124 """
124 shadow) repositories all names are equal. In case of requests to a
125 # Get url repo name from environment.
125 shadow repository the acl-name points to the target repo of the pull
126 self.url_repo_name = self._get_repository_name(environ)
126 request and the vcs-name points to the shadow repo file system path.
127 The url-name is always the URL used by the vcs client program.
127
128
128 # Check if this is a request to a shadow repository. In case of a
129 Example in case of a shadow repo:
129 # shadow repo set vcs_repo_name to the file system path pointing to the
130 acl_repo_name = RepoGroup/MyRepo
130 # shadow repo. And set acl_repo_name to the pull request target repo
131 url_repo_name = RepoGroup/MyRepo/pull-request/3/repository
131 # because we use the target repo for permission checks. Otherwise all
132 vcs_repo_name = /repo/base/path/RepoGroup/.__shadow_MyRepo_pr-3'
132 # names are equal.
133 """
134 # First we set the repo name from URL for all attributes. This is the
135 # default if handling normal (non shadow) repo requests.
136 self.url_repo_name = self._get_repository_name(environ)
137 self.acl_repo_name = self.vcs_repo_name = self.url_repo_name
138 self.is_shadow_repo = False
139
140 # Check if this is a request to a shadow repository.
133 match = self.shadow_repo_re.match(self.url_repo_name)
141 match = self.shadow_repo_re.match(self.url_repo_name)
134 # TODO: martinb: Think about checking the target repo from PR against
135 # the part in the URL. Otherwise we only rely on the PR id in the URL
136 # and the variable parts can be anything. This will lead to 500 errors
137 # from the VCSServer.
138 if match:
142 if match:
139 # Get pull request instance.
140 match_dict = match.groupdict()
143 match_dict = match.groupdict()
141 pr_id = match_dict['pr_id']
144
142 pull_request = PullRequest.get(pr_id)
145 # Build acl repo name from regex match.
146 acl_repo_name = safe_unicode(
147 '{groups}/{target}'.format(**match_dict))
143
148
149 # Retrieve pull request instance by ID from regex match.
150 pull_request = PullRequest.get(match_dict['pr_id'])
151
152 # Only proceed if we got a pull request and if acl repo name from
153 # URL equals the target repo name of the pull request.
154 if pull_request and acl_repo_name == pull_request.target_repo.repo_name:
144 # Get file system path to shadow repository.
155 # Get file system path to shadow repository.
145 workspace_id = PullRequestModel()._workspace_id(pull_request)
156 workspace_id = PullRequestModel()._workspace_id(pull_request)
146 target_vcs = pull_request.target_repo.scm_instance()
157 target_vcs = pull_request.target_repo.scm_instance()
147 vcs_repo_name = target_vcs._get_shadow_repository_path(
158 vcs_repo_name = target_vcs._get_shadow_repository_path(
148 workspace_id)
159 workspace_id)
149
160
150 # Store names for later usage.
161 # Store names for later usage.
151 self.vcs_repo_name = vcs_repo_name
162 self.vcs_repo_name = vcs_repo_name
152 self.acl_repo_name = pull_request.target_repo.repo_name
163 self.acl_repo_name = acl_repo_name
153 self.is_shadow_repo = True
164 self.is_shadow_repo = True
154 else:
165
155 # All names are equal for normal (non shadow) repositories.
166 log.debug('Repository names: %s', {
156 self.acl_repo_name = self.url_repo_name
167 'acl_repo_name': self.acl_repo_name,
157 self.vcs_repo_name = self.url_repo_name
168 'url_repo_name': self.url_repo_name,
158 self.is_shadow_repo = False
169 'vcs_repo_name': self.vcs_repo_name,
170 })
159
171
160 @property
172 @property
161 def scm_app(self):
173 def scm_app(self):
162 custom_implementation = self.config.get('vcs.scm_app_implementation')
174 custom_implementation = self.config.get('vcs.scm_app_implementation')
163 if custom_implementation and custom_implementation != 'pyro4':
175 if custom_implementation and custom_implementation != 'pyro4':
164 log.info(
176 log.info(
165 "Using custom implementation of scm_app: %s",
177 "Using custom implementation of scm_app: %s",
166 custom_implementation)
178 custom_implementation)
167 scm_app_impl = importlib.import_module(custom_implementation)
179 scm_app_impl = importlib.import_module(custom_implementation)
168 else:
180 else:
169 scm_app_impl = scm_app
181 scm_app_impl = scm_app
170 return scm_app_impl
182 return scm_app_impl
171
183
172 def _get_by_id(self, repo_name):
184 def _get_by_id(self, repo_name):
173 """
185 """
174 Gets a special pattern _<ID> from clone url and tries to replace it
186 Gets a special pattern _<ID> from clone url and tries to replace it
175 with a repository_name for support of _<ID> non changeable urls
187 with a repository_name for support of _<ID> non changeable urls
176 """
188 """
177
189
178 data = repo_name.split('/')
190 data = repo_name.split('/')
179 if len(data) >= 2:
191 if len(data) >= 2:
180 from rhodecode.model.repo import RepoModel
192 from rhodecode.model.repo import RepoModel
181 by_id_match = RepoModel().get_repo_by_id(repo_name)
193 by_id_match = RepoModel().get_repo_by_id(repo_name)
182 if by_id_match:
194 if by_id_match:
183 data[1] = by_id_match.repo_name
195 data[1] = by_id_match.repo_name
184
196
185 return safe_str('/'.join(data))
197 return safe_str('/'.join(data))
186
198
187 def _invalidate_cache(self, repo_name):
199 def _invalidate_cache(self, repo_name):
188 """
200 """
189 Set's cache for this repository for invalidation on next access
201 Set's cache for this repository for invalidation on next access
190
202
191 :param repo_name: full repo name, also a cache key
203 :param repo_name: full repo name, also a cache key
192 """
204 """
193 ScmModel().mark_for_invalidation(repo_name)
205 ScmModel().mark_for_invalidation(repo_name)
194
206
195 def is_valid_and_existing_repo(self, repo_name, base_path, scm_type):
207 def is_valid_and_existing_repo(self, repo_name, base_path, scm_type):
196 db_repo = Repository.get_by_repo_name(repo_name)
208 db_repo = Repository.get_by_repo_name(repo_name)
197 if not db_repo:
209 if not db_repo:
198 log.debug('Repository `%s` not found inside the database.',
210 log.debug('Repository `%s` not found inside the database.',
199 repo_name)
211 repo_name)
200 return False
212 return False
201
213
202 if db_repo.repo_type != scm_type:
214 if db_repo.repo_type != scm_type:
203 log.warning(
215 log.warning(
204 'Repository `%s` have incorrect scm_type, expected %s got %s',
216 'Repository `%s` have incorrect scm_type, expected %s got %s',
205 repo_name, db_repo.repo_type, scm_type)
217 repo_name, db_repo.repo_type, scm_type)
206 return False
218 return False
207
219
208 return is_valid_repo(repo_name, base_path, expect_scm=scm_type)
220 return is_valid_repo(repo_name, base_path, expect_scm=scm_type)
209
221
210 def valid_and_active_user(self, user):
222 def valid_and_active_user(self, user):
211 """
223 """
212 Checks if that user is not empty, and if it's actually object it checks
224 Checks if that user is not empty, and if it's actually object it checks
213 if he's active.
225 if he's active.
214
226
215 :param user: user object or None
227 :param user: user object or None
216 :return: boolean
228 :return: boolean
217 """
229 """
218 if user is None:
230 if user is None:
219 return False
231 return False
220
232
221 elif user.active:
233 elif user.active:
222 return True
234 return True
223
235
224 return False
236 return False
225
237
226 def _check_permission(self, action, user, repo_name, ip_addr=None):
238 def _check_permission(self, action, user, repo_name, ip_addr=None):
227 """
239 """
228 Checks permissions using action (push/pull) user and repository
240 Checks permissions using action (push/pull) user and repository
229 name
241 name
230
242
231 :param action: push or pull action
243 :param action: push or pull action
232 :param user: user instance
244 :param user: user instance
233 :param repo_name: repository name
245 :param repo_name: repository name
234 """
246 """
235 # check IP
247 # check IP
236 inherit = user.inherit_default_permissions
248 inherit = user.inherit_default_permissions
237 ip_allowed = AuthUser.check_ip_allowed(user.user_id, ip_addr,
249 ip_allowed = AuthUser.check_ip_allowed(user.user_id, ip_addr,
238 inherit_from_default=inherit)
250 inherit_from_default=inherit)
239 if ip_allowed:
251 if ip_allowed:
240 log.info('Access for IP:%s allowed', ip_addr)
252 log.info('Access for IP:%s allowed', ip_addr)
241 else:
253 else:
242 return False
254 return False
243
255
244 if action == 'push':
256 if action == 'push':
245 if not HasPermissionAnyMiddleware('repository.write',
257 if not HasPermissionAnyMiddleware('repository.write',
246 'repository.admin')(user,
258 'repository.admin')(user,
247 repo_name):
259 repo_name):
248 return False
260 return False
249
261
250 else:
262 else:
251 # any other action need at least read permission
263 # any other action need at least read permission
252 if not HasPermissionAnyMiddleware('repository.read',
264 if not HasPermissionAnyMiddleware('repository.read',
253 'repository.write',
265 'repository.write',
254 'repository.admin')(user,
266 'repository.admin')(user,
255 repo_name):
267 repo_name):
256 return False
268 return False
257
269
258 return True
270 return True
259
271
260 def _check_ssl(self, environ, start_response):
272 def _check_ssl(self, environ, start_response):
261 """
273 """
262 Checks the SSL check flag and returns False if SSL is not present
274 Checks the SSL check flag and returns False if SSL is not present
263 and required True otherwise
275 and required True otherwise
264 """
276 """
265 org_proto = environ['wsgi._org_proto']
277 org_proto = environ['wsgi._org_proto']
266 # check if we have SSL required ! if not it's a bad request !
278 # check if we have SSL required ! if not it's a bad request !
267 require_ssl = str2bool(self.repo_vcs_config.get('web', 'push_ssl'))
279 require_ssl = str2bool(self.repo_vcs_config.get('web', 'push_ssl'))
268 if require_ssl and org_proto == 'http':
280 if require_ssl and org_proto == 'http':
269 log.debug('proto is %s and SSL is required BAD REQUEST !',
281 log.debug('proto is %s and SSL is required BAD REQUEST !',
270 org_proto)
282 org_proto)
271 return False
283 return False
272 return True
284 return True
273
285
274 def __call__(self, environ, start_response):
286 def __call__(self, environ, start_response):
275 try:
287 try:
276 return self._handle_request(environ, start_response)
288 return self._handle_request(environ, start_response)
277 except Exception:
289 except Exception:
278 log.exception("Exception while handling request")
290 log.exception("Exception while handling request")
279 appenlight.track_exception(environ)
291 appenlight.track_exception(environ)
280 return HTTPInternalServerError()(environ, start_response)
292 return HTTPInternalServerError()(environ, start_response)
281 finally:
293 finally:
282 meta.Session.remove()
294 meta.Session.remove()
283
295
284 def _handle_request(self, environ, start_response):
296 def _handle_request(self, environ, start_response):
285
297
286 if not self._check_ssl(environ, start_response):
298 if not self._check_ssl(environ, start_response):
287 reason = ('SSL required, while RhodeCode was unable '
299 reason = ('SSL required, while RhodeCode was unable '
288 'to detect this as SSL request')
300 'to detect this as SSL request')
289 log.debug('User not allowed to proceed, %s', reason)
301 log.debug('User not allowed to proceed, %s', reason)
290 return HTTPNotAcceptable(reason)(environ, start_response)
302 return HTTPNotAcceptable(reason)(environ, start_response)
291
303
292 if not self.url_repo_name:
304 if not self.url_repo_name:
293 log.warning('Repository name is empty: %s', self.url_repo_name)
305 log.warning('Repository name is empty: %s', self.url_repo_name)
294 # failed to get repo name, we fail now
306 # failed to get repo name, we fail now
295 return HTTPNotFound()(environ, start_response)
307 return HTTPNotFound()(environ, start_response)
296 log.debug('Extracted repo name is %s', self.url_repo_name)
308 log.debug('Extracted repo name is %s', self.url_repo_name)
297
309
298 ip_addr = get_ip_addr(environ)
310 ip_addr = get_ip_addr(environ)
299 username = None
311 username = None
300
312
301 # skip passing error to error controller
313 # skip passing error to error controller
302 environ['pylons.status_code_redirect'] = True
314 environ['pylons.status_code_redirect'] = True
303
315
304 # ======================================================================
316 # ======================================================================
305 # GET ACTION PULL or PUSH
317 # GET ACTION PULL or PUSH
306 # ======================================================================
318 # ======================================================================
307 action = self._get_action(environ)
319 action = self._get_action(environ)
308
320
309 # ======================================================================
321 # ======================================================================
310 # Check if this is a request to a shadow repository of a pull request.
322 # Check if this is a request to a shadow repository of a pull request.
311 # In this case only pull action is allowed.
323 # In this case only pull action is allowed.
312 # ======================================================================
324 # ======================================================================
313 if self.is_shadow_repo and action != 'pull':
325 if self.is_shadow_repo and action != 'pull':
314 reason = 'Only pull action is allowed for shadow repositories.'
326 reason = 'Only pull action is allowed for shadow repositories.'
315 log.debug('User not allowed to proceed, %s', reason)
327 log.debug('User not allowed to proceed, %s', reason)
316 return HTTPNotAcceptable(reason)(environ, start_response)
328 return HTTPNotAcceptable(reason)(environ, start_response)
317
329
318 # ======================================================================
330 # ======================================================================
319 # CHECK ANONYMOUS PERMISSION
331 # CHECK ANONYMOUS PERMISSION
320 # ======================================================================
332 # ======================================================================
321 if action in ['pull', 'push']:
333 if action in ['pull', 'push']:
322 anonymous_user = User.get_default_user()
334 anonymous_user = User.get_default_user()
323 username = anonymous_user.username
335 username = anonymous_user.username
324 if anonymous_user.active:
336 if anonymous_user.active:
325 # ONLY check permissions if the user is activated
337 # ONLY check permissions if the user is activated
326 anonymous_perm = self._check_permission(
338 anonymous_perm = self._check_permission(
327 action, anonymous_user, self.acl_repo_name, ip_addr)
339 action, anonymous_user, self.acl_repo_name, ip_addr)
328 else:
340 else:
329 anonymous_perm = False
341 anonymous_perm = False
330
342
331 if not anonymous_user.active or not anonymous_perm:
343 if not anonymous_user.active or not anonymous_perm:
332 if not anonymous_user.active:
344 if not anonymous_user.active:
333 log.debug('Anonymous access is disabled, running '
345 log.debug('Anonymous access is disabled, running '
334 'authentication')
346 'authentication')
335
347
336 if not anonymous_perm:
348 if not anonymous_perm:
337 log.debug('Not enough credentials to access this '
349 log.debug('Not enough credentials to access this '
338 'repository as anonymous user')
350 'repository as anonymous user')
339
351
340 username = None
352 username = None
341 # ==============================================================
353 # ==============================================================
342 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
354 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
343 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
355 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
344 # ==============================================================
356 # ==============================================================
345
357
346 # try to auth based on environ, container auth methods
358 # try to auth based on environ, container auth methods
347 log.debug('Running PRE-AUTH for container based authentication')
359 log.debug('Running PRE-AUTH for container based authentication')
348 pre_auth = authenticate(
360 pre_auth = authenticate(
349 '', '', environ, VCS_TYPE, registry=self.registry)
361 '', '', environ, VCS_TYPE, registry=self.registry)
350 if pre_auth and pre_auth.get('username'):
362 if pre_auth and pre_auth.get('username'):
351 username = pre_auth['username']
363 username = pre_auth['username']
352 log.debug('PRE-AUTH got %s as username', username)
364 log.debug('PRE-AUTH got %s as username', username)
353
365
354 # If not authenticated by the container, running basic auth
366 # If not authenticated by the container, running basic auth
355 if not username:
367 if not username:
356 self.authenticate.realm = get_rhodecode_realm()
368 self.authenticate.realm = get_rhodecode_realm()
357
369
358 try:
370 try:
359 result = self.authenticate(environ)
371 result = self.authenticate(environ)
360 except (UserCreationError, NotAllowedToCreateUserError) as e:
372 except (UserCreationError, NotAllowedToCreateUserError) as e:
361 log.error(e)
373 log.error(e)
362 reason = safe_str(e)
374 reason = safe_str(e)
363 return HTTPNotAcceptable(reason)(environ, start_response)
375 return HTTPNotAcceptable(reason)(environ, start_response)
364
376
365 if isinstance(result, str):
377 if isinstance(result, str):
366 AUTH_TYPE.update(environ, 'basic')
378 AUTH_TYPE.update(environ, 'basic')
367 REMOTE_USER.update(environ, result)
379 REMOTE_USER.update(environ, result)
368 username = result
380 username = result
369 else:
381 else:
370 return result.wsgi_application(environ, start_response)
382 return result.wsgi_application(environ, start_response)
371
383
372 # ==============================================================
384 # ==============================================================
373 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
385 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
374 # ==============================================================
386 # ==============================================================
375 user = User.get_by_username(username)
387 user = User.get_by_username(username)
376 if not self.valid_and_active_user(user):
388 if not self.valid_and_active_user(user):
377 return HTTPForbidden()(environ, start_response)
389 return HTTPForbidden()(environ, start_response)
378 username = user.username
390 username = user.username
379 user.update_lastactivity()
391 user.update_lastactivity()
380 meta.Session().commit()
392 meta.Session().commit()
381
393
382 # check user attributes for password change flag
394 # check user attributes for password change flag
383 user_obj = user
395 user_obj = user
384 if user_obj and user_obj.username != User.DEFAULT_USER and \
396 if user_obj and user_obj.username != User.DEFAULT_USER and \
385 user_obj.user_data.get('force_password_change'):
397 user_obj.user_data.get('force_password_change'):
386 reason = 'password change required'
398 reason = 'password change required'
387 log.debug('User not allowed to authenticate, %s', reason)
399 log.debug('User not allowed to authenticate, %s', reason)
388 return HTTPNotAcceptable(reason)(environ, start_response)
400 return HTTPNotAcceptable(reason)(environ, start_response)
389
401
390 # check permissions for this repository
402 # check permissions for this repository
391 perm = self._check_permission(
403 perm = self._check_permission(
392 action, user, self.acl_repo_name, ip_addr)
404 action, user, self.acl_repo_name, ip_addr)
393 if not perm:
405 if not perm:
394 return HTTPForbidden()(environ, start_response)
406 return HTTPForbidden()(environ, start_response)
395
407
396 # extras are injected into UI object and later available
408 # extras are injected into UI object and later available
397 # in hooks executed by rhodecode
409 # in hooks executed by rhodecode
398 check_locking = _should_check_locking(environ.get('QUERY_STRING'))
410 check_locking = _should_check_locking(environ.get('QUERY_STRING'))
399 extras = vcs_operation_context(
411 extras = vcs_operation_context(
400 environ, repo_name=self.acl_repo_name, username=username,
412 environ, repo_name=self.acl_repo_name, username=username,
401 action=action, scm=self.SCM, check_locking=check_locking,
413 action=action, scm=self.SCM, check_locking=check_locking,
402 is_shadow_repo=self.is_shadow_repo
414 is_shadow_repo=self.is_shadow_repo
403 )
415 )
404
416
405 # ======================================================================
417 # ======================================================================
406 # REQUEST HANDLING
418 # REQUEST HANDLING
407 # ======================================================================
419 # ======================================================================
408 str_repo_name = safe_str(self.url_repo_name)
420 str_repo_name = safe_str(self.url_repo_name)
409 repo_path = os.path.join(
421 repo_path = os.path.join(
410 safe_str(self.basepath), safe_str(self.vcs_repo_name))
422 safe_str(self.basepath), safe_str(self.vcs_repo_name))
411 log.debug('Repository path is %s', repo_path)
423 log.debug('Repository path is %s', repo_path)
412
424
413 fix_PATH()
425 fix_PATH()
414
426
415 log.info(
427 log.info(
416 '%s action on %s repo "%s" by "%s" from %s',
428 '%s action on %s repo "%s" by "%s" from %s',
417 action, self.SCM, str_repo_name, safe_str(username), ip_addr)
429 action, self.SCM, str_repo_name, safe_str(username), ip_addr)
418
430
419 return self._generate_vcs_response(
431 return self._generate_vcs_response(
420 environ, start_response, repo_path, self.url_repo_name, extras, action)
432 environ, start_response, repo_path, self.url_repo_name, extras, action)
421
433
422 @initialize_generator
434 @initialize_generator
423 def _generate_vcs_response(
435 def _generate_vcs_response(
424 self, environ, start_response, repo_path, repo_name, extras,
436 self, environ, start_response, repo_path, repo_name, extras,
425 action):
437 action):
426 """
438 """
427 Returns a generator for the response content.
439 Returns a generator for the response content.
428
440
429 This method is implemented as a generator, so that it can trigger
441 This method is implemented as a generator, so that it can trigger
430 the cache validation after all content sent back to the client. It
442 the cache validation after all content sent back to the client. It
431 also handles the locking exceptions which will be triggered when
443 also handles the locking exceptions which will be triggered when
432 the first chunk is produced by the underlying WSGI application.
444 the first chunk is produced by the underlying WSGI application.
433 """
445 """
434 callback_daemon, extras = self._prepare_callback_daemon(extras)
446 callback_daemon, extras = self._prepare_callback_daemon(extras)
435 config = self._create_config(extras, self.acl_repo_name)
447 config = self._create_config(extras, self.acl_repo_name)
436 log.debug('HOOKS extras is %s', extras)
448 log.debug('HOOKS extras is %s', extras)
437 app = self._create_wsgi_app(repo_path, repo_name, config)
449 app = self._create_wsgi_app(repo_path, repo_name, config)
438
450
439 try:
451 try:
440 with callback_daemon:
452 with callback_daemon:
441 try:
453 try:
442 response = app(environ, start_response)
454 response = app(environ, start_response)
443 finally:
455 finally:
444 # This statement works together with the decorator
456 # This statement works together with the decorator
445 # "initialize_generator" above. The decorator ensures that
457 # "initialize_generator" above. The decorator ensures that
446 # we hit the first yield statement before the generator is
458 # we hit the first yield statement before the generator is
447 # returned back to the WSGI server. This is needed to
459 # returned back to the WSGI server. This is needed to
448 # ensure that the call to "app" above triggers the
460 # ensure that the call to "app" above triggers the
449 # needed callback to "start_response" before the
461 # needed callback to "start_response" before the
450 # generator is actually used.
462 # generator is actually used.
451 yield "__init__"
463 yield "__init__"
452
464
453 for chunk in response:
465 for chunk in response:
454 yield chunk
466 yield chunk
455 except Exception as exc:
467 except Exception as exc:
456 # TODO: johbo: Improve "translating" back the exception.
468 # TODO: johbo: Improve "translating" back the exception.
457 if getattr(exc, '_vcs_kind', None) == 'repo_locked':
469 if getattr(exc, '_vcs_kind', None) == 'repo_locked':
458 exc = HTTPLockedRC(*exc.args)
470 exc = HTTPLockedRC(*exc.args)
459 _code = rhodecode.CONFIG.get('lock_ret_code')
471 _code = rhodecode.CONFIG.get('lock_ret_code')
460 log.debug('Repository LOCKED ret code %s!', (_code,))
472 log.debug('Repository LOCKED ret code %s!', (_code,))
461 elif getattr(exc, '_vcs_kind', None) == 'requirement':
473 elif getattr(exc, '_vcs_kind', None) == 'requirement':
462 log.debug(
474 log.debug(
463 'Repository requires features unknown to this Mercurial')
475 'Repository requires features unknown to this Mercurial')
464 exc = HTTPRequirementError(*exc.args)
476 exc = HTTPRequirementError(*exc.args)
465 else:
477 else:
466 raise
478 raise
467
479
468 for chunk in exc(environ, start_response):
480 for chunk in exc(environ, start_response):
469 yield chunk
481 yield chunk
470 finally:
482 finally:
471 # invalidate cache on push
483 # invalidate cache on push
472 try:
484 try:
473 if action == 'push':
485 if action == 'push':
474 self._invalidate_cache(repo_name)
486 self._invalidate_cache(repo_name)
475 finally:
487 finally:
476 meta.Session.remove()
488 meta.Session.remove()
477
489
478 def _get_repository_name(self, environ):
490 def _get_repository_name(self, environ):
479 """Get repository name out of the environmnent
491 """Get repository name out of the environmnent
480
492
481 :param environ: WSGI environment
493 :param environ: WSGI environment
482 """
494 """
483 raise NotImplementedError()
495 raise NotImplementedError()
484
496
485 def _get_action(self, environ):
497 def _get_action(self, environ):
486 """Map request commands into a pull or push command.
498 """Map request commands into a pull or push command.
487
499
488 :param environ: WSGI environment
500 :param environ: WSGI environment
489 """
501 """
490 raise NotImplementedError()
502 raise NotImplementedError()
491
503
492 def _create_wsgi_app(self, repo_path, repo_name, config):
504 def _create_wsgi_app(self, repo_path, repo_name, config):
493 """Return the WSGI app that will finally handle the request."""
505 """Return the WSGI app that will finally handle the request."""
494 raise NotImplementedError()
506 raise NotImplementedError()
495
507
496 def _create_config(self, extras, repo_name):
508 def _create_config(self, extras, repo_name):
497 """Create a Pyro safe config representation."""
509 """Create a Pyro safe config representation."""
498 raise NotImplementedError()
510 raise NotImplementedError()
499
511
500 def _prepare_callback_daemon(self, extras):
512 def _prepare_callback_daemon(self, extras):
501 return prepare_callback_daemon(
513 return prepare_callback_daemon(
502 extras, protocol=vcs_settings.HOOKS_PROTOCOL,
514 extras, protocol=vcs_settings.HOOKS_PROTOCOL,
503 use_direct_calls=vcs_settings.HOOKS_DIRECT_CALLS)
515 use_direct_calls=vcs_settings.HOOKS_DIRECT_CALLS)
504
516
505
517
506 def _should_check_locking(query_string):
518 def _should_check_locking(query_string):
507 # this is kind of hacky, but due to how mercurial handles client-server
519 # this is kind of hacky, but due to how mercurial handles client-server
508 # server see all operation on commit; bookmarks, phases and
520 # server see all operation on commit; bookmarks, phases and
509 # obsolescence marker in different transaction, we don't want to check
521 # obsolescence marker in different transaction, we don't want to check
510 # locking on those
522 # locking on those
511 return query_string not in ['cmd=listkeys']
523 return query_string not in ['cmd=listkeys']
@@ -1,204 +1,204 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2010-2016 RhodeCode GmbH
3 # Copyright (C) 2010-2016 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import gzip
21 import gzip
22 import shutil
22 import shutil
23 import logging
23 import logging
24 import tempfile
24 import tempfile
25 import urlparse
25 import urlparse
26
26
27 from webob.exc import HTTPNotFound
27 from webob.exc import HTTPNotFound
28
28
29 import rhodecode
29 import rhodecode
30 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
30 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
31 from rhodecode.lib.middleware.simplegit import SimpleGit, GIT_PROTO_PAT
31 from rhodecode.lib.middleware.simplegit import SimpleGit, GIT_PROTO_PAT
32 from rhodecode.lib.middleware.simplehg import SimpleHg
32 from rhodecode.lib.middleware.simplehg import SimpleHg
33 from rhodecode.lib.middleware.simplesvn import SimpleSvn
33 from rhodecode.lib.middleware.simplesvn import SimpleSvn
34 from rhodecode.model.settings import VcsSettingsModel
34 from rhodecode.model.settings import VcsSettingsModel
35
35
36 log = logging.getLogger(__name__)
36 log = logging.getLogger(__name__)
37
37
38
38
39 def is_git(environ):
39 def is_git(environ):
40 """
40 """
41 Returns True if requests should be handled by GIT wsgi middleware
41 Returns True if requests should be handled by GIT wsgi middleware
42 """
42 """
43 is_git_path = GIT_PROTO_PAT.match(environ['PATH_INFO'])
43 is_git_path = GIT_PROTO_PAT.match(environ['PATH_INFO'])
44 log.debug(
44 log.debug(
45 'request path: `%s` detected as GIT PROTOCOL %s', environ['PATH_INFO'],
45 'request path: `%s` detected as GIT PROTOCOL %s', environ['PATH_INFO'],
46 is_git_path is not None)
46 is_git_path is not None)
47
47
48 return is_git_path
48 return is_git_path
49
49
50
50
51 def is_hg(environ):
51 def is_hg(environ):
52 """
52 """
53 Returns True if requests target is mercurial server - header
53 Returns True if requests target is mercurial server - header
54 ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
54 ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
55 """
55 """
56 is_hg_path = False
56 is_hg_path = False
57
57
58 http_accept = environ.get('HTTP_ACCEPT')
58 http_accept = environ.get('HTTP_ACCEPT')
59
59
60 if http_accept and http_accept.startswith('application/mercurial'):
60 if http_accept and http_accept.startswith('application/mercurial'):
61 query = urlparse.parse_qs(environ['QUERY_STRING'])
61 query = urlparse.parse_qs(environ['QUERY_STRING'])
62 if 'cmd' in query:
62 if 'cmd' in query:
63 is_hg_path = True
63 is_hg_path = True
64
64
65 log.debug(
65 log.debug(
66 'request path: `%s` detected as HG PROTOCOL %s', environ['PATH_INFO'],
66 'request path: `%s` detected as HG PROTOCOL %s', environ['PATH_INFO'],
67 is_hg_path)
67 is_hg_path)
68
68
69 return is_hg_path
69 return is_hg_path
70
70
71
71
72 def is_svn(environ):
72 def is_svn(environ):
73 """
73 """
74 Returns True if requests target is Subversion server
74 Returns True if requests target is Subversion server
75 """
75 """
76 http_dav = environ.get('HTTP_DAV', '')
76 http_dav = environ.get('HTTP_DAV', '')
77 magic_path_segment = rhodecode.CONFIG.get(
77 magic_path_segment = rhodecode.CONFIG.get(
78 'rhodecode_subversion_magic_path', '/!svn')
78 'rhodecode_subversion_magic_path', '/!svn')
79 is_svn_path = (
79 is_svn_path = (
80 'subversion' in http_dav or
80 'subversion' in http_dav or
81 magic_path_segment in environ['PATH_INFO'])
81 magic_path_segment in environ['PATH_INFO'])
82 log.debug(
82 log.debug(
83 'request path: `%s` detected as SVN PROTOCOL %s', environ['PATH_INFO'],
83 'request path: `%s` detected as SVN PROTOCOL %s', environ['PATH_INFO'],
84 is_svn_path)
84 is_svn_path)
85
85
86 return is_svn_path
86 return is_svn_path
87
87
88
88
89 class GunzipMiddleware(object):
89 class GunzipMiddleware(object):
90 """
90 """
91 WSGI middleware that unzips gzip-encoded requests before
91 WSGI middleware that unzips gzip-encoded requests before
92 passing on to the underlying application.
92 passing on to the underlying application.
93 """
93 """
94
94
95 def __init__(self, application):
95 def __init__(self, application):
96 self.app = application
96 self.app = application
97
97
98 def __call__(self, environ, start_response):
98 def __call__(self, environ, start_response):
99 accepts_encoding_header = environ.get('HTTP_CONTENT_ENCODING', b'')
99 accepts_encoding_header = environ.get('HTTP_CONTENT_ENCODING', b'')
100
100
101 if b'gzip' in accepts_encoding_header:
101 if b'gzip' in accepts_encoding_header:
102 log.debug('gzip detected, now running gunzip wrapper')
102 log.debug('gzip detected, now running gunzip wrapper')
103 wsgi_input = environ['wsgi.input']
103 wsgi_input = environ['wsgi.input']
104
104
105 if not hasattr(environ['wsgi.input'], 'seek'):
105 if not hasattr(environ['wsgi.input'], 'seek'):
106 # The gzip implementation in the standard library of Python 2.x
106 # The gzip implementation in the standard library of Python 2.x
107 # requires the '.seek()' and '.tell()' methods to be available
107 # requires the '.seek()' and '.tell()' methods to be available
108 # on the input stream. Read the data into a temporary file to
108 # on the input stream. Read the data into a temporary file to
109 # work around this limitation.
109 # work around this limitation.
110
110
111 wsgi_input = tempfile.SpooledTemporaryFile(64 * 1024 * 1024)
111 wsgi_input = tempfile.SpooledTemporaryFile(64 * 1024 * 1024)
112 shutil.copyfileobj(environ['wsgi.input'], wsgi_input)
112 shutil.copyfileobj(environ['wsgi.input'], wsgi_input)
113 wsgi_input.seek(0)
113 wsgi_input.seek(0)
114
114
115 environ['wsgi.input'] = gzip.GzipFile(fileobj=wsgi_input, mode='r')
115 environ['wsgi.input'] = gzip.GzipFile(fileobj=wsgi_input, mode='r')
116 # since we "Ungzipped" the content we say now it's no longer gzip
116 # since we "Ungzipped" the content we say now it's no longer gzip
117 # content encoding
117 # content encoding
118 del environ['HTTP_CONTENT_ENCODING']
118 del environ['HTTP_CONTENT_ENCODING']
119
119
120 # content length has changes ? or i'm not sure
120 # content length has changes ? or i'm not sure
121 if 'CONTENT_LENGTH' in environ:
121 if 'CONTENT_LENGTH' in environ:
122 del environ['CONTENT_LENGTH']
122 del environ['CONTENT_LENGTH']
123 else:
123 else:
124 log.debug('content not gzipped, gzipMiddleware passing '
124 log.debug('content not gzipped, gzipMiddleware passing '
125 'request further')
125 'request further')
126 return self.app(environ, start_response)
126 return self.app(environ, start_response)
127
127
128
128
129 class VCSMiddleware(object):
129 class VCSMiddleware(object):
130
130
131 def __init__(self, app, config, appenlight_client, registry):
131 def __init__(self, app, config, appenlight_client, registry):
132 self.application = app
132 self.application = app
133 self.config = config
133 self.config = config
134 self.appenlight_client = appenlight_client
134 self.appenlight_client = appenlight_client
135 self.registry = registry
135 self.registry = registry
136 self.use_gzip = True
136 self.use_gzip = True
137 # order in which we check the middlewares, based on vcs.backends config
137 # order in which we check the middlewares, based on vcs.backends config
138 self.check_middlewares = config['vcs.backends']
138 self.check_middlewares = config['vcs.backends']
139 self.checks = {
139 self.checks = {
140 'hg': (is_hg, SimpleHg),
140 'hg': (is_hg, SimpleHg),
141 'git': (is_git, SimpleGit),
141 'git': (is_git, SimpleGit),
142 'svn': (is_svn, SimpleSvn),
142 'svn': (is_svn, SimpleSvn),
143 }
143 }
144
144
145 def vcs_config(self, repo_name=None):
145 def vcs_config(self, repo_name=None):
146 """
146 """
147 returns serialized VcsSettings
147 returns serialized VcsSettings
148 """
148 """
149 return VcsSettingsModel(repo=repo_name).get_ui_settings_as_config_obj()
149 return VcsSettingsModel(repo=repo_name).get_ui_settings_as_config_obj()
150
150
151 def wrap_in_gzip_if_enabled(self, app, config):
151 def wrap_in_gzip_if_enabled(self, app, config):
152 if self.use_gzip:
152 if self.use_gzip:
153 app = GunzipMiddleware(app)
153 app = GunzipMiddleware(app)
154 return app
154 return app
155
155
156 def _get_handler_app(self, environ):
156 def _get_handler_app(self, environ):
157 app = None
157 app = None
158 log.debug('Checking vcs types in order: %r', self.check_middlewares)
158 log.debug('Checking vcs types in order: %r', self.check_middlewares)
159 for vcs_type in self.check_middlewares:
159 for vcs_type in self.check_middlewares:
160 vcs_check, handler = self.checks[vcs_type]
160 vcs_check, handler = self.checks[vcs_type]
161 if vcs_check(environ):
161 if vcs_check(environ):
162 log.debug(
162 log.debug(
163 'Found VCS Middleware to handle the request %s', handler)
163 'Found VCS Middleware to handle the request %s', handler)
164 app = handler(self.application, self.config, self.registry)
164 app = handler(self.application, self.config, self.registry)
165 break
165 break
166
166
167 return app
167 return app
168
168
169 def __call__(self, environ, start_response):
169 def __call__(self, environ, start_response):
170 # check if we handle one of interesting protocols, optionally extract
170 # check if we handle one of interesting protocols, optionally extract
171 # specific vcsSettings and allow changes of how things are wrapped
171 # specific vcsSettings and allow changes of how things are wrapped
172 vcs_handler = self._get_handler_app(environ)
172 vcs_handler = self._get_handler_app(environ)
173 if vcs_handler:
173 if vcs_handler:
174 # translate the _REPO_ID into real repo NAME for usage
174 # translate the _REPO_ID into real repo NAME for usage
175 # in middleware
175 # in middleware
176 environ['PATH_INFO'] = vcs_handler._get_by_id(environ['PATH_INFO'])
176 environ['PATH_INFO'] = vcs_handler._get_by_id(environ['PATH_INFO'])
177
177
178 # Set repo names for permission checks, vcs and web interaction.
178 # Set acl, url and vcs repo names.
179 vcs_handler.set_repo_names(environ)
179 vcs_handler.set_repo_names(environ)
180
180
181 # check for type, presence in database and on filesystem
181 # check for type, presence in database and on filesystem
182 if not vcs_handler.is_valid_and_existing_repo(
182 if not vcs_handler.is_valid_and_existing_repo(
183 vcs_handler.acl_repo_name,
183 vcs_handler.acl_repo_name,
184 vcs_handler.basepath,
184 vcs_handler.basepath,
185 vcs_handler.SCM):
185 vcs_handler.SCM):
186 return HTTPNotFound()(environ, start_response)
186 return HTTPNotFound()(environ, start_response)
187
187
188 # TODO: johbo: Needed for the Pyro4 backend and Mercurial only.
188 # TODO: johbo: Needed for the Pyro4 backend and Mercurial only.
189 # Remove once we fully switched to the HTTP backend.
189 # Remove once we fully switched to the HTTP backend.
190 environ['REPO_NAME'] = vcs_handler.url_repo_name
190 environ['REPO_NAME'] = vcs_handler.url_repo_name
191
191
192 # register repo config back to the handler
192 # register repo config back to the handler
193 vcs_handler.repo_vcs_config = self.vcs_config(
193 vcs_handler.repo_vcs_config = self.vcs_config(
194 vcs_handler.acl_repo_name)
194 vcs_handler.acl_repo_name)
195
195
196 # Wrap handler in middlewares if they are enabled.
196 # Wrap handler in middlewares if they are enabled.
197 vcs_handler = self.wrap_in_gzip_if_enabled(
197 vcs_handler = self.wrap_in_gzip_if_enabled(
198 vcs_handler, self.config)
198 vcs_handler, self.config)
199 vcs_handler, _ = wrap_in_appenlight_if_enabled(
199 vcs_handler, _ = wrap_in_appenlight_if_enabled(
200 vcs_handler, self.config, self.appenlight_client)
200 vcs_handler, self.config, self.appenlight_client)
201
201
202 return vcs_handler(environ, start_response)
202 return vcs_handler(environ, start_response)
203
203
204 return self.application(environ, start_response)
204 return self.application(environ, start_response)
@@ -1,998 +1,1003 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2010-2016 RhodeCode GmbH
3 # Copyright (C) 2010-2016 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 """
21 """
22 Utilities library for RhodeCode
22 Utilities library for RhodeCode
23 """
23 """
24
24
25 import datetime
25 import datetime
26 import decorator
26 import decorator
27 import json
27 import json
28 import logging
28 import logging
29 import os
29 import os
30 import re
30 import re
31 import shutil
31 import shutil
32 import tempfile
32 import tempfile
33 import traceback
33 import traceback
34 import tarfile
34 import tarfile
35 import warnings
35 import warnings
36 from os.path import join as jn
36 from os.path import join as jn
37
37
38 import paste
38 import paste
39 import pkg_resources
39 import pkg_resources
40 from paste.script.command import Command, BadCommand
40 from paste.script.command import Command, BadCommand
41 from webhelpers.text import collapse, remove_formatting, strip_tags
41 from webhelpers.text import collapse, remove_formatting, strip_tags
42 from mako import exceptions
42 from mako import exceptions
43 from pyramid.threadlocal import get_current_registry
43 from pyramid.threadlocal import get_current_registry
44
44
45 from rhodecode.lib.fakemod import create_module
45 from rhodecode.lib.fakemod import create_module
46 from rhodecode.lib.vcs.backends.base import Config
46 from rhodecode.lib.vcs.backends.base import Config
47 from rhodecode.lib.vcs.exceptions import VCSError
47 from rhodecode.lib.vcs.exceptions import VCSError
48 from rhodecode.lib.vcs.utils.helpers import get_scm, get_scm_backend
48 from rhodecode.lib.vcs.utils.helpers import get_scm, get_scm_backend
49 from rhodecode.lib.utils2 import (
49 from rhodecode.lib.utils2 import (
50 safe_str, safe_unicode, get_current_rhodecode_user, md5)
50 safe_str, safe_unicode, get_current_rhodecode_user, md5)
51 from rhodecode.model import meta
51 from rhodecode.model import meta
52 from rhodecode.model.db import (
52 from rhodecode.model.db import (
53 Repository, User, RhodeCodeUi, UserLog, RepoGroup, UserGroup)
53 Repository, User, RhodeCodeUi, UserLog, RepoGroup, UserGroup)
54 from rhodecode.model.meta import Session
54 from rhodecode.model.meta import Session
55
55
56
56
57 log = logging.getLogger(__name__)
57 log = logging.getLogger(__name__)
58
58
59 REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}__.*')
59 REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}__.*')
60
60
61 # String of characters which are not allowed in repo/group slugs.
61 # String which contains characters that are not allowed in slug names for
62 # repositories or repository groups. It is properly escaped to use it in
63 # regular expressions.
62 SLUG_BAD_CHARS = re.escape('`?=[]\;\'"<>,/~!@#$%^&*()+{}|:')
64 SLUG_BAD_CHARS = re.escape('`?=[]\;\'"<>,/~!@#$%^&*()+{}|:')
65
63 # Regex that matches forbidden characters in repo/group slugs.
66 # Regex that matches forbidden characters in repo/group slugs.
64 SLUG_BAD_CHAR_RE = re.compile('[{}]'.format(SLUG_BAD_CHARS))
67 SLUG_BAD_CHAR_RE = re.compile('[{}]'.format(SLUG_BAD_CHARS))
68
65 # Regex that matches allowed characters in repo/group slugs.
69 # Regex that matches allowed characters in repo/group slugs.
66 SLUG_GOOD_CHAR_RE = re.compile('[^{}]'.format(SLUG_BAD_CHARS))
70 SLUG_GOOD_CHAR_RE = re.compile('[^{}]'.format(SLUG_BAD_CHARS))
71
67 # Regex that matches whole repo/group slugs.
72 # Regex that matches whole repo/group slugs.
68 SLUG_RE = re.compile('[^{}]+'.format(SLUG_BAD_CHARS))
73 SLUG_RE = re.compile('[^{}]+'.format(SLUG_BAD_CHARS))
69
74
70 _license_cache = None
75 _license_cache = None
71
76
72
77
73 def repo_name_slug(value):
78 def repo_name_slug(value):
74 """
79 """
75 Return slug of name of repository
80 Return slug of name of repository
76 This function is called on each creation/modification
81 This function is called on each creation/modification
77 of repository to prevent bad names in repo
82 of repository to prevent bad names in repo
78 """
83 """
79 replacement_char = '-'
84 replacement_char = '-'
80
85
81 slug = remove_formatting(value)
86 slug = remove_formatting(value)
82 slug = SLUG_BAD_CHAR_RE.sub(replacement_char, slug)
87 slug = SLUG_BAD_CHAR_RE.sub(replacement_char, slug)
83 slug = collapse(slug, replacement_char)
88 slug = collapse(slug, replacement_char)
84 return slug
89 return slug
85
90
86
91
87 #==============================================================================
92 #==============================================================================
88 # PERM DECORATOR HELPERS FOR EXTRACTING NAMES FOR PERM CHECKS
93 # PERM DECORATOR HELPERS FOR EXTRACTING NAMES FOR PERM CHECKS
89 #==============================================================================
94 #==============================================================================
90 def get_repo_slug(request):
95 def get_repo_slug(request):
91 _repo = request.environ['pylons.routes_dict'].get('repo_name')
96 _repo = request.environ['pylons.routes_dict'].get('repo_name')
92 if _repo:
97 if _repo:
93 _repo = _repo.rstrip('/')
98 _repo = _repo.rstrip('/')
94 return _repo
99 return _repo
95
100
96
101
97 def get_repo_group_slug(request):
102 def get_repo_group_slug(request):
98 _group = request.environ['pylons.routes_dict'].get('group_name')
103 _group = request.environ['pylons.routes_dict'].get('group_name')
99 if _group:
104 if _group:
100 _group = _group.rstrip('/')
105 _group = _group.rstrip('/')
101 return _group
106 return _group
102
107
103
108
104 def get_user_group_slug(request):
109 def get_user_group_slug(request):
105 _group = request.environ['pylons.routes_dict'].get('user_group_id')
110 _group = request.environ['pylons.routes_dict'].get('user_group_id')
106 try:
111 try:
107 _group = UserGroup.get(_group)
112 _group = UserGroup.get(_group)
108 if _group:
113 if _group:
109 _group = _group.users_group_name
114 _group = _group.users_group_name
110 except Exception:
115 except Exception:
111 log.debug(traceback.format_exc())
116 log.debug(traceback.format_exc())
112 #catch all failures here
117 #catch all failures here
113 pass
118 pass
114
119
115 return _group
120 return _group
116
121
117
122
118 def action_logger(user, action, repo, ipaddr='', sa=None, commit=False):
123 def action_logger(user, action, repo, ipaddr='', sa=None, commit=False):
119 """
124 """
120 Action logger for various actions made by users
125 Action logger for various actions made by users
121
126
122 :param user: user that made this action, can be a unique username string or
127 :param user: user that made this action, can be a unique username string or
123 object containing user_id attribute
128 object containing user_id attribute
124 :param action: action to log, should be on of predefined unique actions for
129 :param action: action to log, should be on of predefined unique actions for
125 easy translations
130 easy translations
126 :param repo: string name of repository or object containing repo_id,
131 :param repo: string name of repository or object containing repo_id,
127 that action was made on
132 that action was made on
128 :param ipaddr: optional ip address from what the action was made
133 :param ipaddr: optional ip address from what the action was made
129 :param sa: optional sqlalchemy session
134 :param sa: optional sqlalchemy session
130
135
131 """
136 """
132
137
133 if not sa:
138 if not sa:
134 sa = meta.Session()
139 sa = meta.Session()
135 # if we don't get explicit IP address try to get one from registered user
140 # if we don't get explicit IP address try to get one from registered user
136 # in tmpl context var
141 # in tmpl context var
137 if not ipaddr:
142 if not ipaddr:
138 ipaddr = getattr(get_current_rhodecode_user(), 'ip_addr', '')
143 ipaddr = getattr(get_current_rhodecode_user(), 'ip_addr', '')
139
144
140 try:
145 try:
141 if getattr(user, 'user_id', None):
146 if getattr(user, 'user_id', None):
142 user_obj = User.get(user.user_id)
147 user_obj = User.get(user.user_id)
143 elif isinstance(user, basestring):
148 elif isinstance(user, basestring):
144 user_obj = User.get_by_username(user)
149 user_obj = User.get_by_username(user)
145 else:
150 else:
146 raise Exception('You have to provide a user object or a username')
151 raise Exception('You have to provide a user object or a username')
147
152
148 if getattr(repo, 'repo_id', None):
153 if getattr(repo, 'repo_id', None):
149 repo_obj = Repository.get(repo.repo_id)
154 repo_obj = Repository.get(repo.repo_id)
150 repo_name = repo_obj.repo_name
155 repo_name = repo_obj.repo_name
151 elif isinstance(repo, basestring):
156 elif isinstance(repo, basestring):
152 repo_name = repo.lstrip('/')
157 repo_name = repo.lstrip('/')
153 repo_obj = Repository.get_by_repo_name(repo_name)
158 repo_obj = Repository.get_by_repo_name(repo_name)
154 else:
159 else:
155 repo_obj = None
160 repo_obj = None
156 repo_name = ''
161 repo_name = ''
157
162
158 user_log = UserLog()
163 user_log = UserLog()
159 user_log.user_id = user_obj.user_id
164 user_log.user_id = user_obj.user_id
160 user_log.username = user_obj.username
165 user_log.username = user_obj.username
161 action = safe_unicode(action)
166 action = safe_unicode(action)
162 user_log.action = action[:1200000]
167 user_log.action = action[:1200000]
163
168
164 user_log.repository = repo_obj
169 user_log.repository = repo_obj
165 user_log.repository_name = repo_name
170 user_log.repository_name = repo_name
166
171
167 user_log.action_date = datetime.datetime.now()
172 user_log.action_date = datetime.datetime.now()
168 user_log.user_ip = ipaddr
173 user_log.user_ip = ipaddr
169 sa.add(user_log)
174 sa.add(user_log)
170
175
171 log.info('Logging action:`%s` on repo:`%s` by user:%s ip:%s',
176 log.info('Logging action:`%s` on repo:`%s` by user:%s ip:%s',
172 action, safe_unicode(repo), user_obj, ipaddr)
177 action, safe_unicode(repo), user_obj, ipaddr)
173 if commit:
178 if commit:
174 sa.commit()
179 sa.commit()
175 except Exception:
180 except Exception:
176 log.error(traceback.format_exc())
181 log.error(traceback.format_exc())
177 raise
182 raise
178
183
179
184
180 def get_filesystem_repos(path, recursive=False, skip_removed_repos=True):
185 def get_filesystem_repos(path, recursive=False, skip_removed_repos=True):
181 """
186 """
182 Scans given path for repos and return (name,(type,path)) tuple
187 Scans given path for repos and return (name,(type,path)) tuple
183
188
184 :param path: path to scan for repositories
189 :param path: path to scan for repositories
185 :param recursive: recursive search and return names with subdirs in front
190 :param recursive: recursive search and return names with subdirs in front
186 """
191 """
187
192
188 # remove ending slash for better results
193 # remove ending slash for better results
189 path = path.rstrip(os.sep)
194 path = path.rstrip(os.sep)
190 log.debug('now scanning in %s location recursive:%s...', path, recursive)
195 log.debug('now scanning in %s location recursive:%s...', path, recursive)
191
196
192 def _get_repos(p):
197 def _get_repos(p):
193 dirpaths = _get_dirpaths(p)
198 dirpaths = _get_dirpaths(p)
194 if not _is_dir_writable(p):
199 if not _is_dir_writable(p):
195 log.warning('repo path without write access: %s', p)
200 log.warning('repo path without write access: %s', p)
196
201
197 for dirpath in dirpaths:
202 for dirpath in dirpaths:
198 if os.path.isfile(os.path.join(p, dirpath)):
203 if os.path.isfile(os.path.join(p, dirpath)):
199 continue
204 continue
200 cur_path = os.path.join(p, dirpath)
205 cur_path = os.path.join(p, dirpath)
201
206
202 # skip removed repos
207 # skip removed repos
203 if skip_removed_repos and REMOVED_REPO_PAT.match(dirpath):
208 if skip_removed_repos and REMOVED_REPO_PAT.match(dirpath):
204 continue
209 continue
205
210
206 #skip .<somethin> dirs
211 #skip .<somethin> dirs
207 if dirpath.startswith('.'):
212 if dirpath.startswith('.'):
208 continue
213 continue
209
214
210 try:
215 try:
211 scm_info = get_scm(cur_path)
216 scm_info = get_scm(cur_path)
212 yield scm_info[1].split(path, 1)[-1].lstrip(os.sep), scm_info
217 yield scm_info[1].split(path, 1)[-1].lstrip(os.sep), scm_info
213 except VCSError:
218 except VCSError:
214 if not recursive:
219 if not recursive:
215 continue
220 continue
216 #check if this dir containts other repos for recursive scan
221 #check if this dir containts other repos for recursive scan
217 rec_path = os.path.join(p, dirpath)
222 rec_path = os.path.join(p, dirpath)
218 if os.path.isdir(rec_path):
223 if os.path.isdir(rec_path):
219 for inner_scm in _get_repos(rec_path):
224 for inner_scm in _get_repos(rec_path):
220 yield inner_scm
225 yield inner_scm
221
226
222 return _get_repos(path)
227 return _get_repos(path)
223
228
224
229
225 def _get_dirpaths(p):
230 def _get_dirpaths(p):
226 try:
231 try:
227 # OS-independable way of checking if we have at least read-only
232 # OS-independable way of checking if we have at least read-only
228 # access or not.
233 # access or not.
229 dirpaths = os.listdir(p)
234 dirpaths = os.listdir(p)
230 except OSError:
235 except OSError:
231 log.warning('ignoring repo path without read access: %s', p)
236 log.warning('ignoring repo path without read access: %s', p)
232 return []
237 return []
233
238
234 # os.listpath has a tweak: If a unicode is passed into it, then it tries to
239 # os.listpath has a tweak: If a unicode is passed into it, then it tries to
235 # decode paths and suddenly returns unicode objects itself. The items it
240 # decode paths and suddenly returns unicode objects itself. The items it
236 # cannot decode are returned as strings and cause issues.
241 # cannot decode are returned as strings and cause issues.
237 #
242 #
238 # Those paths are ignored here until a solid solution for path handling has
243 # Those paths are ignored here until a solid solution for path handling has
239 # been built.
244 # been built.
240 expected_type = type(p)
245 expected_type = type(p)
241
246
242 def _has_correct_type(item):
247 def _has_correct_type(item):
243 if type(item) is not expected_type:
248 if type(item) is not expected_type:
244 log.error(
249 log.error(
245 u"Ignoring path %s since it cannot be decoded into unicode.",
250 u"Ignoring path %s since it cannot be decoded into unicode.",
246 # Using "repr" to make sure that we see the byte value in case
251 # Using "repr" to make sure that we see the byte value in case
247 # of support.
252 # of support.
248 repr(item))
253 repr(item))
249 return False
254 return False
250 return True
255 return True
251
256
252 dirpaths = [item for item in dirpaths if _has_correct_type(item)]
257 dirpaths = [item for item in dirpaths if _has_correct_type(item)]
253
258
254 return dirpaths
259 return dirpaths
255
260
256
261
257 def _is_dir_writable(path):
262 def _is_dir_writable(path):
258 """
263 """
259 Probe if `path` is writable.
264 Probe if `path` is writable.
260
265
261 Due to trouble on Cygwin / Windows, this is actually probing if it is
266 Due to trouble on Cygwin / Windows, this is actually probing if it is
262 possible to create a file inside of `path`, stat does not produce reliable
267 possible to create a file inside of `path`, stat does not produce reliable
263 results in this case.
268 results in this case.
264 """
269 """
265 try:
270 try:
266 with tempfile.TemporaryFile(dir=path):
271 with tempfile.TemporaryFile(dir=path):
267 pass
272 pass
268 except OSError:
273 except OSError:
269 return False
274 return False
270 return True
275 return True
271
276
272
277
273 def is_valid_repo(repo_name, base_path, expect_scm=None, explicit_scm=None):
278 def is_valid_repo(repo_name, base_path, expect_scm=None, explicit_scm=None):
274 """
279 """
275 Returns True if given path is a valid repository False otherwise.
280 Returns True if given path is a valid repository False otherwise.
276 If expect_scm param is given also, compare if given scm is the same
281 If expect_scm param is given also, compare if given scm is the same
277 as expected from scm parameter. If explicit_scm is given don't try to
282 as expected from scm parameter. If explicit_scm is given don't try to
278 detect the scm, just use the given one to check if repo is valid
283 detect the scm, just use the given one to check if repo is valid
279
284
280 :param repo_name:
285 :param repo_name:
281 :param base_path:
286 :param base_path:
282 :param expect_scm:
287 :param expect_scm:
283 :param explicit_scm:
288 :param explicit_scm:
284
289
285 :return True: if given path is a valid repository
290 :return True: if given path is a valid repository
286 """
291 """
287 full_path = os.path.join(safe_str(base_path), safe_str(repo_name))
292 full_path = os.path.join(safe_str(base_path), safe_str(repo_name))
288 log.debug('Checking if `%s` is a valid path for repository', repo_name)
293 log.debug('Checking if `%s` is a valid path for repository', repo_name)
289
294
290 try:
295 try:
291 if explicit_scm:
296 if explicit_scm:
292 detected_scms = [get_scm_backend(explicit_scm)]
297 detected_scms = [get_scm_backend(explicit_scm)]
293 else:
298 else:
294 detected_scms = get_scm(full_path)
299 detected_scms = get_scm(full_path)
295
300
296 if expect_scm:
301 if expect_scm:
297 return detected_scms[0] == expect_scm
302 return detected_scms[0] == expect_scm
298 log.debug('path: %s is an vcs object:%s', full_path, detected_scms)
303 log.debug('path: %s is an vcs object:%s', full_path, detected_scms)
299 return True
304 return True
300 except VCSError:
305 except VCSError:
301 log.debug('path: %s is not a valid repo !', full_path)
306 log.debug('path: %s is not a valid repo !', full_path)
302 return False
307 return False
303
308
304
309
305 def is_valid_repo_group(repo_group_name, base_path, skip_path_check=False):
310 def is_valid_repo_group(repo_group_name, base_path, skip_path_check=False):
306 """
311 """
307 Returns True if given path is a repository group, False otherwise
312 Returns True if given path is a repository group, False otherwise
308
313
309 :param repo_name:
314 :param repo_name:
310 :param base_path:
315 :param base_path:
311 """
316 """
312 full_path = os.path.join(safe_str(base_path), safe_str(repo_group_name))
317 full_path = os.path.join(safe_str(base_path), safe_str(repo_group_name))
313 log.debug('Checking if `%s` is a valid path for repository group',
318 log.debug('Checking if `%s` is a valid path for repository group',
314 repo_group_name)
319 repo_group_name)
315
320
316 # check if it's not a repo
321 # check if it's not a repo
317 if is_valid_repo(repo_group_name, base_path):
322 if is_valid_repo(repo_group_name, base_path):
318 log.debug('Repo called %s exist, it is not a valid '
323 log.debug('Repo called %s exist, it is not a valid '
319 'repo group' % repo_group_name)
324 'repo group' % repo_group_name)
320 return False
325 return False
321
326
322 try:
327 try:
323 # we need to check bare git repos at higher level
328 # we need to check bare git repos at higher level
324 # since we might match branches/hooks/info/objects or possible
329 # since we might match branches/hooks/info/objects or possible
325 # other things inside bare git repo
330 # other things inside bare git repo
326 scm_ = get_scm(os.path.dirname(full_path))
331 scm_ = get_scm(os.path.dirname(full_path))
327 log.debug('path: %s is a vcs object:%s, not valid '
332 log.debug('path: %s is a vcs object:%s, not valid '
328 'repo group' % (full_path, scm_))
333 'repo group' % (full_path, scm_))
329 return False
334 return False
330 except VCSError:
335 except VCSError:
331 pass
336 pass
332
337
333 # check if it's a valid path
338 # check if it's a valid path
334 if skip_path_check or os.path.isdir(full_path):
339 if skip_path_check or os.path.isdir(full_path):
335 log.debug('path: %s is a valid repo group !', full_path)
340 log.debug('path: %s is a valid repo group !', full_path)
336 return True
341 return True
337
342
338 log.debug('path: %s is not a valid repo group !', full_path)
343 log.debug('path: %s is not a valid repo group !', full_path)
339 return False
344 return False
340
345
341
346
342 def ask_ok(prompt, retries=4, complaint='[y]es or [n]o please!'):
347 def ask_ok(prompt, retries=4, complaint='[y]es or [n]o please!'):
343 while True:
348 while True:
344 ok = raw_input(prompt)
349 ok = raw_input(prompt)
345 if ok.lower() in ('y', 'ye', 'yes'):
350 if ok.lower() in ('y', 'ye', 'yes'):
346 return True
351 return True
347 if ok.lower() in ('n', 'no', 'nop', 'nope'):
352 if ok.lower() in ('n', 'no', 'nop', 'nope'):
348 return False
353 return False
349 retries = retries - 1
354 retries = retries - 1
350 if retries < 0:
355 if retries < 0:
351 raise IOError
356 raise IOError
352 print(complaint)
357 print(complaint)
353
358
354 # propagated from mercurial documentation
359 # propagated from mercurial documentation
355 ui_sections = [
360 ui_sections = [
356 'alias', 'auth',
361 'alias', 'auth',
357 'decode/encode', 'defaults',
362 'decode/encode', 'defaults',
358 'diff', 'email',
363 'diff', 'email',
359 'extensions', 'format',
364 'extensions', 'format',
360 'merge-patterns', 'merge-tools',
365 'merge-patterns', 'merge-tools',
361 'hooks', 'http_proxy',
366 'hooks', 'http_proxy',
362 'smtp', 'patch',
367 'smtp', 'patch',
363 'paths', 'profiling',
368 'paths', 'profiling',
364 'server', 'trusted',
369 'server', 'trusted',
365 'ui', 'web', ]
370 'ui', 'web', ]
366
371
367
372
368 def config_data_from_db(clear_session=True, repo=None):
373 def config_data_from_db(clear_session=True, repo=None):
369 """
374 """
370 Read the configuration data from the database and return configuration
375 Read the configuration data from the database and return configuration
371 tuples.
376 tuples.
372 """
377 """
373 from rhodecode.model.settings import VcsSettingsModel
378 from rhodecode.model.settings import VcsSettingsModel
374
379
375 config = []
380 config = []
376
381
377 sa = meta.Session()
382 sa = meta.Session()
378 settings_model = VcsSettingsModel(repo=repo, sa=sa)
383 settings_model = VcsSettingsModel(repo=repo, sa=sa)
379
384
380 ui_settings = settings_model.get_ui_settings()
385 ui_settings = settings_model.get_ui_settings()
381
386
382 for setting in ui_settings:
387 for setting in ui_settings:
383 if setting.active:
388 if setting.active:
384 log.debug(
389 log.debug(
385 'settings ui from db: [%s] %s=%s',
390 'settings ui from db: [%s] %s=%s',
386 setting.section, setting.key, setting.value)
391 setting.section, setting.key, setting.value)
387 config.append((
392 config.append((
388 safe_str(setting.section), safe_str(setting.key),
393 safe_str(setting.section), safe_str(setting.key),
389 safe_str(setting.value)))
394 safe_str(setting.value)))
390 if setting.key == 'push_ssl':
395 if setting.key == 'push_ssl':
391 # force set push_ssl requirement to False, rhodecode
396 # force set push_ssl requirement to False, rhodecode
392 # handles that
397 # handles that
393 config.append((
398 config.append((
394 safe_str(setting.section), safe_str(setting.key), False))
399 safe_str(setting.section), safe_str(setting.key), False))
395 if clear_session:
400 if clear_session:
396 meta.Session.remove()
401 meta.Session.remove()
397
402
398 # TODO: mikhail: probably it makes no sense to re-read hooks information.
403 # TODO: mikhail: probably it makes no sense to re-read hooks information.
399 # It's already there and activated/deactivated
404 # It's already there and activated/deactivated
400 skip_entries = []
405 skip_entries = []
401 enabled_hook_classes = get_enabled_hook_classes(ui_settings)
406 enabled_hook_classes = get_enabled_hook_classes(ui_settings)
402 if 'pull' not in enabled_hook_classes:
407 if 'pull' not in enabled_hook_classes:
403 skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PULL))
408 skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PULL))
404 if 'push' not in enabled_hook_classes:
409 if 'push' not in enabled_hook_classes:
405 skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PUSH))
410 skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PUSH))
406
411
407 config = [entry for entry in config if entry[:2] not in skip_entries]
412 config = [entry for entry in config if entry[:2] not in skip_entries]
408
413
409 return config
414 return config
410
415
411
416
412 def make_db_config(clear_session=True, repo=None):
417 def make_db_config(clear_session=True, repo=None):
413 """
418 """
414 Create a :class:`Config` instance based on the values in the database.
419 Create a :class:`Config` instance based on the values in the database.
415 """
420 """
416 config = Config()
421 config = Config()
417 config_data = config_data_from_db(clear_session=clear_session, repo=repo)
422 config_data = config_data_from_db(clear_session=clear_session, repo=repo)
418 for section, option, value in config_data:
423 for section, option, value in config_data:
419 config.set(section, option, value)
424 config.set(section, option, value)
420 return config
425 return config
421
426
422
427
423 def get_enabled_hook_classes(ui_settings):
428 def get_enabled_hook_classes(ui_settings):
424 """
429 """
425 Return the enabled hook classes.
430 Return the enabled hook classes.
426
431
427 :param ui_settings: List of ui_settings as returned
432 :param ui_settings: List of ui_settings as returned
428 by :meth:`VcsSettingsModel.get_ui_settings`
433 by :meth:`VcsSettingsModel.get_ui_settings`
429
434
430 :return: a list with the enabled hook classes. The order is not guaranteed.
435 :return: a list with the enabled hook classes. The order is not guaranteed.
431 :rtype: list
436 :rtype: list
432 """
437 """
433 enabled_hooks = []
438 enabled_hooks = []
434 active_hook_keys = [
439 active_hook_keys = [
435 key for section, key, value, active in ui_settings
440 key for section, key, value, active in ui_settings
436 if section == 'hooks' and active]
441 if section == 'hooks' and active]
437
442
438 hook_names = {
443 hook_names = {
439 RhodeCodeUi.HOOK_PUSH: 'push',
444 RhodeCodeUi.HOOK_PUSH: 'push',
440 RhodeCodeUi.HOOK_PULL: 'pull',
445 RhodeCodeUi.HOOK_PULL: 'pull',
441 RhodeCodeUi.HOOK_REPO_SIZE: 'repo_size'
446 RhodeCodeUi.HOOK_REPO_SIZE: 'repo_size'
442 }
447 }
443
448
444 for key in active_hook_keys:
449 for key in active_hook_keys:
445 hook = hook_names.get(key)
450 hook = hook_names.get(key)
446 if hook:
451 if hook:
447 enabled_hooks.append(hook)
452 enabled_hooks.append(hook)
448
453
449 return enabled_hooks
454 return enabled_hooks
450
455
451
456
452 def set_rhodecode_config(config):
457 def set_rhodecode_config(config):
453 """
458 """
454 Updates pylons config with new settings from database
459 Updates pylons config with new settings from database
455
460
456 :param config:
461 :param config:
457 """
462 """
458 from rhodecode.model.settings import SettingsModel
463 from rhodecode.model.settings import SettingsModel
459 app_settings = SettingsModel().get_all_settings()
464 app_settings = SettingsModel().get_all_settings()
460
465
461 for k, v in app_settings.items():
466 for k, v in app_settings.items():
462 config[k] = v
467 config[k] = v
463
468
464
469
465 def get_rhodecode_realm():
470 def get_rhodecode_realm():
466 """
471 """
467 Return the rhodecode realm from database.
472 Return the rhodecode realm from database.
468 """
473 """
469 from rhodecode.model.settings import SettingsModel
474 from rhodecode.model.settings import SettingsModel
470 realm = SettingsModel().get_setting_by_name('realm')
475 realm = SettingsModel().get_setting_by_name('realm')
471 return safe_str(realm.app_settings_value)
476 return safe_str(realm.app_settings_value)
472
477
473
478
474 def get_rhodecode_base_path():
479 def get_rhodecode_base_path():
475 """
480 """
476 Returns the base path. The base path is the filesystem path which points
481 Returns the base path. The base path is the filesystem path which points
477 to the repository store.
482 to the repository store.
478 """
483 """
479 from rhodecode.model.settings import SettingsModel
484 from rhodecode.model.settings import SettingsModel
480 paths_ui = SettingsModel().get_ui_by_section_and_key('paths', '/')
485 paths_ui = SettingsModel().get_ui_by_section_and_key('paths', '/')
481 return safe_str(paths_ui.ui_value)
486 return safe_str(paths_ui.ui_value)
482
487
483
488
484 def map_groups(path):
489 def map_groups(path):
485 """
490 """
486 Given a full path to a repository, create all nested groups that this
491 Given a full path to a repository, create all nested groups that this
487 repo is inside. This function creates parent-child relationships between
492 repo is inside. This function creates parent-child relationships between
488 groups and creates default perms for all new groups.
493 groups and creates default perms for all new groups.
489
494
490 :param paths: full path to repository
495 :param paths: full path to repository
491 """
496 """
492 from rhodecode.model.repo_group import RepoGroupModel
497 from rhodecode.model.repo_group import RepoGroupModel
493 sa = meta.Session()
498 sa = meta.Session()
494 groups = path.split(Repository.NAME_SEP)
499 groups = path.split(Repository.NAME_SEP)
495 parent = None
500 parent = None
496 group = None
501 group = None
497
502
498 # last element is repo in nested groups structure
503 # last element is repo in nested groups structure
499 groups = groups[:-1]
504 groups = groups[:-1]
500 rgm = RepoGroupModel(sa)
505 rgm = RepoGroupModel(sa)
501 owner = User.get_first_super_admin()
506 owner = User.get_first_super_admin()
502 for lvl, group_name in enumerate(groups):
507 for lvl, group_name in enumerate(groups):
503 group_name = '/'.join(groups[:lvl] + [group_name])
508 group_name = '/'.join(groups[:lvl] + [group_name])
504 group = RepoGroup.get_by_group_name(group_name)
509 group = RepoGroup.get_by_group_name(group_name)
505 desc = '%s group' % group_name
510 desc = '%s group' % group_name
506
511
507 # skip folders that are now removed repos
512 # skip folders that are now removed repos
508 if REMOVED_REPO_PAT.match(group_name):
513 if REMOVED_REPO_PAT.match(group_name):
509 break
514 break
510
515
511 if group is None:
516 if group is None:
512 log.debug('creating group level: %s group_name: %s',
517 log.debug('creating group level: %s group_name: %s',
513 lvl, group_name)
518 lvl, group_name)
514 group = RepoGroup(group_name, parent)
519 group = RepoGroup(group_name, parent)
515 group.group_description = desc
520 group.group_description = desc
516 group.user = owner
521 group.user = owner
517 sa.add(group)
522 sa.add(group)
518 perm_obj = rgm._create_default_perms(group)
523 perm_obj = rgm._create_default_perms(group)
519 sa.add(perm_obj)
524 sa.add(perm_obj)
520 sa.flush()
525 sa.flush()
521
526
522 parent = group
527 parent = group
523 return group
528 return group
524
529
525
530
526 def repo2db_mapper(initial_repo_list, remove_obsolete=False):
531 def repo2db_mapper(initial_repo_list, remove_obsolete=False):
527 """
532 """
528 maps all repos given in initial_repo_list, non existing repositories
533 maps all repos given in initial_repo_list, non existing repositories
529 are created, if remove_obsolete is True it also checks for db entries
534 are created, if remove_obsolete is True it also checks for db entries
530 that are not in initial_repo_list and removes them.
535 that are not in initial_repo_list and removes them.
531
536
532 :param initial_repo_list: list of repositories found by scanning methods
537 :param initial_repo_list: list of repositories found by scanning methods
533 :param remove_obsolete: check for obsolete entries in database
538 :param remove_obsolete: check for obsolete entries in database
534 """
539 """
535 from rhodecode.model.repo import RepoModel
540 from rhodecode.model.repo import RepoModel
536 from rhodecode.model.scm import ScmModel
541 from rhodecode.model.scm import ScmModel
537 from rhodecode.model.repo_group import RepoGroupModel
542 from rhodecode.model.repo_group import RepoGroupModel
538 from rhodecode.model.settings import SettingsModel
543 from rhodecode.model.settings import SettingsModel
539
544
540 sa = meta.Session()
545 sa = meta.Session()
541 repo_model = RepoModel()
546 repo_model = RepoModel()
542 user = User.get_first_super_admin()
547 user = User.get_first_super_admin()
543 added = []
548 added = []
544
549
545 # creation defaults
550 # creation defaults
546 defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
551 defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
547 enable_statistics = defs.get('repo_enable_statistics')
552 enable_statistics = defs.get('repo_enable_statistics')
548 enable_locking = defs.get('repo_enable_locking')
553 enable_locking = defs.get('repo_enable_locking')
549 enable_downloads = defs.get('repo_enable_downloads')
554 enable_downloads = defs.get('repo_enable_downloads')
550 private = defs.get('repo_private')
555 private = defs.get('repo_private')
551
556
552 for name, repo in initial_repo_list.items():
557 for name, repo in initial_repo_list.items():
553 group = map_groups(name)
558 group = map_groups(name)
554 unicode_name = safe_unicode(name)
559 unicode_name = safe_unicode(name)
555 db_repo = repo_model.get_by_repo_name(unicode_name)
560 db_repo = repo_model.get_by_repo_name(unicode_name)
556 # found repo that is on filesystem not in RhodeCode database
561 # found repo that is on filesystem not in RhodeCode database
557 if not db_repo:
562 if not db_repo:
558 log.info('repository %s not found, creating now', name)
563 log.info('repository %s not found, creating now', name)
559 added.append(name)
564 added.append(name)
560 desc = (repo.description
565 desc = (repo.description
561 if repo.description != 'unknown'
566 if repo.description != 'unknown'
562 else '%s repository' % name)
567 else '%s repository' % name)
563
568
564 db_repo = repo_model._create_repo(
569 db_repo = repo_model._create_repo(
565 repo_name=name,
570 repo_name=name,
566 repo_type=repo.alias,
571 repo_type=repo.alias,
567 description=desc,
572 description=desc,
568 repo_group=getattr(group, 'group_id', None),
573 repo_group=getattr(group, 'group_id', None),
569 owner=user,
574 owner=user,
570 enable_locking=enable_locking,
575 enable_locking=enable_locking,
571 enable_downloads=enable_downloads,
576 enable_downloads=enable_downloads,
572 enable_statistics=enable_statistics,
577 enable_statistics=enable_statistics,
573 private=private,
578 private=private,
574 state=Repository.STATE_CREATED
579 state=Repository.STATE_CREATED
575 )
580 )
576 sa.commit()
581 sa.commit()
577 # we added that repo just now, and make sure we updated server info
582 # we added that repo just now, and make sure we updated server info
578 if db_repo.repo_type == 'git':
583 if db_repo.repo_type == 'git':
579 git_repo = db_repo.scm_instance()
584 git_repo = db_repo.scm_instance()
580 # update repository server-info
585 # update repository server-info
581 log.debug('Running update server info')
586 log.debug('Running update server info')
582 git_repo._update_server_info()
587 git_repo._update_server_info()
583
588
584 db_repo.update_commit_cache()
589 db_repo.update_commit_cache()
585
590
586 config = db_repo._config
591 config = db_repo._config
587 config.set('extensions', 'largefiles', '')
592 config.set('extensions', 'largefiles', '')
588 ScmModel().install_hooks(
593 ScmModel().install_hooks(
589 db_repo.scm_instance(config=config),
594 db_repo.scm_instance(config=config),
590 repo_type=db_repo.repo_type)
595 repo_type=db_repo.repo_type)
591
596
592 removed = []
597 removed = []
593 if remove_obsolete:
598 if remove_obsolete:
594 # remove from database those repositories that are not in the filesystem
599 # remove from database those repositories that are not in the filesystem
595 for repo in sa.query(Repository).all():
600 for repo in sa.query(Repository).all():
596 if repo.repo_name not in initial_repo_list.keys():
601 if repo.repo_name not in initial_repo_list.keys():
597 log.debug("Removing non-existing repository found in db `%s`",
602 log.debug("Removing non-existing repository found in db `%s`",
598 repo.repo_name)
603 repo.repo_name)
599 try:
604 try:
600 RepoModel(sa).delete(repo, forks='detach', fs_remove=False)
605 RepoModel(sa).delete(repo, forks='detach', fs_remove=False)
601 sa.commit()
606 sa.commit()
602 removed.append(repo.repo_name)
607 removed.append(repo.repo_name)
603 except Exception:
608 except Exception:
604 # don't hold further removals on error
609 # don't hold further removals on error
605 log.error(traceback.format_exc())
610 log.error(traceback.format_exc())
606 sa.rollback()
611 sa.rollback()
607
612
608 def splitter(full_repo_name):
613 def splitter(full_repo_name):
609 _parts = full_repo_name.rsplit(RepoGroup.url_sep(), 1)
614 _parts = full_repo_name.rsplit(RepoGroup.url_sep(), 1)
610 gr_name = None
615 gr_name = None
611 if len(_parts) == 2:
616 if len(_parts) == 2:
612 gr_name = _parts[0]
617 gr_name = _parts[0]
613 return gr_name
618 return gr_name
614
619
615 initial_repo_group_list = [splitter(x) for x in
620 initial_repo_group_list = [splitter(x) for x in
616 initial_repo_list.keys() if splitter(x)]
621 initial_repo_list.keys() if splitter(x)]
617
622
618 # remove from database those repository groups that are not in the
623 # remove from database those repository groups that are not in the
619 # filesystem due to parent child relationships we need to delete them
624 # filesystem due to parent child relationships we need to delete them
620 # in a specific order of most nested first
625 # in a specific order of most nested first
621 all_groups = [x.group_name for x in sa.query(RepoGroup).all()]
626 all_groups = [x.group_name for x in sa.query(RepoGroup).all()]
622 nested_sort = lambda gr: len(gr.split('/'))
627 nested_sort = lambda gr: len(gr.split('/'))
623 for group_name in sorted(all_groups, key=nested_sort, reverse=True):
628 for group_name in sorted(all_groups, key=nested_sort, reverse=True):
624 if group_name not in initial_repo_group_list:
629 if group_name not in initial_repo_group_list:
625 repo_group = RepoGroup.get_by_group_name(group_name)
630 repo_group = RepoGroup.get_by_group_name(group_name)
626 if (repo_group.children.all() or
631 if (repo_group.children.all() or
627 not RepoGroupModel().check_exist_filesystem(
632 not RepoGroupModel().check_exist_filesystem(
628 group_name=group_name, exc_on_failure=False)):
633 group_name=group_name, exc_on_failure=False)):
629 continue
634 continue
630
635
631 log.info(
636 log.info(
632 'Removing non-existing repository group found in db `%s`',
637 'Removing non-existing repository group found in db `%s`',
633 group_name)
638 group_name)
634 try:
639 try:
635 RepoGroupModel(sa).delete(group_name, fs_remove=False)
640 RepoGroupModel(sa).delete(group_name, fs_remove=False)
636 sa.commit()
641 sa.commit()
637 removed.append(group_name)
642 removed.append(group_name)
638 except Exception:
643 except Exception:
639 # don't hold further removals on error
644 # don't hold further removals on error
640 log.exception(
645 log.exception(
641 'Unable to remove repository group `%s`',
646 'Unable to remove repository group `%s`',
642 group_name)
647 group_name)
643 sa.rollback()
648 sa.rollback()
644 raise
649 raise
645
650
646 return added, removed
651 return added, removed
647
652
648
653
649 def get_default_cache_settings(settings):
654 def get_default_cache_settings(settings):
650 cache_settings = {}
655 cache_settings = {}
651 for key in settings.keys():
656 for key in settings.keys():
652 for prefix in ['beaker.cache.', 'cache.']:
657 for prefix in ['beaker.cache.', 'cache.']:
653 if key.startswith(prefix):
658 if key.startswith(prefix):
654 name = key.split(prefix)[1].strip()
659 name = key.split(prefix)[1].strip()
655 cache_settings[name] = settings[key].strip()
660 cache_settings[name] = settings[key].strip()
656 return cache_settings
661 return cache_settings
657
662
658
663
659 # set cache regions for beaker so celery can utilise it
664 # set cache regions for beaker so celery can utilise it
660 def add_cache(settings):
665 def add_cache(settings):
661 from rhodecode.lib import caches
666 from rhodecode.lib import caches
662 cache_settings = {'regions': None}
667 cache_settings = {'regions': None}
663 # main cache settings used as default ...
668 # main cache settings used as default ...
664 cache_settings.update(get_default_cache_settings(settings))
669 cache_settings.update(get_default_cache_settings(settings))
665
670
666 if cache_settings['regions']:
671 if cache_settings['regions']:
667 for region in cache_settings['regions'].split(','):
672 for region in cache_settings['regions'].split(','):
668 region = region.strip()
673 region = region.strip()
669 region_settings = {}
674 region_settings = {}
670 for key, value in cache_settings.items():
675 for key, value in cache_settings.items():
671 if key.startswith(region):
676 if key.startswith(region):
672 region_settings[key.split('.')[1]] = value
677 region_settings[key.split('.')[1]] = value
673
678
674 caches.configure_cache_region(
679 caches.configure_cache_region(
675 region, region_settings, cache_settings)
680 region, region_settings, cache_settings)
676
681
677
682
678 def load_rcextensions(root_path):
683 def load_rcextensions(root_path):
679 import rhodecode
684 import rhodecode
680 from rhodecode.config import conf
685 from rhodecode.config import conf
681
686
682 path = os.path.join(root_path, 'rcextensions', '__init__.py')
687 path = os.path.join(root_path, 'rcextensions', '__init__.py')
683 if os.path.isfile(path):
688 if os.path.isfile(path):
684 rcext = create_module('rc', path)
689 rcext = create_module('rc', path)
685 EXT = rhodecode.EXTENSIONS = rcext
690 EXT = rhodecode.EXTENSIONS = rcext
686 log.debug('Found rcextensions now loading %s...', rcext)
691 log.debug('Found rcextensions now loading %s...', rcext)
687
692
688 # Additional mappings that are not present in the pygments lexers
693 # Additional mappings that are not present in the pygments lexers
689 conf.LANGUAGES_EXTENSIONS_MAP.update(getattr(EXT, 'EXTRA_MAPPINGS', {}))
694 conf.LANGUAGES_EXTENSIONS_MAP.update(getattr(EXT, 'EXTRA_MAPPINGS', {}))
690
695
691 # auto check if the module is not missing any data, set to default if is
696 # auto check if the module is not missing any data, set to default if is
692 # this will help autoupdate new feature of rcext module
697 # this will help autoupdate new feature of rcext module
693 #from rhodecode.config import rcextensions
698 #from rhodecode.config import rcextensions
694 #for k in dir(rcextensions):
699 #for k in dir(rcextensions):
695 # if not k.startswith('_') and not hasattr(EXT, k):
700 # if not k.startswith('_') and not hasattr(EXT, k):
696 # setattr(EXT, k, getattr(rcextensions, k))
701 # setattr(EXT, k, getattr(rcextensions, k))
697
702
698
703
699 def get_custom_lexer(extension):
704 def get_custom_lexer(extension):
700 """
705 """
701 returns a custom lexer if it is defined in rcextensions module, or None
706 returns a custom lexer if it is defined in rcextensions module, or None
702 if there's no custom lexer defined
707 if there's no custom lexer defined
703 """
708 """
704 import rhodecode
709 import rhodecode
705 from pygments import lexers
710 from pygments import lexers
706 # check if we didn't define this extension as other lexer
711 # check if we didn't define this extension as other lexer
707 extensions = rhodecode.EXTENSIONS and getattr(rhodecode.EXTENSIONS, 'EXTRA_LEXERS', None)
712 extensions = rhodecode.EXTENSIONS and getattr(rhodecode.EXTENSIONS, 'EXTRA_LEXERS', None)
708 if extensions and extension in rhodecode.EXTENSIONS.EXTRA_LEXERS:
713 if extensions and extension in rhodecode.EXTENSIONS.EXTRA_LEXERS:
709 _lexer_name = rhodecode.EXTENSIONS.EXTRA_LEXERS[extension]
714 _lexer_name = rhodecode.EXTENSIONS.EXTRA_LEXERS[extension]
710 return lexers.get_lexer_by_name(_lexer_name)
715 return lexers.get_lexer_by_name(_lexer_name)
711
716
712
717
713 #==============================================================================
718 #==============================================================================
714 # TEST FUNCTIONS AND CREATORS
719 # TEST FUNCTIONS AND CREATORS
715 #==============================================================================
720 #==============================================================================
716 def create_test_index(repo_location, config):
721 def create_test_index(repo_location, config):
717 """
722 """
718 Makes default test index.
723 Makes default test index.
719 """
724 """
720 import rc_testdata
725 import rc_testdata
721
726
722 rc_testdata.extract_search_index(
727 rc_testdata.extract_search_index(
723 'vcs_search_index', os.path.dirname(config['search.location']))
728 'vcs_search_index', os.path.dirname(config['search.location']))
724
729
725
730
726 def create_test_directory(test_path):
731 def create_test_directory(test_path):
727 """
732 """
728 Create test directory if it doesn't exist.
733 Create test directory if it doesn't exist.
729 """
734 """
730 if not os.path.isdir(test_path):
735 if not os.path.isdir(test_path):
731 log.debug('Creating testdir %s', test_path)
736 log.debug('Creating testdir %s', test_path)
732 os.makedirs(test_path)
737 os.makedirs(test_path)
733
738
734
739
735 def create_test_database(test_path, config):
740 def create_test_database(test_path, config):
736 """
741 """
737 Makes a fresh database.
742 Makes a fresh database.
738 """
743 """
739 from rhodecode.lib.db_manage import DbManage
744 from rhodecode.lib.db_manage import DbManage
740
745
741 # PART ONE create db
746 # PART ONE create db
742 dbconf = config['sqlalchemy.db1.url']
747 dbconf = config['sqlalchemy.db1.url']
743 log.debug('making test db %s', dbconf)
748 log.debug('making test db %s', dbconf)
744
749
745 dbmanage = DbManage(log_sql=False, dbconf=dbconf, root=config['here'],
750 dbmanage = DbManage(log_sql=False, dbconf=dbconf, root=config['here'],
746 tests=True, cli_args={'force_ask': True})
751 tests=True, cli_args={'force_ask': True})
747 dbmanage.create_tables(override=True)
752 dbmanage.create_tables(override=True)
748 dbmanage.set_db_version()
753 dbmanage.set_db_version()
749 # for tests dynamically set new root paths based on generated content
754 # for tests dynamically set new root paths based on generated content
750 dbmanage.create_settings(dbmanage.config_prompt(test_path))
755 dbmanage.create_settings(dbmanage.config_prompt(test_path))
751 dbmanage.create_default_user()
756 dbmanage.create_default_user()
752 dbmanage.create_test_admin_and_users()
757 dbmanage.create_test_admin_and_users()
753 dbmanage.create_permissions()
758 dbmanage.create_permissions()
754 dbmanage.populate_default_permissions()
759 dbmanage.populate_default_permissions()
755 Session().commit()
760 Session().commit()
756
761
757
762
758 def create_test_repositories(test_path, config):
763 def create_test_repositories(test_path, config):
759 """
764 """
760 Creates test repositories in the temporary directory. Repositories are
765 Creates test repositories in the temporary directory. Repositories are
761 extracted from archives within the rc_testdata package.
766 extracted from archives within the rc_testdata package.
762 """
767 """
763 import rc_testdata
768 import rc_testdata
764 from rhodecode.tests import HG_REPO, GIT_REPO, SVN_REPO
769 from rhodecode.tests import HG_REPO, GIT_REPO, SVN_REPO
765
770
766 log.debug('making test vcs repositories')
771 log.debug('making test vcs repositories')
767
772
768 idx_path = config['search.location']
773 idx_path = config['search.location']
769 data_path = config['cache_dir']
774 data_path = config['cache_dir']
770
775
771 # clean index and data
776 # clean index and data
772 if idx_path and os.path.exists(idx_path):
777 if idx_path and os.path.exists(idx_path):
773 log.debug('remove %s', idx_path)
778 log.debug('remove %s', idx_path)
774 shutil.rmtree(idx_path)
779 shutil.rmtree(idx_path)
775
780
776 if data_path and os.path.exists(data_path):
781 if data_path and os.path.exists(data_path):
777 log.debug('remove %s', data_path)
782 log.debug('remove %s', data_path)
778 shutil.rmtree(data_path)
783 shutil.rmtree(data_path)
779
784
780 rc_testdata.extract_hg_dump('vcs_test_hg', jn(test_path, HG_REPO))
785 rc_testdata.extract_hg_dump('vcs_test_hg', jn(test_path, HG_REPO))
781 rc_testdata.extract_git_dump('vcs_test_git', jn(test_path, GIT_REPO))
786 rc_testdata.extract_git_dump('vcs_test_git', jn(test_path, GIT_REPO))
782
787
783 # Note: Subversion is in the process of being integrated with the system,
788 # Note: Subversion is in the process of being integrated with the system,
784 # until we have a properly packed version of the test svn repository, this
789 # until we have a properly packed version of the test svn repository, this
785 # tries to copy over the repo from a package "rc_testdata"
790 # tries to copy over the repo from a package "rc_testdata"
786 svn_repo_path = rc_testdata.get_svn_repo_archive()
791 svn_repo_path = rc_testdata.get_svn_repo_archive()
787 with tarfile.open(svn_repo_path) as tar:
792 with tarfile.open(svn_repo_path) as tar:
788 tar.extractall(jn(test_path, SVN_REPO))
793 tar.extractall(jn(test_path, SVN_REPO))
789
794
790
795
791 #==============================================================================
796 #==============================================================================
792 # PASTER COMMANDS
797 # PASTER COMMANDS
793 #==============================================================================
798 #==============================================================================
794 class BasePasterCommand(Command):
799 class BasePasterCommand(Command):
795 """
800 """
796 Abstract Base Class for paster commands.
801 Abstract Base Class for paster commands.
797
802
798 The celery commands are somewhat aggressive about loading
803 The celery commands are somewhat aggressive about loading
799 celery.conf, and since our module sets the `CELERY_LOADER`
804 celery.conf, and since our module sets the `CELERY_LOADER`
800 environment variable to our loader, we have to bootstrap a bit and
805 environment variable to our loader, we have to bootstrap a bit and
801 make sure we've had a chance to load the pylons config off of the
806 make sure we've had a chance to load the pylons config off of the
802 command line, otherwise everything fails.
807 command line, otherwise everything fails.
803 """
808 """
804 min_args = 1
809 min_args = 1
805 min_args_error = "Please provide a paster config file as an argument."
810 min_args_error = "Please provide a paster config file as an argument."
806 takes_config_file = 1
811 takes_config_file = 1
807 requires_config_file = True
812 requires_config_file = True
808
813
809 def notify_msg(self, msg, log=False):
814 def notify_msg(self, msg, log=False):
810 """Make a notification to user, additionally if logger is passed
815 """Make a notification to user, additionally if logger is passed
811 it logs this action using given logger
816 it logs this action using given logger
812
817
813 :param msg: message that will be printed to user
818 :param msg: message that will be printed to user
814 :param log: logging instance, to use to additionally log this message
819 :param log: logging instance, to use to additionally log this message
815
820
816 """
821 """
817 if log and isinstance(log, logging):
822 if log and isinstance(log, logging):
818 log(msg)
823 log(msg)
819
824
820 def run(self, args):
825 def run(self, args):
821 """
826 """
822 Overrides Command.run
827 Overrides Command.run
823
828
824 Checks for a config file argument and loads it.
829 Checks for a config file argument and loads it.
825 """
830 """
826 if len(args) < self.min_args:
831 if len(args) < self.min_args:
827 raise BadCommand(
832 raise BadCommand(
828 self.min_args_error % {'min_args': self.min_args,
833 self.min_args_error % {'min_args': self.min_args,
829 'actual_args': len(args)})
834 'actual_args': len(args)})
830
835
831 # Decrement because we're going to lob off the first argument.
836 # Decrement because we're going to lob off the first argument.
832 # @@ This is hacky
837 # @@ This is hacky
833 self.min_args -= 1
838 self.min_args -= 1
834 self.bootstrap_config(args[0])
839 self.bootstrap_config(args[0])
835 self.update_parser()
840 self.update_parser()
836 return super(BasePasterCommand, self).run(args[1:])
841 return super(BasePasterCommand, self).run(args[1:])
837
842
838 def update_parser(self):
843 def update_parser(self):
839 """
844 """
840 Abstract method. Allows for the class' parser to be updated
845 Abstract method. Allows for the class' parser to be updated
841 before the superclass' `run` method is called. Necessary to
846 before the superclass' `run` method is called. Necessary to
842 allow options/arguments to be passed through to the underlying
847 allow options/arguments to be passed through to the underlying
843 celery command.
848 celery command.
844 """
849 """
845 raise NotImplementedError("Abstract Method.")
850 raise NotImplementedError("Abstract Method.")
846
851
847 def bootstrap_config(self, conf):
852 def bootstrap_config(self, conf):
848 """
853 """
849 Loads the pylons configuration.
854 Loads the pylons configuration.
850 """
855 """
851 from pylons import config as pylonsconfig
856 from pylons import config as pylonsconfig
852
857
853 self.path_to_ini_file = os.path.realpath(conf)
858 self.path_to_ini_file = os.path.realpath(conf)
854 conf = paste.deploy.appconfig('config:' + self.path_to_ini_file)
859 conf = paste.deploy.appconfig('config:' + self.path_to_ini_file)
855 pylonsconfig.init_app(conf.global_conf, conf.local_conf)
860 pylonsconfig.init_app(conf.global_conf, conf.local_conf)
856
861
857 def _init_session(self):
862 def _init_session(self):
858 """
863 """
859 Inits SqlAlchemy Session
864 Inits SqlAlchemy Session
860 """
865 """
861 logging.config.fileConfig(self.path_to_ini_file)
866 logging.config.fileConfig(self.path_to_ini_file)
862 from pylons import config
867 from pylons import config
863 from rhodecode.config.utils import initialize_database
868 from rhodecode.config.utils import initialize_database
864
869
865 # get to remove repos !!
870 # get to remove repos !!
866 add_cache(config)
871 add_cache(config)
867 initialize_database(config)
872 initialize_database(config)
868
873
869
874
870 @decorator.decorator
875 @decorator.decorator
871 def jsonify(func, *args, **kwargs):
876 def jsonify(func, *args, **kwargs):
872 """Action decorator that formats output for JSON
877 """Action decorator that formats output for JSON
873
878
874 Given a function that will return content, this decorator will turn
879 Given a function that will return content, this decorator will turn
875 the result into JSON, with a content-type of 'application/json' and
880 the result into JSON, with a content-type of 'application/json' and
876 output it.
881 output it.
877
882
878 """
883 """
879 from pylons.decorators.util import get_pylons
884 from pylons.decorators.util import get_pylons
880 from rhodecode.lib.ext_json import json
885 from rhodecode.lib.ext_json import json
881 pylons = get_pylons(args)
886 pylons = get_pylons(args)
882 pylons.response.headers['Content-Type'] = 'application/json; charset=utf-8'
887 pylons.response.headers['Content-Type'] = 'application/json; charset=utf-8'
883 data = func(*args, **kwargs)
888 data = func(*args, **kwargs)
884 if isinstance(data, (list, tuple)):
889 if isinstance(data, (list, tuple)):
885 msg = "JSON responses with Array envelopes are susceptible to " \
890 msg = "JSON responses with Array envelopes are susceptible to " \
886 "cross-site data leak attacks, see " \
891 "cross-site data leak attacks, see " \
887 "http://wiki.pylonshq.com/display/pylonsfaq/Warnings"
892 "http://wiki.pylonshq.com/display/pylonsfaq/Warnings"
888 warnings.warn(msg, Warning, 2)
893 warnings.warn(msg, Warning, 2)
889 log.warning(msg)
894 log.warning(msg)
890 log.debug("Returning JSON wrapped action output")
895 log.debug("Returning JSON wrapped action output")
891 return json.dumps(data, encoding='utf-8')
896 return json.dumps(data, encoding='utf-8')
892
897
893
898
894 class PartialRenderer(object):
899 class PartialRenderer(object):
895 """
900 """
896 Partial renderer used to render chunks of html used in datagrids
901 Partial renderer used to render chunks of html used in datagrids
897 use like::
902 use like::
898
903
899 _render = PartialRenderer('data_table/_dt_elements.html')
904 _render = PartialRenderer('data_table/_dt_elements.html')
900 _render('quick_menu', args, kwargs)
905 _render('quick_menu', args, kwargs)
901 PartialRenderer.h,
906 PartialRenderer.h,
902 c,
907 c,
903 _,
908 _,
904 ungettext
909 ungettext
905 are the template stuff initialized inside and can be re-used later
910 are the template stuff initialized inside and can be re-used later
906
911
907 :param tmpl_name: template path relate to /templates/ dir
912 :param tmpl_name: template path relate to /templates/ dir
908 """
913 """
909
914
910 def __init__(self, tmpl_name):
915 def __init__(self, tmpl_name):
911 import rhodecode
916 import rhodecode
912 from pylons import request, tmpl_context as c
917 from pylons import request, tmpl_context as c
913 from pylons.i18n.translation import _, ungettext
918 from pylons.i18n.translation import _, ungettext
914 from rhodecode.lib import helpers as h
919 from rhodecode.lib import helpers as h
915
920
916 self.tmpl_name = tmpl_name
921 self.tmpl_name = tmpl_name
917 self.rhodecode = rhodecode
922 self.rhodecode = rhodecode
918 self.c = c
923 self.c = c
919 self._ = _
924 self._ = _
920 self.ungettext = ungettext
925 self.ungettext = ungettext
921 self.h = h
926 self.h = h
922 self.request = request
927 self.request = request
923
928
924 def _mako_lookup(self):
929 def _mako_lookup(self):
925 _tmpl_lookup = self.rhodecode.CONFIG['pylons.app_globals'].mako_lookup
930 _tmpl_lookup = self.rhodecode.CONFIG['pylons.app_globals'].mako_lookup
926 return _tmpl_lookup.get_template(self.tmpl_name)
931 return _tmpl_lookup.get_template(self.tmpl_name)
927
932
928 def _update_kwargs_for_render(self, kwargs):
933 def _update_kwargs_for_render(self, kwargs):
929 """
934 """
930 Inject params required for Mako rendering
935 Inject params required for Mako rendering
931 """
936 """
932 _kwargs = {
937 _kwargs = {
933 '_': self._,
938 '_': self._,
934 'h': self.h,
939 'h': self.h,
935 'c': self.c,
940 'c': self.c,
936 'request': self.request,
941 'request': self.request,
937 'ungettext': self.ungettext,
942 'ungettext': self.ungettext,
938 }
943 }
939 _kwargs.update(kwargs)
944 _kwargs.update(kwargs)
940 return _kwargs
945 return _kwargs
941
946
942 def _render_with_exc(self, render_func, args, kwargs):
947 def _render_with_exc(self, render_func, args, kwargs):
943 try:
948 try:
944 return render_func.render(*args, **kwargs)
949 return render_func.render(*args, **kwargs)
945 except:
950 except:
946 log.error(exceptions.text_error_template().render())
951 log.error(exceptions.text_error_template().render())
947 raise
952 raise
948
953
949 def _get_template(self, template_obj, def_name):
954 def _get_template(self, template_obj, def_name):
950 if def_name:
955 if def_name:
951 tmpl = template_obj.get_def(def_name)
956 tmpl = template_obj.get_def(def_name)
952 else:
957 else:
953 tmpl = template_obj
958 tmpl = template_obj
954 return tmpl
959 return tmpl
955
960
956 def render(self, def_name, *args, **kwargs):
961 def render(self, def_name, *args, **kwargs):
957 lookup_obj = self._mako_lookup()
962 lookup_obj = self._mako_lookup()
958 tmpl = self._get_template(lookup_obj, def_name=def_name)
963 tmpl = self._get_template(lookup_obj, def_name=def_name)
959 kwargs = self._update_kwargs_for_render(kwargs)
964 kwargs = self._update_kwargs_for_render(kwargs)
960 return self._render_with_exc(tmpl, args, kwargs)
965 return self._render_with_exc(tmpl, args, kwargs)
961
966
962 def __call__(self, tmpl, *args, **kwargs):
967 def __call__(self, tmpl, *args, **kwargs):
963 return self.render(tmpl, *args, **kwargs)
968 return self.render(tmpl, *args, **kwargs)
964
969
965
970
966 def password_changed(auth_user, session):
971 def password_changed(auth_user, session):
967 # Never report password change in case of default user or anonymous user.
972 # Never report password change in case of default user or anonymous user.
968 if auth_user.username == User.DEFAULT_USER or auth_user.user_id is None:
973 if auth_user.username == User.DEFAULT_USER or auth_user.user_id is None:
969 return False
974 return False
970
975
971 password_hash = md5(auth_user.password) if auth_user.password else None
976 password_hash = md5(auth_user.password) if auth_user.password else None
972 rhodecode_user = session.get('rhodecode_user', {})
977 rhodecode_user = session.get('rhodecode_user', {})
973 session_password_hash = rhodecode_user.get('password', '')
978 session_password_hash = rhodecode_user.get('password', '')
974 return password_hash != session_password_hash
979 return password_hash != session_password_hash
975
980
976
981
977 def read_opensource_licenses():
982 def read_opensource_licenses():
978 global _license_cache
983 global _license_cache
979
984
980 if not _license_cache:
985 if not _license_cache:
981 licenses = pkg_resources.resource_string(
986 licenses = pkg_resources.resource_string(
982 'rhodecode', 'config/licenses.json')
987 'rhodecode', 'config/licenses.json')
983 _license_cache = json.loads(licenses)
988 _license_cache = json.loads(licenses)
984
989
985 return _license_cache
990 return _license_cache
986
991
987
992
988 def get_registry(request):
993 def get_registry(request):
989 """
994 """
990 Utility to get the pyramid registry from a request. During migration to
995 Utility to get the pyramid registry from a request. During migration to
991 pyramid we sometimes want to use the pyramid registry from pylons context.
996 pyramid we sometimes want to use the pyramid registry from pylons context.
992 Therefore this utility returns `request.registry` for pyramid requests and
997 Therefore this utility returns `request.registry` for pyramid requests and
993 uses `get_current_registry()` for pylons requests.
998 uses `get_current_registry()` for pylons requests.
994 """
999 """
995 try:
1000 try:
996 return request.registry
1001 return request.registry
997 except AttributeError:
1002 except AttributeError:
998 return get_current_registry()
1003 return get_current_registry()
General Comments 0
You need to be logged in to leave comments. Login now