##// END OF EJS Templates
vcs: Clean up the shadow-repo-expose code and make some nicer comments.
Martin Bornhold -
r904:8e8700f6 default
parent child Browse files
Show More
@@ -1,511 +1,523 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2014-2016 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21 """
22 22 SimpleVCS middleware for handling protocol request (push/clone etc.)
23 23 It's implemented with basic auth function
24 24 """
25 25
26 26 import os
27 27 import logging
28 28 import importlib
29 29 import re
30 30 from functools import wraps
31 31
32 32 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
33 33 from webob.exc import (
34 34 HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
35 35
36 36 import rhodecode
37 37 from rhodecode.authentication.base import authenticate, VCS_TYPE
38 38 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
39 39 from rhodecode.lib.base import BasicAuth, get_ip_addr, vcs_operation_context
40 40 from rhodecode.lib.exceptions import (
41 41 HTTPLockedRC, HTTPRequirementError, UserCreationError,
42 42 NotAllowedToCreateUserError)
43 43 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
44 44 from rhodecode.lib.middleware import appenlight
45 45 from rhodecode.lib.middleware.utils import scm_app
46 46 from rhodecode.lib.utils import (
47 47 is_valid_repo, get_rhodecode_realm, get_rhodecode_base_path, SLUG_RE)
48 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool
48 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool, safe_unicode
49 49 from rhodecode.lib.vcs.conf import settings as vcs_settings
50 50 from rhodecode.lib.vcs.backends import base
51 51 from rhodecode.model import meta
52 52 from rhodecode.model.db import User, Repository, PullRequest
53 53 from rhodecode.model.scm import ScmModel
54 54 from rhodecode.model.pull_request import PullRequestModel
55 55
56 56
57 57 log = logging.getLogger(__name__)
58 58
59 59
60 60 def initialize_generator(factory):
61 61 """
62 62 Initializes the returned generator by draining its first element.
63 63
64 64 This can be used to give a generator an initializer, which is the code
65 65 up to the first yield statement. This decorator enforces that the first
66 66 produced element has the value ``"__init__"`` to make its special
67 67 purpose very explicit in the using code.
68 68 """
69 69
70 70 @wraps(factory)
71 71 def wrapper(*args, **kwargs):
72 72 gen = factory(*args, **kwargs)
73 73 try:
74 74 init = gen.next()
75 75 except StopIteration:
76 76 raise ValueError('Generator must yield at least one element.')
77 77 if init != "__init__":
78 78 raise ValueError('First yielded element must be "__init__".')
79 79 return gen
80 80 return wrapper
81 81
82 82
83 83 class SimpleVCS(object):
84 84 """Common functionality for SCM HTTP handlers."""
85 85
86 86 SCM = 'unknown'
87 87
88 88 acl_repo_name = None
89 89 url_repo_name = None
90 90 vcs_repo_name = None
91 91
92 92 # We have to handle requests to shadow repositories different than requests
93 93 # to normal repositories. Therefore we have to distinguish them. To do this
94 94 # we use this regex which will match only on URLs pointing to shadow
95 95 # repositories.
96 96 shadow_repo_re = re.compile(
97 97 '(?P<groups>(?:{slug_pat})(?:/{slug_pat})*)' # repo groups
98 98 '/(?P<target>{slug_pat})' # target repo
99 99 '/pull-request/(?P<pr_id>\d+)' # pull request
100 100 '/repository$' # shadow repo
101 101 .format(slug_pat=SLUG_RE.pattern))
102 102
103 103 def __init__(self, application, config, registry):
104 104 self.registry = registry
105 105 self.application = application
106 106 self.config = config
107 107 # re-populated by specialized middleware
108 108 self.repo_vcs_config = base.Config()
109 109
110 110 # base path of repo locations
111 111 self.basepath = get_rhodecode_base_path()
112 112 # authenticate this VCS request using authfunc
113 113 auth_ret_code_detection = \
114 114 str2bool(self.config.get('auth_ret_code_detection', False))
115 115 self.authenticate = BasicAuth(
116 116 '', authenticate, registry, config.get('auth_ret_code'),
117 117 auth_ret_code_detection)
118 118 self.ip_addr = '0.0.0.0'
119 119
120 120 def set_repo_names(self, environ):
121 121 """
122 122 This will populate the attributes acl_repo_name, url_repo_name,
123 vcs_repo_name and is_shadow_repo on the current instance.
124 """
125 # Get url repo name from environment.
126 self.url_repo_name = self._get_repository_name(environ)
123 vcs_repo_name and is_shadow_repo. In case of requests to normal (non
124 shadow) repositories all names are equal. In case of requests to a
125 shadow repository the acl-name points to the target repo of the pull
126 request and the vcs-name points to the shadow repo file system path.
127 The url-name is always the URL used by the vcs client program.
127 128
128 # Check if this is a request to a shadow repository. In case of a
129 # shadow repo set vcs_repo_name to the file system path pointing to the
130 # shadow repo. And set acl_repo_name to the pull request target repo
131 # because we use the target repo for permission checks. Otherwise all
132 # names are equal.
129 Example in case of a shadow repo:
130 acl_repo_name = RepoGroup/MyRepo
131 url_repo_name = RepoGroup/MyRepo/pull-request/3/repository
132 vcs_repo_name = /repo/base/path/RepoGroup/.__shadow_MyRepo_pr-3'
133 """
134 # First we set the repo name from URL for all attributes. This is the
135 # default if handling normal (non shadow) repo requests.
136 self.url_repo_name = self._get_repository_name(environ)
137 self.acl_repo_name = self.vcs_repo_name = self.url_repo_name
138 self.is_shadow_repo = False
139
140 # Check if this is a request to a shadow repository.
133 141 match = self.shadow_repo_re.match(self.url_repo_name)
134 # TODO: martinb: Think about checking the target repo from PR against
135 # the part in the URL. Otherwise we only rely on the PR id in the URL
136 # and the variable parts can be anything. This will lead to 500 errors
137 # from the VCSServer.
138 142 if match:
139 # Get pull request instance.
140 143 match_dict = match.groupdict()
141 pr_id = match_dict['pr_id']
142 pull_request = PullRequest.get(pr_id)
144
145 # Build acl repo name from regex match.
146 acl_repo_name = safe_unicode(
147 '{groups}/{target}'.format(**match_dict))
143 148
149 # Retrieve pull request instance by ID from regex match.
150 pull_request = PullRequest.get(match_dict['pr_id'])
151
152 # Only proceed if we got a pull request and if acl repo name from
153 # URL equals the target repo name of the pull request.
154 if pull_request and acl_repo_name == pull_request.target_repo.repo_name:
144 155 # Get file system path to shadow repository.
145 156 workspace_id = PullRequestModel()._workspace_id(pull_request)
146 157 target_vcs = pull_request.target_repo.scm_instance()
147 158 vcs_repo_name = target_vcs._get_shadow_repository_path(
148 159 workspace_id)
149 160
150 161 # Store names for later usage.
151 162 self.vcs_repo_name = vcs_repo_name
152 self.acl_repo_name = pull_request.target_repo.repo_name
163 self.acl_repo_name = acl_repo_name
153 164 self.is_shadow_repo = True
154 else:
155 # All names are equal for normal (non shadow) repositories.
156 self.acl_repo_name = self.url_repo_name
157 self.vcs_repo_name = self.url_repo_name
158 self.is_shadow_repo = False
165
166 log.debug('Repository names: %s', {
167 'acl_repo_name': self.acl_repo_name,
168 'url_repo_name': self.url_repo_name,
169 'vcs_repo_name': self.vcs_repo_name,
170 })
159 171
160 172 @property
161 173 def scm_app(self):
162 174 custom_implementation = self.config.get('vcs.scm_app_implementation')
163 175 if custom_implementation and custom_implementation != 'pyro4':
164 176 log.info(
165 177 "Using custom implementation of scm_app: %s",
166 178 custom_implementation)
167 179 scm_app_impl = importlib.import_module(custom_implementation)
168 180 else:
169 181 scm_app_impl = scm_app
170 182 return scm_app_impl
171 183
172 184 def _get_by_id(self, repo_name):
173 185 """
174 186 Gets a special pattern _<ID> from clone url and tries to replace it
175 187 with a repository_name for support of _<ID> non changeable urls
176 188 """
177 189
178 190 data = repo_name.split('/')
179 191 if len(data) >= 2:
180 192 from rhodecode.model.repo import RepoModel
181 193 by_id_match = RepoModel().get_repo_by_id(repo_name)
182 194 if by_id_match:
183 195 data[1] = by_id_match.repo_name
184 196
185 197 return safe_str('/'.join(data))
186 198
187 199 def _invalidate_cache(self, repo_name):
188 200 """
189 201 Set's cache for this repository for invalidation on next access
190 202
191 203 :param repo_name: full repo name, also a cache key
192 204 """
193 205 ScmModel().mark_for_invalidation(repo_name)
194 206
195 207 def is_valid_and_existing_repo(self, repo_name, base_path, scm_type):
196 208 db_repo = Repository.get_by_repo_name(repo_name)
197 209 if not db_repo:
198 210 log.debug('Repository `%s` not found inside the database.',
199 211 repo_name)
200 212 return False
201 213
202 214 if db_repo.repo_type != scm_type:
203 215 log.warning(
204 216 'Repository `%s` have incorrect scm_type, expected %s got %s',
205 217 repo_name, db_repo.repo_type, scm_type)
206 218 return False
207 219
208 220 return is_valid_repo(repo_name, base_path, expect_scm=scm_type)
209 221
210 222 def valid_and_active_user(self, user):
211 223 """
212 224 Checks if that user is not empty, and if it's actually object it checks
213 225 if he's active.
214 226
215 227 :param user: user object or None
216 228 :return: boolean
217 229 """
218 230 if user is None:
219 231 return False
220 232
221 233 elif user.active:
222 234 return True
223 235
224 236 return False
225 237
226 238 def _check_permission(self, action, user, repo_name, ip_addr=None):
227 239 """
228 240 Checks permissions using action (push/pull) user and repository
229 241 name
230 242
231 243 :param action: push or pull action
232 244 :param user: user instance
233 245 :param repo_name: repository name
234 246 """
235 247 # check IP
236 248 inherit = user.inherit_default_permissions
237 249 ip_allowed = AuthUser.check_ip_allowed(user.user_id, ip_addr,
238 250 inherit_from_default=inherit)
239 251 if ip_allowed:
240 252 log.info('Access for IP:%s allowed', ip_addr)
241 253 else:
242 254 return False
243 255
244 256 if action == 'push':
245 257 if not HasPermissionAnyMiddleware('repository.write',
246 258 'repository.admin')(user,
247 259 repo_name):
248 260 return False
249 261
250 262 else:
251 263 # any other action need at least read permission
252 264 if not HasPermissionAnyMiddleware('repository.read',
253 265 'repository.write',
254 266 'repository.admin')(user,
255 267 repo_name):
256 268 return False
257 269
258 270 return True
259 271
260 272 def _check_ssl(self, environ, start_response):
261 273 """
262 274 Checks the SSL check flag and returns False if SSL is not present
263 275 and required True otherwise
264 276 """
265 277 org_proto = environ['wsgi._org_proto']
266 278 # check if we have SSL required ! if not it's a bad request !
267 279 require_ssl = str2bool(self.repo_vcs_config.get('web', 'push_ssl'))
268 280 if require_ssl and org_proto == 'http':
269 281 log.debug('proto is %s and SSL is required BAD REQUEST !',
270 282 org_proto)
271 283 return False
272 284 return True
273 285
274 286 def __call__(self, environ, start_response):
275 287 try:
276 288 return self._handle_request(environ, start_response)
277 289 except Exception:
278 290 log.exception("Exception while handling request")
279 291 appenlight.track_exception(environ)
280 292 return HTTPInternalServerError()(environ, start_response)
281 293 finally:
282 294 meta.Session.remove()
283 295
284 296 def _handle_request(self, environ, start_response):
285 297
286 298 if not self._check_ssl(environ, start_response):
287 299 reason = ('SSL required, while RhodeCode was unable '
288 300 'to detect this as SSL request')
289 301 log.debug('User not allowed to proceed, %s', reason)
290 302 return HTTPNotAcceptable(reason)(environ, start_response)
291 303
292 304 if not self.url_repo_name:
293 305 log.warning('Repository name is empty: %s', self.url_repo_name)
294 306 # failed to get repo name, we fail now
295 307 return HTTPNotFound()(environ, start_response)
296 308 log.debug('Extracted repo name is %s', self.url_repo_name)
297 309
298 310 ip_addr = get_ip_addr(environ)
299 311 username = None
300 312
301 313 # skip passing error to error controller
302 314 environ['pylons.status_code_redirect'] = True
303 315
304 316 # ======================================================================
305 317 # GET ACTION PULL or PUSH
306 318 # ======================================================================
307 319 action = self._get_action(environ)
308 320
309 321 # ======================================================================
310 322 # Check if this is a request to a shadow repository of a pull request.
311 323 # In this case only pull action is allowed.
312 324 # ======================================================================
313 325 if self.is_shadow_repo and action != 'pull':
314 326 reason = 'Only pull action is allowed for shadow repositories.'
315 327 log.debug('User not allowed to proceed, %s', reason)
316 328 return HTTPNotAcceptable(reason)(environ, start_response)
317 329
318 330 # ======================================================================
319 331 # CHECK ANONYMOUS PERMISSION
320 332 # ======================================================================
321 333 if action in ['pull', 'push']:
322 334 anonymous_user = User.get_default_user()
323 335 username = anonymous_user.username
324 336 if anonymous_user.active:
325 337 # ONLY check permissions if the user is activated
326 338 anonymous_perm = self._check_permission(
327 339 action, anonymous_user, self.acl_repo_name, ip_addr)
328 340 else:
329 341 anonymous_perm = False
330 342
331 343 if not anonymous_user.active or not anonymous_perm:
332 344 if not anonymous_user.active:
333 345 log.debug('Anonymous access is disabled, running '
334 346 'authentication')
335 347
336 348 if not anonymous_perm:
337 349 log.debug('Not enough credentials to access this '
338 350 'repository as anonymous user')
339 351
340 352 username = None
341 353 # ==============================================================
342 354 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
343 355 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
344 356 # ==============================================================
345 357
346 358 # try to auth based on environ, container auth methods
347 359 log.debug('Running PRE-AUTH for container based authentication')
348 360 pre_auth = authenticate(
349 361 '', '', environ, VCS_TYPE, registry=self.registry)
350 362 if pre_auth and pre_auth.get('username'):
351 363 username = pre_auth['username']
352 364 log.debug('PRE-AUTH got %s as username', username)
353 365
354 366 # If not authenticated by the container, running basic auth
355 367 if not username:
356 368 self.authenticate.realm = get_rhodecode_realm()
357 369
358 370 try:
359 371 result = self.authenticate(environ)
360 372 except (UserCreationError, NotAllowedToCreateUserError) as e:
361 373 log.error(e)
362 374 reason = safe_str(e)
363 375 return HTTPNotAcceptable(reason)(environ, start_response)
364 376
365 377 if isinstance(result, str):
366 378 AUTH_TYPE.update(environ, 'basic')
367 379 REMOTE_USER.update(environ, result)
368 380 username = result
369 381 else:
370 382 return result.wsgi_application(environ, start_response)
371 383
372 384 # ==============================================================
373 385 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
374 386 # ==============================================================
375 387 user = User.get_by_username(username)
376 388 if not self.valid_and_active_user(user):
377 389 return HTTPForbidden()(environ, start_response)
378 390 username = user.username
379 391 user.update_lastactivity()
380 392 meta.Session().commit()
381 393
382 394 # check user attributes for password change flag
383 395 user_obj = user
384 396 if user_obj and user_obj.username != User.DEFAULT_USER and \
385 397 user_obj.user_data.get('force_password_change'):
386 398 reason = 'password change required'
387 399 log.debug('User not allowed to authenticate, %s', reason)
388 400 return HTTPNotAcceptable(reason)(environ, start_response)
389 401
390 402 # check permissions for this repository
391 403 perm = self._check_permission(
392 404 action, user, self.acl_repo_name, ip_addr)
393 405 if not perm:
394 406 return HTTPForbidden()(environ, start_response)
395 407
396 408 # extras are injected into UI object and later available
397 409 # in hooks executed by rhodecode
398 410 check_locking = _should_check_locking(environ.get('QUERY_STRING'))
399 411 extras = vcs_operation_context(
400 412 environ, repo_name=self.acl_repo_name, username=username,
401 413 action=action, scm=self.SCM, check_locking=check_locking,
402 414 is_shadow_repo=self.is_shadow_repo
403 415 )
404 416
405 417 # ======================================================================
406 418 # REQUEST HANDLING
407 419 # ======================================================================
408 420 str_repo_name = safe_str(self.url_repo_name)
409 421 repo_path = os.path.join(
410 422 safe_str(self.basepath), safe_str(self.vcs_repo_name))
411 423 log.debug('Repository path is %s', repo_path)
412 424
413 425 fix_PATH()
414 426
415 427 log.info(
416 428 '%s action on %s repo "%s" by "%s" from %s',
417 429 action, self.SCM, str_repo_name, safe_str(username), ip_addr)
418 430
419 431 return self._generate_vcs_response(
420 432 environ, start_response, repo_path, self.url_repo_name, extras, action)
421 433
422 434 @initialize_generator
423 435 def _generate_vcs_response(
424 436 self, environ, start_response, repo_path, repo_name, extras,
425 437 action):
426 438 """
427 439 Returns a generator for the response content.
428 440
429 441 This method is implemented as a generator, so that it can trigger
430 442 the cache validation after all content sent back to the client. It
431 443 also handles the locking exceptions which will be triggered when
432 444 the first chunk is produced by the underlying WSGI application.
433 445 """
434 446 callback_daemon, extras = self._prepare_callback_daemon(extras)
435 447 config = self._create_config(extras, self.acl_repo_name)
436 448 log.debug('HOOKS extras is %s', extras)
437 449 app = self._create_wsgi_app(repo_path, repo_name, config)
438 450
439 451 try:
440 452 with callback_daemon:
441 453 try:
442 454 response = app(environ, start_response)
443 455 finally:
444 456 # This statement works together with the decorator
445 457 # "initialize_generator" above. The decorator ensures that
446 458 # we hit the first yield statement before the generator is
447 459 # returned back to the WSGI server. This is needed to
448 460 # ensure that the call to "app" above triggers the
449 461 # needed callback to "start_response" before the
450 462 # generator is actually used.
451 463 yield "__init__"
452 464
453 465 for chunk in response:
454 466 yield chunk
455 467 except Exception as exc:
456 468 # TODO: johbo: Improve "translating" back the exception.
457 469 if getattr(exc, '_vcs_kind', None) == 'repo_locked':
458 470 exc = HTTPLockedRC(*exc.args)
459 471 _code = rhodecode.CONFIG.get('lock_ret_code')
460 472 log.debug('Repository LOCKED ret code %s!', (_code,))
461 473 elif getattr(exc, '_vcs_kind', None) == 'requirement':
462 474 log.debug(
463 475 'Repository requires features unknown to this Mercurial')
464 476 exc = HTTPRequirementError(*exc.args)
465 477 else:
466 478 raise
467 479
468 480 for chunk in exc(environ, start_response):
469 481 yield chunk
470 482 finally:
471 483 # invalidate cache on push
472 484 try:
473 485 if action == 'push':
474 486 self._invalidate_cache(repo_name)
475 487 finally:
476 488 meta.Session.remove()
477 489
478 490 def _get_repository_name(self, environ):
479 491 """Get repository name out of the environmnent
480 492
481 493 :param environ: WSGI environment
482 494 """
483 495 raise NotImplementedError()
484 496
485 497 def _get_action(self, environ):
486 498 """Map request commands into a pull or push command.
487 499
488 500 :param environ: WSGI environment
489 501 """
490 502 raise NotImplementedError()
491 503
492 504 def _create_wsgi_app(self, repo_path, repo_name, config):
493 505 """Return the WSGI app that will finally handle the request."""
494 506 raise NotImplementedError()
495 507
496 508 def _create_config(self, extras, repo_name):
497 509 """Create a Pyro safe config representation."""
498 510 raise NotImplementedError()
499 511
500 512 def _prepare_callback_daemon(self, extras):
501 513 return prepare_callback_daemon(
502 514 extras, protocol=vcs_settings.HOOKS_PROTOCOL,
503 515 use_direct_calls=vcs_settings.HOOKS_DIRECT_CALLS)
504 516
505 517
506 518 def _should_check_locking(query_string):
507 519 # this is kind of hacky, but due to how mercurial handles client-server
508 520 # server see all operation on commit; bookmarks, phases and
509 521 # obsolescence marker in different transaction, we don't want to check
510 522 # locking on those
511 523 return query_string not in ['cmd=listkeys']
@@ -1,204 +1,204 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2010-2016 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21 import gzip
22 22 import shutil
23 23 import logging
24 24 import tempfile
25 25 import urlparse
26 26
27 27 from webob.exc import HTTPNotFound
28 28
29 29 import rhodecode
30 30 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
31 31 from rhodecode.lib.middleware.simplegit import SimpleGit, GIT_PROTO_PAT
32 32 from rhodecode.lib.middleware.simplehg import SimpleHg
33 33 from rhodecode.lib.middleware.simplesvn import SimpleSvn
34 34 from rhodecode.model.settings import VcsSettingsModel
35 35
36 36 log = logging.getLogger(__name__)
37 37
38 38
39 39 def is_git(environ):
40 40 """
41 41 Returns True if requests should be handled by GIT wsgi middleware
42 42 """
43 43 is_git_path = GIT_PROTO_PAT.match(environ['PATH_INFO'])
44 44 log.debug(
45 45 'request path: `%s` detected as GIT PROTOCOL %s', environ['PATH_INFO'],
46 46 is_git_path is not None)
47 47
48 48 return is_git_path
49 49
50 50
51 51 def is_hg(environ):
52 52 """
53 53 Returns True if requests target is mercurial server - header
54 54 ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
55 55 """
56 56 is_hg_path = False
57 57
58 58 http_accept = environ.get('HTTP_ACCEPT')
59 59
60 60 if http_accept and http_accept.startswith('application/mercurial'):
61 61 query = urlparse.parse_qs(environ['QUERY_STRING'])
62 62 if 'cmd' in query:
63 63 is_hg_path = True
64 64
65 65 log.debug(
66 66 'request path: `%s` detected as HG PROTOCOL %s', environ['PATH_INFO'],
67 67 is_hg_path)
68 68
69 69 return is_hg_path
70 70
71 71
72 72 def is_svn(environ):
73 73 """
74 74 Returns True if requests target is Subversion server
75 75 """
76 76 http_dav = environ.get('HTTP_DAV', '')
77 77 magic_path_segment = rhodecode.CONFIG.get(
78 78 'rhodecode_subversion_magic_path', '/!svn')
79 79 is_svn_path = (
80 80 'subversion' in http_dav or
81 81 magic_path_segment in environ['PATH_INFO'])
82 82 log.debug(
83 83 'request path: `%s` detected as SVN PROTOCOL %s', environ['PATH_INFO'],
84 84 is_svn_path)
85 85
86 86 return is_svn_path
87 87
88 88
89 89 class GunzipMiddleware(object):
90 90 """
91 91 WSGI middleware that unzips gzip-encoded requests before
92 92 passing on to the underlying application.
93 93 """
94 94
95 95 def __init__(self, application):
96 96 self.app = application
97 97
98 98 def __call__(self, environ, start_response):
99 99 accepts_encoding_header = environ.get('HTTP_CONTENT_ENCODING', b'')
100 100
101 101 if b'gzip' in accepts_encoding_header:
102 102 log.debug('gzip detected, now running gunzip wrapper')
103 103 wsgi_input = environ['wsgi.input']
104 104
105 105 if not hasattr(environ['wsgi.input'], 'seek'):
106 106 # The gzip implementation in the standard library of Python 2.x
107 107 # requires the '.seek()' and '.tell()' methods to be available
108 108 # on the input stream. Read the data into a temporary file to
109 109 # work around this limitation.
110 110
111 111 wsgi_input = tempfile.SpooledTemporaryFile(64 * 1024 * 1024)
112 112 shutil.copyfileobj(environ['wsgi.input'], wsgi_input)
113 113 wsgi_input.seek(0)
114 114
115 115 environ['wsgi.input'] = gzip.GzipFile(fileobj=wsgi_input, mode='r')
116 116 # since we "Ungzipped" the content we say now it's no longer gzip
117 117 # content encoding
118 118 del environ['HTTP_CONTENT_ENCODING']
119 119
120 120 # content length has changes ? or i'm not sure
121 121 if 'CONTENT_LENGTH' in environ:
122 122 del environ['CONTENT_LENGTH']
123 123 else:
124 124 log.debug('content not gzipped, gzipMiddleware passing '
125 125 'request further')
126 126 return self.app(environ, start_response)
127 127
128 128
129 129 class VCSMiddleware(object):
130 130
131 131 def __init__(self, app, config, appenlight_client, registry):
132 132 self.application = app
133 133 self.config = config
134 134 self.appenlight_client = appenlight_client
135 135 self.registry = registry
136 136 self.use_gzip = True
137 137 # order in which we check the middlewares, based on vcs.backends config
138 138 self.check_middlewares = config['vcs.backends']
139 139 self.checks = {
140 140 'hg': (is_hg, SimpleHg),
141 141 'git': (is_git, SimpleGit),
142 142 'svn': (is_svn, SimpleSvn),
143 143 }
144 144
145 145 def vcs_config(self, repo_name=None):
146 146 """
147 147 returns serialized VcsSettings
148 148 """
149 149 return VcsSettingsModel(repo=repo_name).get_ui_settings_as_config_obj()
150 150
151 151 def wrap_in_gzip_if_enabled(self, app, config):
152 152 if self.use_gzip:
153 153 app = GunzipMiddleware(app)
154 154 return app
155 155
156 156 def _get_handler_app(self, environ):
157 157 app = None
158 158 log.debug('Checking vcs types in order: %r', self.check_middlewares)
159 159 for vcs_type in self.check_middlewares:
160 160 vcs_check, handler = self.checks[vcs_type]
161 161 if vcs_check(environ):
162 162 log.debug(
163 163 'Found VCS Middleware to handle the request %s', handler)
164 164 app = handler(self.application, self.config, self.registry)
165 165 break
166 166
167 167 return app
168 168
169 169 def __call__(self, environ, start_response):
170 170 # check if we handle one of interesting protocols, optionally extract
171 171 # specific vcsSettings and allow changes of how things are wrapped
172 172 vcs_handler = self._get_handler_app(environ)
173 173 if vcs_handler:
174 174 # translate the _REPO_ID into real repo NAME for usage
175 175 # in middleware
176 176 environ['PATH_INFO'] = vcs_handler._get_by_id(environ['PATH_INFO'])
177 177
178 # Set repo names for permission checks, vcs and web interaction.
178 # Set acl, url and vcs repo names.
179 179 vcs_handler.set_repo_names(environ)
180 180
181 181 # check for type, presence in database and on filesystem
182 182 if not vcs_handler.is_valid_and_existing_repo(
183 183 vcs_handler.acl_repo_name,
184 184 vcs_handler.basepath,
185 185 vcs_handler.SCM):
186 186 return HTTPNotFound()(environ, start_response)
187 187
188 188 # TODO: johbo: Needed for the Pyro4 backend and Mercurial only.
189 189 # Remove once we fully switched to the HTTP backend.
190 190 environ['REPO_NAME'] = vcs_handler.url_repo_name
191 191
192 192 # register repo config back to the handler
193 193 vcs_handler.repo_vcs_config = self.vcs_config(
194 194 vcs_handler.acl_repo_name)
195 195
196 196 # Wrap handler in middlewares if they are enabled.
197 197 vcs_handler = self.wrap_in_gzip_if_enabled(
198 198 vcs_handler, self.config)
199 199 vcs_handler, _ = wrap_in_appenlight_if_enabled(
200 200 vcs_handler, self.config, self.appenlight_client)
201 201
202 202 return vcs_handler(environ, start_response)
203 203
204 204 return self.application(environ, start_response)
@@ -1,998 +1,1003 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2010-2016 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21 """
22 22 Utilities library for RhodeCode
23 23 """
24 24
25 25 import datetime
26 26 import decorator
27 27 import json
28 28 import logging
29 29 import os
30 30 import re
31 31 import shutil
32 32 import tempfile
33 33 import traceback
34 34 import tarfile
35 35 import warnings
36 36 from os.path import join as jn
37 37
38 38 import paste
39 39 import pkg_resources
40 40 from paste.script.command import Command, BadCommand
41 41 from webhelpers.text import collapse, remove_formatting, strip_tags
42 42 from mako import exceptions
43 43 from pyramid.threadlocal import get_current_registry
44 44
45 45 from rhodecode.lib.fakemod import create_module
46 46 from rhodecode.lib.vcs.backends.base import Config
47 47 from rhodecode.lib.vcs.exceptions import VCSError
48 48 from rhodecode.lib.vcs.utils.helpers import get_scm, get_scm_backend
49 49 from rhodecode.lib.utils2 import (
50 50 safe_str, safe_unicode, get_current_rhodecode_user, md5)
51 51 from rhodecode.model import meta
52 52 from rhodecode.model.db import (
53 53 Repository, User, RhodeCodeUi, UserLog, RepoGroup, UserGroup)
54 54 from rhodecode.model.meta import Session
55 55
56 56
57 57 log = logging.getLogger(__name__)
58 58
59 59 REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}__.*')
60 60
61 # String of characters which are not allowed in repo/group slugs.
61 # String which contains characters that are not allowed in slug names for
62 # repositories or repository groups. It is properly escaped to use it in
63 # regular expressions.
62 64 SLUG_BAD_CHARS = re.escape('`?=[]\;\'"<>,/~!@#$%^&*()+{}|:')
65
63 66 # Regex that matches forbidden characters in repo/group slugs.
64 67 SLUG_BAD_CHAR_RE = re.compile('[{}]'.format(SLUG_BAD_CHARS))
68
65 69 # Regex that matches allowed characters in repo/group slugs.
66 70 SLUG_GOOD_CHAR_RE = re.compile('[^{}]'.format(SLUG_BAD_CHARS))
71
67 72 # Regex that matches whole repo/group slugs.
68 73 SLUG_RE = re.compile('[^{}]+'.format(SLUG_BAD_CHARS))
69 74
70 75 _license_cache = None
71 76
72 77
73 78 def repo_name_slug(value):
74 79 """
75 80 Return slug of name of repository
76 81 This function is called on each creation/modification
77 82 of repository to prevent bad names in repo
78 83 """
79 84 replacement_char = '-'
80 85
81 86 slug = remove_formatting(value)
82 87 slug = SLUG_BAD_CHAR_RE.sub(replacement_char, slug)
83 88 slug = collapse(slug, replacement_char)
84 89 return slug
85 90
86 91
87 92 #==============================================================================
88 93 # PERM DECORATOR HELPERS FOR EXTRACTING NAMES FOR PERM CHECKS
89 94 #==============================================================================
90 95 def get_repo_slug(request):
91 96 _repo = request.environ['pylons.routes_dict'].get('repo_name')
92 97 if _repo:
93 98 _repo = _repo.rstrip('/')
94 99 return _repo
95 100
96 101
97 102 def get_repo_group_slug(request):
98 103 _group = request.environ['pylons.routes_dict'].get('group_name')
99 104 if _group:
100 105 _group = _group.rstrip('/')
101 106 return _group
102 107
103 108
104 109 def get_user_group_slug(request):
105 110 _group = request.environ['pylons.routes_dict'].get('user_group_id')
106 111 try:
107 112 _group = UserGroup.get(_group)
108 113 if _group:
109 114 _group = _group.users_group_name
110 115 except Exception:
111 116 log.debug(traceback.format_exc())
112 117 #catch all failures here
113 118 pass
114 119
115 120 return _group
116 121
117 122
118 123 def action_logger(user, action, repo, ipaddr='', sa=None, commit=False):
119 124 """
120 125 Action logger for various actions made by users
121 126
122 127 :param user: user that made this action, can be a unique username string or
123 128 object containing user_id attribute
124 129 :param action: action to log, should be on of predefined unique actions for
125 130 easy translations
126 131 :param repo: string name of repository or object containing repo_id,
127 132 that action was made on
128 133 :param ipaddr: optional ip address from what the action was made
129 134 :param sa: optional sqlalchemy session
130 135
131 136 """
132 137
133 138 if not sa:
134 139 sa = meta.Session()
135 140 # if we don't get explicit IP address try to get one from registered user
136 141 # in tmpl context var
137 142 if not ipaddr:
138 143 ipaddr = getattr(get_current_rhodecode_user(), 'ip_addr', '')
139 144
140 145 try:
141 146 if getattr(user, 'user_id', None):
142 147 user_obj = User.get(user.user_id)
143 148 elif isinstance(user, basestring):
144 149 user_obj = User.get_by_username(user)
145 150 else:
146 151 raise Exception('You have to provide a user object or a username')
147 152
148 153 if getattr(repo, 'repo_id', None):
149 154 repo_obj = Repository.get(repo.repo_id)
150 155 repo_name = repo_obj.repo_name
151 156 elif isinstance(repo, basestring):
152 157 repo_name = repo.lstrip('/')
153 158 repo_obj = Repository.get_by_repo_name(repo_name)
154 159 else:
155 160 repo_obj = None
156 161 repo_name = ''
157 162
158 163 user_log = UserLog()
159 164 user_log.user_id = user_obj.user_id
160 165 user_log.username = user_obj.username
161 166 action = safe_unicode(action)
162 167 user_log.action = action[:1200000]
163 168
164 169 user_log.repository = repo_obj
165 170 user_log.repository_name = repo_name
166 171
167 172 user_log.action_date = datetime.datetime.now()
168 173 user_log.user_ip = ipaddr
169 174 sa.add(user_log)
170 175
171 176 log.info('Logging action:`%s` on repo:`%s` by user:%s ip:%s',
172 177 action, safe_unicode(repo), user_obj, ipaddr)
173 178 if commit:
174 179 sa.commit()
175 180 except Exception:
176 181 log.error(traceback.format_exc())
177 182 raise
178 183
179 184
180 185 def get_filesystem_repos(path, recursive=False, skip_removed_repos=True):
181 186 """
182 187 Scans given path for repos and return (name,(type,path)) tuple
183 188
184 189 :param path: path to scan for repositories
185 190 :param recursive: recursive search and return names with subdirs in front
186 191 """
187 192
188 193 # remove ending slash for better results
189 194 path = path.rstrip(os.sep)
190 195 log.debug('now scanning in %s location recursive:%s...', path, recursive)
191 196
192 197 def _get_repos(p):
193 198 dirpaths = _get_dirpaths(p)
194 199 if not _is_dir_writable(p):
195 200 log.warning('repo path without write access: %s', p)
196 201
197 202 for dirpath in dirpaths:
198 203 if os.path.isfile(os.path.join(p, dirpath)):
199 204 continue
200 205 cur_path = os.path.join(p, dirpath)
201 206
202 207 # skip removed repos
203 208 if skip_removed_repos and REMOVED_REPO_PAT.match(dirpath):
204 209 continue
205 210
206 211 #skip .<somethin> dirs
207 212 if dirpath.startswith('.'):
208 213 continue
209 214
210 215 try:
211 216 scm_info = get_scm(cur_path)
212 217 yield scm_info[1].split(path, 1)[-1].lstrip(os.sep), scm_info
213 218 except VCSError:
214 219 if not recursive:
215 220 continue
216 221 #check if this dir containts other repos for recursive scan
217 222 rec_path = os.path.join(p, dirpath)
218 223 if os.path.isdir(rec_path):
219 224 for inner_scm in _get_repos(rec_path):
220 225 yield inner_scm
221 226
222 227 return _get_repos(path)
223 228
224 229
225 230 def _get_dirpaths(p):
226 231 try:
227 232 # OS-independable way of checking if we have at least read-only
228 233 # access or not.
229 234 dirpaths = os.listdir(p)
230 235 except OSError:
231 236 log.warning('ignoring repo path without read access: %s', p)
232 237 return []
233 238
234 239 # os.listpath has a tweak: If a unicode is passed into it, then it tries to
235 240 # decode paths and suddenly returns unicode objects itself. The items it
236 241 # cannot decode are returned as strings and cause issues.
237 242 #
238 243 # Those paths are ignored here until a solid solution for path handling has
239 244 # been built.
240 245 expected_type = type(p)
241 246
242 247 def _has_correct_type(item):
243 248 if type(item) is not expected_type:
244 249 log.error(
245 250 u"Ignoring path %s since it cannot be decoded into unicode.",
246 251 # Using "repr" to make sure that we see the byte value in case
247 252 # of support.
248 253 repr(item))
249 254 return False
250 255 return True
251 256
252 257 dirpaths = [item for item in dirpaths if _has_correct_type(item)]
253 258
254 259 return dirpaths
255 260
256 261
257 262 def _is_dir_writable(path):
258 263 """
259 264 Probe if `path` is writable.
260 265
261 266 Due to trouble on Cygwin / Windows, this is actually probing if it is
262 267 possible to create a file inside of `path`, stat does not produce reliable
263 268 results in this case.
264 269 """
265 270 try:
266 271 with tempfile.TemporaryFile(dir=path):
267 272 pass
268 273 except OSError:
269 274 return False
270 275 return True
271 276
272 277
273 278 def is_valid_repo(repo_name, base_path, expect_scm=None, explicit_scm=None):
274 279 """
275 280 Returns True if given path is a valid repository False otherwise.
276 281 If expect_scm param is given also, compare if given scm is the same
277 282 as expected from scm parameter. If explicit_scm is given don't try to
278 283 detect the scm, just use the given one to check if repo is valid
279 284
280 285 :param repo_name:
281 286 :param base_path:
282 287 :param expect_scm:
283 288 :param explicit_scm:
284 289
285 290 :return True: if given path is a valid repository
286 291 """
287 292 full_path = os.path.join(safe_str(base_path), safe_str(repo_name))
288 293 log.debug('Checking if `%s` is a valid path for repository', repo_name)
289 294
290 295 try:
291 296 if explicit_scm:
292 297 detected_scms = [get_scm_backend(explicit_scm)]
293 298 else:
294 299 detected_scms = get_scm(full_path)
295 300
296 301 if expect_scm:
297 302 return detected_scms[0] == expect_scm
298 303 log.debug('path: %s is an vcs object:%s', full_path, detected_scms)
299 304 return True
300 305 except VCSError:
301 306 log.debug('path: %s is not a valid repo !', full_path)
302 307 return False
303 308
304 309
305 310 def is_valid_repo_group(repo_group_name, base_path, skip_path_check=False):
306 311 """
307 312 Returns True if given path is a repository group, False otherwise
308 313
309 314 :param repo_name:
310 315 :param base_path:
311 316 """
312 317 full_path = os.path.join(safe_str(base_path), safe_str(repo_group_name))
313 318 log.debug('Checking if `%s` is a valid path for repository group',
314 319 repo_group_name)
315 320
316 321 # check if it's not a repo
317 322 if is_valid_repo(repo_group_name, base_path):
318 323 log.debug('Repo called %s exist, it is not a valid '
319 324 'repo group' % repo_group_name)
320 325 return False
321 326
322 327 try:
323 328 # we need to check bare git repos at higher level
324 329 # since we might match branches/hooks/info/objects or possible
325 330 # other things inside bare git repo
326 331 scm_ = get_scm(os.path.dirname(full_path))
327 332 log.debug('path: %s is a vcs object:%s, not valid '
328 333 'repo group' % (full_path, scm_))
329 334 return False
330 335 except VCSError:
331 336 pass
332 337
333 338 # check if it's a valid path
334 339 if skip_path_check or os.path.isdir(full_path):
335 340 log.debug('path: %s is a valid repo group !', full_path)
336 341 return True
337 342
338 343 log.debug('path: %s is not a valid repo group !', full_path)
339 344 return False
340 345
341 346
342 347 def ask_ok(prompt, retries=4, complaint='[y]es or [n]o please!'):
343 348 while True:
344 349 ok = raw_input(prompt)
345 350 if ok.lower() in ('y', 'ye', 'yes'):
346 351 return True
347 352 if ok.lower() in ('n', 'no', 'nop', 'nope'):
348 353 return False
349 354 retries = retries - 1
350 355 if retries < 0:
351 356 raise IOError
352 357 print(complaint)
353 358
354 359 # propagated from mercurial documentation
355 360 ui_sections = [
356 361 'alias', 'auth',
357 362 'decode/encode', 'defaults',
358 363 'diff', 'email',
359 364 'extensions', 'format',
360 365 'merge-patterns', 'merge-tools',
361 366 'hooks', 'http_proxy',
362 367 'smtp', 'patch',
363 368 'paths', 'profiling',
364 369 'server', 'trusted',
365 370 'ui', 'web', ]
366 371
367 372
368 373 def config_data_from_db(clear_session=True, repo=None):
369 374 """
370 375 Read the configuration data from the database and return configuration
371 376 tuples.
372 377 """
373 378 from rhodecode.model.settings import VcsSettingsModel
374 379
375 380 config = []
376 381
377 382 sa = meta.Session()
378 383 settings_model = VcsSettingsModel(repo=repo, sa=sa)
379 384
380 385 ui_settings = settings_model.get_ui_settings()
381 386
382 387 for setting in ui_settings:
383 388 if setting.active:
384 389 log.debug(
385 390 'settings ui from db: [%s] %s=%s',
386 391 setting.section, setting.key, setting.value)
387 392 config.append((
388 393 safe_str(setting.section), safe_str(setting.key),
389 394 safe_str(setting.value)))
390 395 if setting.key == 'push_ssl':
391 396 # force set push_ssl requirement to False, rhodecode
392 397 # handles that
393 398 config.append((
394 399 safe_str(setting.section), safe_str(setting.key), False))
395 400 if clear_session:
396 401 meta.Session.remove()
397 402
398 403 # TODO: mikhail: probably it makes no sense to re-read hooks information.
399 404 # It's already there and activated/deactivated
400 405 skip_entries = []
401 406 enabled_hook_classes = get_enabled_hook_classes(ui_settings)
402 407 if 'pull' not in enabled_hook_classes:
403 408 skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PULL))
404 409 if 'push' not in enabled_hook_classes:
405 410 skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PUSH))
406 411
407 412 config = [entry for entry in config if entry[:2] not in skip_entries]
408 413
409 414 return config
410 415
411 416
412 417 def make_db_config(clear_session=True, repo=None):
413 418 """
414 419 Create a :class:`Config` instance based on the values in the database.
415 420 """
416 421 config = Config()
417 422 config_data = config_data_from_db(clear_session=clear_session, repo=repo)
418 423 for section, option, value in config_data:
419 424 config.set(section, option, value)
420 425 return config
421 426
422 427
423 428 def get_enabled_hook_classes(ui_settings):
424 429 """
425 430 Return the enabled hook classes.
426 431
427 432 :param ui_settings: List of ui_settings as returned
428 433 by :meth:`VcsSettingsModel.get_ui_settings`
429 434
430 435 :return: a list with the enabled hook classes. The order is not guaranteed.
431 436 :rtype: list
432 437 """
433 438 enabled_hooks = []
434 439 active_hook_keys = [
435 440 key for section, key, value, active in ui_settings
436 441 if section == 'hooks' and active]
437 442
438 443 hook_names = {
439 444 RhodeCodeUi.HOOK_PUSH: 'push',
440 445 RhodeCodeUi.HOOK_PULL: 'pull',
441 446 RhodeCodeUi.HOOK_REPO_SIZE: 'repo_size'
442 447 }
443 448
444 449 for key in active_hook_keys:
445 450 hook = hook_names.get(key)
446 451 if hook:
447 452 enabled_hooks.append(hook)
448 453
449 454 return enabled_hooks
450 455
451 456
452 457 def set_rhodecode_config(config):
453 458 """
454 459 Updates pylons config with new settings from database
455 460
456 461 :param config:
457 462 """
458 463 from rhodecode.model.settings import SettingsModel
459 464 app_settings = SettingsModel().get_all_settings()
460 465
461 466 for k, v in app_settings.items():
462 467 config[k] = v
463 468
464 469
465 470 def get_rhodecode_realm():
466 471 """
467 472 Return the rhodecode realm from database.
468 473 """
469 474 from rhodecode.model.settings import SettingsModel
470 475 realm = SettingsModel().get_setting_by_name('realm')
471 476 return safe_str(realm.app_settings_value)
472 477
473 478
474 479 def get_rhodecode_base_path():
475 480 """
476 481 Returns the base path. The base path is the filesystem path which points
477 482 to the repository store.
478 483 """
479 484 from rhodecode.model.settings import SettingsModel
480 485 paths_ui = SettingsModel().get_ui_by_section_and_key('paths', '/')
481 486 return safe_str(paths_ui.ui_value)
482 487
483 488
484 489 def map_groups(path):
485 490 """
486 491 Given a full path to a repository, create all nested groups that this
487 492 repo is inside. This function creates parent-child relationships between
488 493 groups and creates default perms for all new groups.
489 494
490 495 :param paths: full path to repository
491 496 """
492 497 from rhodecode.model.repo_group import RepoGroupModel
493 498 sa = meta.Session()
494 499 groups = path.split(Repository.NAME_SEP)
495 500 parent = None
496 501 group = None
497 502
498 503 # last element is repo in nested groups structure
499 504 groups = groups[:-1]
500 505 rgm = RepoGroupModel(sa)
501 506 owner = User.get_first_super_admin()
502 507 for lvl, group_name in enumerate(groups):
503 508 group_name = '/'.join(groups[:lvl] + [group_name])
504 509 group = RepoGroup.get_by_group_name(group_name)
505 510 desc = '%s group' % group_name
506 511
507 512 # skip folders that are now removed repos
508 513 if REMOVED_REPO_PAT.match(group_name):
509 514 break
510 515
511 516 if group is None:
512 517 log.debug('creating group level: %s group_name: %s',
513 518 lvl, group_name)
514 519 group = RepoGroup(group_name, parent)
515 520 group.group_description = desc
516 521 group.user = owner
517 522 sa.add(group)
518 523 perm_obj = rgm._create_default_perms(group)
519 524 sa.add(perm_obj)
520 525 sa.flush()
521 526
522 527 parent = group
523 528 return group
524 529
525 530
526 531 def repo2db_mapper(initial_repo_list, remove_obsolete=False):
527 532 """
528 533 maps all repos given in initial_repo_list, non existing repositories
529 534 are created, if remove_obsolete is True it also checks for db entries
530 535 that are not in initial_repo_list and removes them.
531 536
532 537 :param initial_repo_list: list of repositories found by scanning methods
533 538 :param remove_obsolete: check for obsolete entries in database
534 539 """
535 540 from rhodecode.model.repo import RepoModel
536 541 from rhodecode.model.scm import ScmModel
537 542 from rhodecode.model.repo_group import RepoGroupModel
538 543 from rhodecode.model.settings import SettingsModel
539 544
540 545 sa = meta.Session()
541 546 repo_model = RepoModel()
542 547 user = User.get_first_super_admin()
543 548 added = []
544 549
545 550 # creation defaults
546 551 defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
547 552 enable_statistics = defs.get('repo_enable_statistics')
548 553 enable_locking = defs.get('repo_enable_locking')
549 554 enable_downloads = defs.get('repo_enable_downloads')
550 555 private = defs.get('repo_private')
551 556
552 557 for name, repo in initial_repo_list.items():
553 558 group = map_groups(name)
554 559 unicode_name = safe_unicode(name)
555 560 db_repo = repo_model.get_by_repo_name(unicode_name)
556 561 # found repo that is on filesystem not in RhodeCode database
557 562 if not db_repo:
558 563 log.info('repository %s not found, creating now', name)
559 564 added.append(name)
560 565 desc = (repo.description
561 566 if repo.description != 'unknown'
562 567 else '%s repository' % name)
563 568
564 569 db_repo = repo_model._create_repo(
565 570 repo_name=name,
566 571 repo_type=repo.alias,
567 572 description=desc,
568 573 repo_group=getattr(group, 'group_id', None),
569 574 owner=user,
570 575 enable_locking=enable_locking,
571 576 enable_downloads=enable_downloads,
572 577 enable_statistics=enable_statistics,
573 578 private=private,
574 579 state=Repository.STATE_CREATED
575 580 )
576 581 sa.commit()
577 582 # we added that repo just now, and make sure we updated server info
578 583 if db_repo.repo_type == 'git':
579 584 git_repo = db_repo.scm_instance()
580 585 # update repository server-info
581 586 log.debug('Running update server info')
582 587 git_repo._update_server_info()
583 588
584 589 db_repo.update_commit_cache()
585 590
586 591 config = db_repo._config
587 592 config.set('extensions', 'largefiles', '')
588 593 ScmModel().install_hooks(
589 594 db_repo.scm_instance(config=config),
590 595 repo_type=db_repo.repo_type)
591 596
592 597 removed = []
593 598 if remove_obsolete:
594 599 # remove from database those repositories that are not in the filesystem
595 600 for repo in sa.query(Repository).all():
596 601 if repo.repo_name not in initial_repo_list.keys():
597 602 log.debug("Removing non-existing repository found in db `%s`",
598 603 repo.repo_name)
599 604 try:
600 605 RepoModel(sa).delete(repo, forks='detach', fs_remove=False)
601 606 sa.commit()
602 607 removed.append(repo.repo_name)
603 608 except Exception:
604 609 # don't hold further removals on error
605 610 log.error(traceback.format_exc())
606 611 sa.rollback()
607 612
608 613 def splitter(full_repo_name):
609 614 _parts = full_repo_name.rsplit(RepoGroup.url_sep(), 1)
610 615 gr_name = None
611 616 if len(_parts) == 2:
612 617 gr_name = _parts[0]
613 618 return gr_name
614 619
615 620 initial_repo_group_list = [splitter(x) for x in
616 621 initial_repo_list.keys() if splitter(x)]
617 622
618 623 # remove from database those repository groups that are not in the
619 624 # filesystem due to parent child relationships we need to delete them
620 625 # in a specific order of most nested first
621 626 all_groups = [x.group_name for x in sa.query(RepoGroup).all()]
622 627 nested_sort = lambda gr: len(gr.split('/'))
623 628 for group_name in sorted(all_groups, key=nested_sort, reverse=True):
624 629 if group_name not in initial_repo_group_list:
625 630 repo_group = RepoGroup.get_by_group_name(group_name)
626 631 if (repo_group.children.all() or
627 632 not RepoGroupModel().check_exist_filesystem(
628 633 group_name=group_name, exc_on_failure=False)):
629 634 continue
630 635
631 636 log.info(
632 637 'Removing non-existing repository group found in db `%s`',
633 638 group_name)
634 639 try:
635 640 RepoGroupModel(sa).delete(group_name, fs_remove=False)
636 641 sa.commit()
637 642 removed.append(group_name)
638 643 except Exception:
639 644 # don't hold further removals on error
640 645 log.exception(
641 646 'Unable to remove repository group `%s`',
642 647 group_name)
643 648 sa.rollback()
644 649 raise
645 650
646 651 return added, removed
647 652
648 653
649 654 def get_default_cache_settings(settings):
650 655 cache_settings = {}
651 656 for key in settings.keys():
652 657 for prefix in ['beaker.cache.', 'cache.']:
653 658 if key.startswith(prefix):
654 659 name = key.split(prefix)[1].strip()
655 660 cache_settings[name] = settings[key].strip()
656 661 return cache_settings
657 662
658 663
659 664 # set cache regions for beaker so celery can utilise it
660 665 def add_cache(settings):
661 666 from rhodecode.lib import caches
662 667 cache_settings = {'regions': None}
663 668 # main cache settings used as default ...
664 669 cache_settings.update(get_default_cache_settings(settings))
665 670
666 671 if cache_settings['regions']:
667 672 for region in cache_settings['regions'].split(','):
668 673 region = region.strip()
669 674 region_settings = {}
670 675 for key, value in cache_settings.items():
671 676 if key.startswith(region):
672 677 region_settings[key.split('.')[1]] = value
673 678
674 679 caches.configure_cache_region(
675 680 region, region_settings, cache_settings)
676 681
677 682
678 683 def load_rcextensions(root_path):
679 684 import rhodecode
680 685 from rhodecode.config import conf
681 686
682 687 path = os.path.join(root_path, 'rcextensions', '__init__.py')
683 688 if os.path.isfile(path):
684 689 rcext = create_module('rc', path)
685 690 EXT = rhodecode.EXTENSIONS = rcext
686 691 log.debug('Found rcextensions now loading %s...', rcext)
687 692
688 693 # Additional mappings that are not present in the pygments lexers
689 694 conf.LANGUAGES_EXTENSIONS_MAP.update(getattr(EXT, 'EXTRA_MAPPINGS', {}))
690 695
691 696 # auto check if the module is not missing any data, set to default if is
692 697 # this will help autoupdate new feature of rcext module
693 698 #from rhodecode.config import rcextensions
694 699 #for k in dir(rcextensions):
695 700 # if not k.startswith('_') and not hasattr(EXT, k):
696 701 # setattr(EXT, k, getattr(rcextensions, k))
697 702
698 703
699 704 def get_custom_lexer(extension):
700 705 """
701 706 returns a custom lexer if it is defined in rcextensions module, or None
702 707 if there's no custom lexer defined
703 708 """
704 709 import rhodecode
705 710 from pygments import lexers
706 711 # check if we didn't define this extension as other lexer
707 712 extensions = rhodecode.EXTENSIONS and getattr(rhodecode.EXTENSIONS, 'EXTRA_LEXERS', None)
708 713 if extensions and extension in rhodecode.EXTENSIONS.EXTRA_LEXERS:
709 714 _lexer_name = rhodecode.EXTENSIONS.EXTRA_LEXERS[extension]
710 715 return lexers.get_lexer_by_name(_lexer_name)
711 716
712 717
713 718 #==============================================================================
714 719 # TEST FUNCTIONS AND CREATORS
715 720 #==============================================================================
716 721 def create_test_index(repo_location, config):
717 722 """
718 723 Makes default test index.
719 724 """
720 725 import rc_testdata
721 726
722 727 rc_testdata.extract_search_index(
723 728 'vcs_search_index', os.path.dirname(config['search.location']))
724 729
725 730
726 731 def create_test_directory(test_path):
727 732 """
728 733 Create test directory if it doesn't exist.
729 734 """
730 735 if not os.path.isdir(test_path):
731 736 log.debug('Creating testdir %s', test_path)
732 737 os.makedirs(test_path)
733 738
734 739
735 740 def create_test_database(test_path, config):
736 741 """
737 742 Makes a fresh database.
738 743 """
739 744 from rhodecode.lib.db_manage import DbManage
740 745
741 746 # PART ONE create db
742 747 dbconf = config['sqlalchemy.db1.url']
743 748 log.debug('making test db %s', dbconf)
744 749
745 750 dbmanage = DbManage(log_sql=False, dbconf=dbconf, root=config['here'],
746 751 tests=True, cli_args={'force_ask': True})
747 752 dbmanage.create_tables(override=True)
748 753 dbmanage.set_db_version()
749 754 # for tests dynamically set new root paths based on generated content
750 755 dbmanage.create_settings(dbmanage.config_prompt(test_path))
751 756 dbmanage.create_default_user()
752 757 dbmanage.create_test_admin_and_users()
753 758 dbmanage.create_permissions()
754 759 dbmanage.populate_default_permissions()
755 760 Session().commit()
756 761
757 762
758 763 def create_test_repositories(test_path, config):
759 764 """
760 765 Creates test repositories in the temporary directory. Repositories are
761 766 extracted from archives within the rc_testdata package.
762 767 """
763 768 import rc_testdata
764 769 from rhodecode.tests import HG_REPO, GIT_REPO, SVN_REPO
765 770
766 771 log.debug('making test vcs repositories')
767 772
768 773 idx_path = config['search.location']
769 774 data_path = config['cache_dir']
770 775
771 776 # clean index and data
772 777 if idx_path and os.path.exists(idx_path):
773 778 log.debug('remove %s', idx_path)
774 779 shutil.rmtree(idx_path)
775 780
776 781 if data_path and os.path.exists(data_path):
777 782 log.debug('remove %s', data_path)
778 783 shutil.rmtree(data_path)
779 784
780 785 rc_testdata.extract_hg_dump('vcs_test_hg', jn(test_path, HG_REPO))
781 786 rc_testdata.extract_git_dump('vcs_test_git', jn(test_path, GIT_REPO))
782 787
783 788 # Note: Subversion is in the process of being integrated with the system,
784 789 # until we have a properly packed version of the test svn repository, this
785 790 # tries to copy over the repo from a package "rc_testdata"
786 791 svn_repo_path = rc_testdata.get_svn_repo_archive()
787 792 with tarfile.open(svn_repo_path) as tar:
788 793 tar.extractall(jn(test_path, SVN_REPO))
789 794
790 795
791 796 #==============================================================================
792 797 # PASTER COMMANDS
793 798 #==============================================================================
794 799 class BasePasterCommand(Command):
795 800 """
796 801 Abstract Base Class for paster commands.
797 802
798 803 The celery commands are somewhat aggressive about loading
799 804 celery.conf, and since our module sets the `CELERY_LOADER`
800 805 environment variable to our loader, we have to bootstrap a bit and
801 806 make sure we've had a chance to load the pylons config off of the
802 807 command line, otherwise everything fails.
803 808 """
804 809 min_args = 1
805 810 min_args_error = "Please provide a paster config file as an argument."
806 811 takes_config_file = 1
807 812 requires_config_file = True
808 813
809 814 def notify_msg(self, msg, log=False):
810 815 """Make a notification to user, additionally if logger is passed
811 816 it logs this action using given logger
812 817
813 818 :param msg: message that will be printed to user
814 819 :param log: logging instance, to use to additionally log this message
815 820
816 821 """
817 822 if log and isinstance(log, logging):
818 823 log(msg)
819 824
820 825 def run(self, args):
821 826 """
822 827 Overrides Command.run
823 828
824 829 Checks for a config file argument and loads it.
825 830 """
826 831 if len(args) < self.min_args:
827 832 raise BadCommand(
828 833 self.min_args_error % {'min_args': self.min_args,
829 834 'actual_args': len(args)})
830 835
831 836 # Decrement because we're going to lob off the first argument.
832 837 # @@ This is hacky
833 838 self.min_args -= 1
834 839 self.bootstrap_config(args[0])
835 840 self.update_parser()
836 841 return super(BasePasterCommand, self).run(args[1:])
837 842
838 843 def update_parser(self):
839 844 """
840 845 Abstract method. Allows for the class' parser to be updated
841 846 before the superclass' `run` method is called. Necessary to
842 847 allow options/arguments to be passed through to the underlying
843 848 celery command.
844 849 """
845 850 raise NotImplementedError("Abstract Method.")
846 851
847 852 def bootstrap_config(self, conf):
848 853 """
849 854 Loads the pylons configuration.
850 855 """
851 856 from pylons import config as pylonsconfig
852 857
853 858 self.path_to_ini_file = os.path.realpath(conf)
854 859 conf = paste.deploy.appconfig('config:' + self.path_to_ini_file)
855 860 pylonsconfig.init_app(conf.global_conf, conf.local_conf)
856 861
857 862 def _init_session(self):
858 863 """
859 864 Inits SqlAlchemy Session
860 865 """
861 866 logging.config.fileConfig(self.path_to_ini_file)
862 867 from pylons import config
863 868 from rhodecode.config.utils import initialize_database
864 869
865 870 # get to remove repos !!
866 871 add_cache(config)
867 872 initialize_database(config)
868 873
869 874
870 875 @decorator.decorator
871 876 def jsonify(func, *args, **kwargs):
872 877 """Action decorator that formats output for JSON
873 878
874 879 Given a function that will return content, this decorator will turn
875 880 the result into JSON, with a content-type of 'application/json' and
876 881 output it.
877 882
878 883 """
879 884 from pylons.decorators.util import get_pylons
880 885 from rhodecode.lib.ext_json import json
881 886 pylons = get_pylons(args)
882 887 pylons.response.headers['Content-Type'] = 'application/json; charset=utf-8'
883 888 data = func(*args, **kwargs)
884 889 if isinstance(data, (list, tuple)):
885 890 msg = "JSON responses with Array envelopes are susceptible to " \
886 891 "cross-site data leak attacks, see " \
887 892 "http://wiki.pylonshq.com/display/pylonsfaq/Warnings"
888 893 warnings.warn(msg, Warning, 2)
889 894 log.warning(msg)
890 895 log.debug("Returning JSON wrapped action output")
891 896 return json.dumps(data, encoding='utf-8')
892 897
893 898
894 899 class PartialRenderer(object):
895 900 """
896 901 Partial renderer used to render chunks of html used in datagrids
897 902 use like::
898 903
899 904 _render = PartialRenderer('data_table/_dt_elements.html')
900 905 _render('quick_menu', args, kwargs)
901 906 PartialRenderer.h,
902 907 c,
903 908 _,
904 909 ungettext
905 910 are the template stuff initialized inside and can be re-used later
906 911
907 912 :param tmpl_name: template path relate to /templates/ dir
908 913 """
909 914
910 915 def __init__(self, tmpl_name):
911 916 import rhodecode
912 917 from pylons import request, tmpl_context as c
913 918 from pylons.i18n.translation import _, ungettext
914 919 from rhodecode.lib import helpers as h
915 920
916 921 self.tmpl_name = tmpl_name
917 922 self.rhodecode = rhodecode
918 923 self.c = c
919 924 self._ = _
920 925 self.ungettext = ungettext
921 926 self.h = h
922 927 self.request = request
923 928
924 929 def _mako_lookup(self):
925 930 _tmpl_lookup = self.rhodecode.CONFIG['pylons.app_globals'].mako_lookup
926 931 return _tmpl_lookup.get_template(self.tmpl_name)
927 932
928 933 def _update_kwargs_for_render(self, kwargs):
929 934 """
930 935 Inject params required for Mako rendering
931 936 """
932 937 _kwargs = {
933 938 '_': self._,
934 939 'h': self.h,
935 940 'c': self.c,
936 941 'request': self.request,
937 942 'ungettext': self.ungettext,
938 943 }
939 944 _kwargs.update(kwargs)
940 945 return _kwargs
941 946
942 947 def _render_with_exc(self, render_func, args, kwargs):
943 948 try:
944 949 return render_func.render(*args, **kwargs)
945 950 except:
946 951 log.error(exceptions.text_error_template().render())
947 952 raise
948 953
949 954 def _get_template(self, template_obj, def_name):
950 955 if def_name:
951 956 tmpl = template_obj.get_def(def_name)
952 957 else:
953 958 tmpl = template_obj
954 959 return tmpl
955 960
956 961 def render(self, def_name, *args, **kwargs):
957 962 lookup_obj = self._mako_lookup()
958 963 tmpl = self._get_template(lookup_obj, def_name=def_name)
959 964 kwargs = self._update_kwargs_for_render(kwargs)
960 965 return self._render_with_exc(tmpl, args, kwargs)
961 966
962 967 def __call__(self, tmpl, *args, **kwargs):
963 968 return self.render(tmpl, *args, **kwargs)
964 969
965 970
966 971 def password_changed(auth_user, session):
967 972 # Never report password change in case of default user or anonymous user.
968 973 if auth_user.username == User.DEFAULT_USER or auth_user.user_id is None:
969 974 return False
970 975
971 976 password_hash = md5(auth_user.password) if auth_user.password else None
972 977 rhodecode_user = session.get('rhodecode_user', {})
973 978 session_password_hash = rhodecode_user.get('password', '')
974 979 return password_hash != session_password_hash
975 980
976 981
977 982 def read_opensource_licenses():
978 983 global _license_cache
979 984
980 985 if not _license_cache:
981 986 licenses = pkg_resources.resource_string(
982 987 'rhodecode', 'config/licenses.json')
983 988 _license_cache = json.loads(licenses)
984 989
985 990 return _license_cache
986 991
987 992
988 993 def get_registry(request):
989 994 """
990 995 Utility to get the pyramid registry from a request. During migration to
991 996 pyramid we sometimes want to use the pyramid registry from pylons context.
992 997 Therefore this utility returns `request.registry` for pyramid requests and
993 998 uses `get_current_registry()` for pylons requests.
994 999 """
995 1000 try:
996 1001 return request.registry
997 1002 except AttributeError:
998 1003 return get_current_registry()
General Comments 0
You need to be logged in to leave comments. Login now