##// END OF EJS Templates
fix(service-api): rely on urljoin for constructing the call url
super-admin -
r5319:cc68847e default
parent child Browse files
Show More
@@ -1,857 +1,859 b''
1 1 # Copyright (C) 2010-2023 RhodeCode GmbH
2 2 #
3 3 # This program is free software: you can redistribute it and/or modify
4 4 # it under the terms of the GNU Affero General Public License, version 3
5 5 # (only), as published by the Free Software Foundation.
6 6 #
7 7 # This program is distributed in the hope that it will be useful,
8 8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 10 # GNU General Public License for more details.
11 11 #
12 12 # You should have received a copy of the GNU Affero General Public License
13 13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 14 #
15 15 # This program is dual-licensed. If you wish to learn more about the
16 16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18 18
19 19 """
20 20 Utilities library for RhodeCode
21 21 """
22 22
23 23 import datetime
24
24 25 import decorator
25 26 import logging
26 27 import os
27 28 import re
28 29 import sys
29 30 import shutil
30 31 import socket
31 32 import tempfile
32 33 import traceback
33 34 import tarfile
35 import urllib.parse
34 36 import warnings
35 37 from functools import wraps
36 38 from os.path import join as jn
37 39 from configparser import NoOptionError
38 40
39 41 import paste
40 42 import pkg_resources
41 43 from webhelpers2.text import collapse, strip_tags, convert_accented_entities, convert_misc_entities
42 44
43 45 from mako import exceptions
44 46
45 47 from rhodecode.lib.hash_utils import sha256_safe, md5, sha1
46 48 from rhodecode.lib.type_utils import AttributeDict
47 49 from rhodecode.lib.str_utils import safe_bytes, safe_str
48 50 from rhodecode.lib.vcs.backends.base import Config
49 51 from rhodecode.lib.vcs.exceptions import VCSError
50 52 from rhodecode.lib.vcs.utils.helpers import get_scm, get_scm_backend
51 53 from rhodecode.lib.ext_json import sjson as json
52 54 from rhodecode.model import meta
53 55 from rhodecode.model.db import (
54 56 Repository, User, RhodeCodeUi, UserLog, RepoGroup, UserGroup)
55 57 from rhodecode.model.meta import Session
56 58 from rhodecode.lib.pyramid_utils import get_config
57 59 from rhodecode.lib.vcs import CurlSession
58 60 from rhodecode.lib.vcs.exceptions import ImproperlyConfiguredError
59 61
60 62
61 63 log = logging.getLogger(__name__)
62 64
63 65 REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}__.*')
64 66
65 67 # String which contains characters that are not allowed in slug names for
66 68 # repositories or repository groups. It is properly escaped to use it in
67 69 # regular expressions.
68 70 SLUG_BAD_CHARS = re.escape(r'`?=[]\;\'"<>,/~!@#$%^&*()+{}|:')
69 71
70 72 # Regex that matches forbidden characters in repo/group slugs.
71 73 SLUG_BAD_CHAR_RE = re.compile(r'[{}\x00-\x08\x0b-\x0c\x0e-\x1f]'.format(SLUG_BAD_CHARS))
72 74
73 75 # Regex that matches allowed characters in repo/group slugs.
74 76 SLUG_GOOD_CHAR_RE = re.compile(r'[^{}]'.format(SLUG_BAD_CHARS))
75 77
76 78 # Regex that matches whole repo/group slugs.
77 79 SLUG_RE = re.compile(r'[^{}]+'.format(SLUG_BAD_CHARS))
78 80
79 81 _license_cache = None
80 82
81 83
82 84 def adopt_for_celery(func):
83 85 """
84 86 Decorator designed to adopt hooks (from rhodecode.lib.hooks_base)
85 87 for further usage as a celery tasks.
86 88 """
87 89 @wraps(func)
88 90 def wrapper(extras):
89 91 extras = AttributeDict(extras)
90 92 # HooksResponse implements to_json method which must be used there.
91 93 return func(extras).to_json()
92 94 return wrapper
93 95
94 96
95 97 def repo_name_slug(value):
96 98 """
97 99 Return slug of name of repository
98 100 This function is called on each creation/modification
99 101 of repository to prevent bad names in repo
100 102 """
101 103
102 104 replacement_char = '-'
103 105
104 106 slug = strip_tags(value)
105 107 slug = convert_accented_entities(slug)
106 108 slug = convert_misc_entities(slug)
107 109
108 110 slug = SLUG_BAD_CHAR_RE.sub('', slug)
109 111 slug = re.sub(r'[\s]+', '-', slug)
110 112 slug = collapse(slug, replacement_char)
111 113
112 114 return slug
113 115
114 116
115 117 #==============================================================================
116 118 # PERM DECORATOR HELPERS FOR EXTRACTING NAMES FOR PERM CHECKS
117 119 #==============================================================================
118 120 def get_repo_slug(request):
119 121 _repo = ''
120 122
121 123 if hasattr(request, 'db_repo_name'):
122 124 # if our requests has set db reference use it for name, this
123 125 # translates the example.com/_<id> into proper repo names
124 126 _repo = request.db_repo_name
125 127 elif getattr(request, 'matchdict', None):
126 128 # pyramid
127 129 _repo = request.matchdict.get('repo_name')
128 130
129 131 if _repo:
130 132 _repo = _repo.rstrip('/')
131 133 return _repo
132 134
133 135
134 136 def get_repo_group_slug(request):
135 137 _group = ''
136 138 if hasattr(request, 'db_repo_group'):
137 139 # if our requests has set db reference use it for name, this
138 140 # translates the example.com/_<id> into proper repo group names
139 141 _group = request.db_repo_group.group_name
140 142 elif getattr(request, 'matchdict', None):
141 143 # pyramid
142 144 _group = request.matchdict.get('repo_group_name')
143 145
144 146 if _group:
145 147 _group = _group.rstrip('/')
146 148 return _group
147 149
148 150
149 151 def get_user_group_slug(request):
150 152 _user_group = ''
151 153
152 154 if hasattr(request, 'db_user_group'):
153 155 _user_group = request.db_user_group.users_group_name
154 156 elif getattr(request, 'matchdict', None):
155 157 # pyramid
156 158 _user_group = request.matchdict.get('user_group_id')
157 159 _user_group_name = request.matchdict.get('user_group_name')
158 160 try:
159 161 if _user_group:
160 162 _user_group = UserGroup.get(_user_group)
161 163 elif _user_group_name:
162 164 _user_group = UserGroup.get_by_group_name(_user_group_name)
163 165
164 166 if _user_group:
165 167 _user_group = _user_group.users_group_name
166 168 except Exception:
167 169 log.exception('Failed to get user group by id and name')
168 170 # catch all failures here
169 171 return None
170 172
171 173 return _user_group
172 174
173 175
174 176 def get_filesystem_repos(path, recursive=False, skip_removed_repos=True):
175 177 """
176 178 Scans given path for repos and return (name,(type,path)) tuple
177 179
178 180 :param path: path to scan for repositories
179 181 :param recursive: recursive search and return names with subdirs in front
180 182 """
181 183
182 184 # remove ending slash for better results
183 185 path = path.rstrip(os.sep)
184 186 log.debug('now scanning in %s location recursive:%s...', path, recursive)
185 187
186 188 def _get_repos(p):
187 189 dirpaths = get_dirpaths(p)
188 190 if not _is_dir_writable(p):
189 191 log.warning('repo path without write access: %s', p)
190 192
191 193 for dirpath in dirpaths:
192 194 if os.path.isfile(os.path.join(p, dirpath)):
193 195 continue
194 196 cur_path = os.path.join(p, dirpath)
195 197
196 198 # skip removed repos
197 199 if skip_removed_repos and REMOVED_REPO_PAT.match(dirpath):
198 200 continue
199 201
200 202 #skip .<somethin> dirs
201 203 if dirpath.startswith('.'):
202 204 continue
203 205
204 206 try:
205 207 scm_info = get_scm(cur_path)
206 208 yield scm_info[1].split(path, 1)[-1].lstrip(os.sep), scm_info
207 209 except VCSError:
208 210 if not recursive:
209 211 continue
210 212 #check if this dir containts other repos for recursive scan
211 213 rec_path = os.path.join(p, dirpath)
212 214 if os.path.isdir(rec_path):
213 215 yield from _get_repos(rec_path)
214 216
215 217 return _get_repos(path)
216 218
217 219
218 220 def get_dirpaths(p: str) -> list:
219 221 try:
220 222 # OS-independable way of checking if we have at least read-only
221 223 # access or not.
222 224 dirpaths = os.listdir(p)
223 225 except OSError:
224 226 log.warning('ignoring repo path without read access: %s', p)
225 227 return []
226 228
227 229 # os.listpath has a tweak: If a unicode is passed into it, then it tries to
228 230 # decode paths and suddenly returns unicode objects itself. The items it
229 231 # cannot decode are returned as strings and cause issues.
230 232 #
231 233 # Those paths are ignored here until a solid solution for path handling has
232 234 # been built.
233 235 expected_type = type(p)
234 236
235 237 def _has_correct_type(item):
236 238 if type(item) is not expected_type:
237 239 log.error(
238 240 "Ignoring path %s since it cannot be decoded into str.",
239 241 # Using "repr" to make sure that we see the byte value in case
240 242 # of support.
241 243 repr(item))
242 244 return False
243 245 return True
244 246
245 247 dirpaths = [item for item in dirpaths if _has_correct_type(item)]
246 248
247 249 return dirpaths
248 250
249 251
250 252 def _is_dir_writable(path):
251 253 """
252 254 Probe if `path` is writable.
253 255
254 256 Due to trouble on Cygwin / Windows, this is actually probing if it is
255 257 possible to create a file inside of `path`, stat does not produce reliable
256 258 results in this case.
257 259 """
258 260 try:
259 261 with tempfile.TemporaryFile(dir=path):
260 262 pass
261 263 except OSError:
262 264 return False
263 265 return True
264 266
265 267
266 268 def is_valid_repo(repo_name, base_path, expect_scm=None, explicit_scm=None, config=None):
267 269 """
268 270 Returns True if given path is a valid repository False otherwise.
269 271 If expect_scm param is given also, compare if given scm is the same
270 272 as expected from scm parameter. If explicit_scm is given don't try to
271 273 detect the scm, just use the given one to check if repo is valid
272 274
273 275 :param repo_name:
274 276 :param base_path:
275 277 :param expect_scm:
276 278 :param explicit_scm:
277 279 :param config:
278 280
279 281 :return True: if given path is a valid repository
280 282 """
281 283 full_path = os.path.join(safe_str(base_path), safe_str(repo_name))
282 284 log.debug('Checking if `%s` is a valid path for repository. '
283 285 'Explicit type: %s', repo_name, explicit_scm)
284 286
285 287 try:
286 288 if explicit_scm:
287 289 detected_scms = [get_scm_backend(explicit_scm)(
288 290 full_path, config=config).alias]
289 291 else:
290 292 detected_scms = get_scm(full_path)
291 293
292 294 if expect_scm:
293 295 return detected_scms[0] == expect_scm
294 296 log.debug('path: %s is an vcs object:%s', full_path, detected_scms)
295 297 return True
296 298 except VCSError:
297 299 log.debug('path: %s is not a valid repo !', full_path)
298 300 return False
299 301
300 302
301 303 def is_valid_repo_group(repo_group_name, base_path, skip_path_check=False):
302 304 """
303 305 Returns True if a given path is a repository group, False otherwise
304 306
305 307 :param repo_group_name:
306 308 :param base_path:
307 309 """
308 310 full_path = os.path.join(safe_str(base_path), safe_str(repo_group_name))
309 311 log.debug('Checking if `%s` is a valid path for repository group',
310 312 repo_group_name)
311 313
312 314 # check if it's not a repo
313 315 if is_valid_repo(repo_group_name, base_path):
314 316 log.debug('Repo called %s exist, it is not a valid repo group', repo_group_name)
315 317 return False
316 318
317 319 try:
318 320 # we need to check bare git repos at higher level
319 321 # since we might match branches/hooks/info/objects or possible
320 322 # other things inside bare git repo
321 323 maybe_repo = os.path.dirname(full_path)
322 324 if maybe_repo == base_path:
323 325 # skip root level repo check; we know root location CANNOT BE a repo group
324 326 return False
325 327
326 328 scm_ = get_scm(maybe_repo)
327 329 log.debug('path: %s is a vcs object:%s, not valid repo group', full_path, scm_)
328 330 return False
329 331 except VCSError:
330 332 pass
331 333
332 334 # check if it's a valid path
333 335 if skip_path_check or os.path.isdir(full_path):
334 336 log.debug('path: %s is a valid repo group !', full_path)
335 337 return True
336 338
337 339 log.debug('path: %s is not a valid repo group !', full_path)
338 340 return False
339 341
340 342
341 343 def ask_ok(prompt, retries=4, complaint='[y]es or [n]o please!'):
342 344 while True:
343 345 ok = input(prompt)
344 346 if ok.lower() in ('y', 'ye', 'yes'):
345 347 return True
346 348 if ok.lower() in ('n', 'no', 'nop', 'nope'):
347 349 return False
348 350 retries = retries - 1
349 351 if retries < 0:
350 352 raise OSError
351 353 print(complaint)
352 354
353 355 # propagated from mercurial documentation
354 356 ui_sections = [
355 357 'alias', 'auth',
356 358 'decode/encode', 'defaults',
357 359 'diff', 'email',
358 360 'extensions', 'format',
359 361 'merge-patterns', 'merge-tools',
360 362 'hooks', 'http_proxy',
361 363 'smtp', 'patch',
362 364 'paths', 'profiling',
363 365 'server', 'trusted',
364 366 'ui', 'web', ]
365 367
366 368
367 369 def config_data_from_db(clear_session=True, repo=None):
368 370 """
369 371 Read the configuration data from the database and return configuration
370 372 tuples.
371 373 """
372 374 from rhodecode.model.settings import VcsSettingsModel
373 375
374 376 config = []
375 377
376 378 sa = meta.Session()
377 379 settings_model = VcsSettingsModel(repo=repo, sa=sa)
378 380
379 381 ui_settings = settings_model.get_ui_settings()
380 382
381 383 ui_data = []
382 384 for setting in ui_settings:
383 385 if setting.active:
384 386 ui_data.append((setting.section, setting.key, setting.value))
385 387 config.append((
386 388 safe_str(setting.section), safe_str(setting.key),
387 389 safe_str(setting.value)))
388 390 if setting.key == 'push_ssl':
389 391 # force set push_ssl requirement to False, rhodecode
390 392 # handles that
391 393 config.append((
392 394 safe_str(setting.section), safe_str(setting.key), False))
393 395 log.debug(
394 396 'settings ui from db@repo[%s]: %s',
395 397 repo,
396 398 ','.join(['[{}] {}={}'.format(*s) for s in ui_data]))
397 399 if clear_session:
398 400 meta.Session.remove()
399 401
400 402 # TODO: mikhail: probably it makes no sense to re-read hooks information.
401 403 # It's already there and activated/deactivated
402 404 skip_entries = []
403 405 enabled_hook_classes = get_enabled_hook_classes(ui_settings)
404 406 if 'pull' not in enabled_hook_classes:
405 407 skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PULL))
406 408 if 'push' not in enabled_hook_classes:
407 409 skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PUSH))
408 410 skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRETX_PUSH))
409 411 skip_entries.append(('hooks', RhodeCodeUi.HOOK_PUSH_KEY))
410 412
411 413 config = [entry for entry in config if entry[:2] not in skip_entries]
412 414
413 415 return config
414 416
415 417
416 418 def make_db_config(clear_session=True, repo=None):
417 419 """
418 420 Create a :class:`Config` instance based on the values in the database.
419 421 """
420 422 config = Config()
421 423 config_data = config_data_from_db(clear_session=clear_session, repo=repo)
422 424 for section, option, value in config_data:
423 425 config.set(section, option, value)
424 426 return config
425 427
426 428
427 429 def get_enabled_hook_classes(ui_settings):
428 430 """
429 431 Return the enabled hook classes.
430 432
431 433 :param ui_settings: List of ui_settings as returned
432 434 by :meth:`VcsSettingsModel.get_ui_settings`
433 435
434 436 :return: a list with the enabled hook classes. The order is not guaranteed.
435 437 :rtype: list
436 438 """
437 439 enabled_hooks = []
438 440 active_hook_keys = [
439 441 key for section, key, value, active in ui_settings
440 442 if section == 'hooks' and active]
441 443
442 444 hook_names = {
443 445 RhodeCodeUi.HOOK_PUSH: 'push',
444 446 RhodeCodeUi.HOOK_PULL: 'pull',
445 447 RhodeCodeUi.HOOK_REPO_SIZE: 'repo_size'
446 448 }
447 449
448 450 for key in active_hook_keys:
449 451 hook = hook_names.get(key)
450 452 if hook:
451 453 enabled_hooks.append(hook)
452 454
453 455 return enabled_hooks
454 456
455 457
456 458 def set_rhodecode_config(config):
457 459 """
458 460 Updates pyramid config with new settings from database
459 461
460 462 :param config:
461 463 """
462 464 from rhodecode.model.settings import SettingsModel
463 465 app_settings = SettingsModel().get_all_settings()
464 466
465 467 for k, v in list(app_settings.items()):
466 468 config[k] = v
467 469
468 470
469 471 def get_rhodecode_realm():
470 472 """
471 473 Return the rhodecode realm from database.
472 474 """
473 475 from rhodecode.model.settings import SettingsModel
474 476 realm = SettingsModel().get_setting_by_name('realm')
475 477 return safe_str(realm.app_settings_value)
476 478
477 479
478 480 def get_rhodecode_base_path():
479 481 """
480 482 Returns the base path. The base path is the filesystem path which points
481 483 to the repository store.
482 484 """
483 485
484 486 import rhodecode
485 487 return rhodecode.CONFIG['default_base_path']
486 488
487 489
488 490 def map_groups(path):
489 491 """
490 492 Given a full path to a repository, create all nested groups that this
491 493 repo is inside. This function creates parent-child relationships between
492 494 groups and creates default perms for all new groups.
493 495
494 496 :param paths: full path to repository
495 497 """
496 498 from rhodecode.model.repo_group import RepoGroupModel
497 499 sa = meta.Session()
498 500 groups = path.split(Repository.NAME_SEP)
499 501 parent = None
500 502 group = None
501 503
502 504 # last element is repo in nested groups structure
503 505 groups = groups[:-1]
504 506 rgm = RepoGroupModel(sa)
505 507 owner = User.get_first_super_admin()
506 508 for lvl, group_name in enumerate(groups):
507 509 group_name = '/'.join(groups[:lvl] + [group_name])
508 510 group = RepoGroup.get_by_group_name(group_name)
509 511 desc = '%s group' % group_name
510 512
511 513 # skip folders that are now removed repos
512 514 if REMOVED_REPO_PAT.match(group_name):
513 515 break
514 516
515 517 if group is None:
516 518 log.debug('creating group level: %s group_name: %s',
517 519 lvl, group_name)
518 520 group = RepoGroup(group_name, parent)
519 521 group.group_description = desc
520 522 group.user = owner
521 523 sa.add(group)
522 524 perm_obj = rgm._create_default_perms(group)
523 525 sa.add(perm_obj)
524 526 sa.flush()
525 527
526 528 parent = group
527 529 return group
528 530
529 531
530 532 def repo2db_mapper(initial_repo_list, remove_obsolete=False, force_hooks_rebuild=False):
531 533 """
532 534 maps all repos given in initial_repo_list, non existing repositories
533 535 are created, if remove_obsolete is True it also checks for db entries
534 536 that are not in initial_repo_list and removes them.
535 537
536 538 :param initial_repo_list: list of repositories found by scanning methods
537 539 :param remove_obsolete: check for obsolete entries in database
538 540 """
539 541 from rhodecode.model.repo import RepoModel
540 542 from rhodecode.model.repo_group import RepoGroupModel
541 543 from rhodecode.model.settings import SettingsModel
542 544
543 545 sa = meta.Session()
544 546 repo_model = RepoModel()
545 547 user = User.get_first_super_admin()
546 548 added = []
547 549
548 550 # creation defaults
549 551 defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
550 552 enable_statistics = defs.get('repo_enable_statistics')
551 553 enable_locking = defs.get('repo_enable_locking')
552 554 enable_downloads = defs.get('repo_enable_downloads')
553 555 private = defs.get('repo_private')
554 556
555 557 for name, repo in list(initial_repo_list.items()):
556 558 group = map_groups(name)
557 559 str_name = safe_str(name)
558 560 db_repo = repo_model.get_by_repo_name(str_name)
559 561
560 562 # found repo that is on filesystem not in RhodeCode database
561 563 if not db_repo:
562 564 log.info('repository `%s` not found in the database, creating now', name)
563 565 added.append(name)
564 566 desc = (repo.description
565 567 if repo.description != 'unknown'
566 568 else '%s repository' % name)
567 569
568 570 db_repo = repo_model._create_repo(
569 571 repo_name=name,
570 572 repo_type=repo.alias,
571 573 description=desc,
572 574 repo_group=getattr(group, 'group_id', None),
573 575 owner=user,
574 576 enable_locking=enable_locking,
575 577 enable_downloads=enable_downloads,
576 578 enable_statistics=enable_statistics,
577 579 private=private,
578 580 state=Repository.STATE_CREATED
579 581 )
580 582 sa.commit()
581 583 # we added that repo just now, and make sure we updated server info
582 584 if db_repo.repo_type == 'git':
583 585 git_repo = db_repo.scm_instance()
584 586 # update repository server-info
585 587 log.debug('Running update server info')
586 588 git_repo._update_server_info(force=True)
587 589
588 590 db_repo.update_commit_cache()
589 591
590 592 config = db_repo._config
591 593 config.set('extensions', 'largefiles', '')
592 594 repo = db_repo.scm_instance(config=config)
593 595 repo.install_hooks(force=force_hooks_rebuild)
594 596
595 597 removed = []
596 598 if remove_obsolete:
597 599 # remove from database those repositories that are not in the filesystem
598 600 for repo in sa.query(Repository).all():
599 601 if repo.repo_name not in list(initial_repo_list.keys()):
600 602 log.debug("Removing non-existing repository found in db `%s`",
601 603 repo.repo_name)
602 604 try:
603 605 RepoModel(sa).delete(repo, forks='detach', fs_remove=False)
604 606 sa.commit()
605 607 removed.append(repo.repo_name)
606 608 except Exception:
607 609 # don't hold further removals on error
608 610 log.error(traceback.format_exc())
609 611 sa.rollback()
610 612
611 613 def splitter(full_repo_name):
612 614 _parts = full_repo_name.rsplit(RepoGroup.url_sep(), 1)
613 615 gr_name = None
614 616 if len(_parts) == 2:
615 617 gr_name = _parts[0]
616 618 return gr_name
617 619
618 620 initial_repo_group_list = [splitter(x) for x in
619 621 list(initial_repo_list.keys()) if splitter(x)]
620 622
621 623 # remove from database those repository groups that are not in the
622 624 # filesystem due to parent child relationships we need to delete them
623 625 # in a specific order of most nested first
624 626 all_groups = [x.group_name for x in sa.query(RepoGroup).all()]
625 627 def nested_sort(gr):
626 628 return len(gr.split('/'))
627 629 for group_name in sorted(all_groups, key=nested_sort, reverse=True):
628 630 if group_name not in initial_repo_group_list:
629 631 repo_group = RepoGroup.get_by_group_name(group_name)
630 632 if (repo_group.children.all() or
631 633 not RepoGroupModel().check_exist_filesystem(
632 634 group_name=group_name, exc_on_failure=False)):
633 635 continue
634 636
635 637 log.info(
636 638 'Removing non-existing repository group found in db `%s`',
637 639 group_name)
638 640 try:
639 641 RepoGroupModel(sa).delete(group_name, fs_remove=False)
640 642 sa.commit()
641 643 removed.append(group_name)
642 644 except Exception:
643 645 # don't hold further removals on error
644 646 log.exception(
645 647 'Unable to remove repository group `%s`',
646 648 group_name)
647 649 sa.rollback()
648 650 raise
649 651
650 652 return added, removed
651 653
652 654
653 655 def load_rcextensions(root_path):
654 656 import rhodecode
655 657 from rhodecode.config import conf
656 658
657 659 path = os.path.join(root_path)
658 660 sys.path.append(path)
659 661
660 662 try:
661 663 rcextensions = __import__('rcextensions')
662 664 except ImportError:
663 665 if os.path.isdir(os.path.join(path, 'rcextensions')):
664 666 log.warning('Unable to load rcextensions from %s', path)
665 667 rcextensions = None
666 668
667 669 if rcextensions:
668 670 log.info('Loaded rcextensions from %s...', rcextensions)
669 671 rhodecode.EXTENSIONS = rcextensions
670 672
671 673 # Additional mappings that are not present in the pygments lexers
672 674 conf.LANGUAGES_EXTENSIONS_MAP.update(
673 675 getattr(rhodecode.EXTENSIONS, 'EXTRA_MAPPINGS', {}))
674 676
675 677
676 678 def get_custom_lexer(extension):
677 679 """
678 680 returns a custom lexer if it is defined in rcextensions module, or None
679 681 if there's no custom lexer defined
680 682 """
681 683 import rhodecode
682 684 from pygments import lexers
683 685
684 686 # custom override made by RhodeCode
685 687 if extension in ['mako']:
686 688 return lexers.get_lexer_by_name('html+mako')
687 689
688 690 # check if we didn't define this extension as other lexer
689 691 extensions = rhodecode.EXTENSIONS and getattr(rhodecode.EXTENSIONS, 'EXTRA_LEXERS', None)
690 692 if extensions and extension in rhodecode.EXTENSIONS.EXTRA_LEXERS:
691 693 _lexer_name = rhodecode.EXTENSIONS.EXTRA_LEXERS[extension]
692 694 return lexers.get_lexer_by_name(_lexer_name)
693 695
694 696
695 697 #==============================================================================
696 698 # TEST FUNCTIONS AND CREATORS
697 699 #==============================================================================
698 700 def create_test_index(repo_location, config):
699 701 """
700 702 Makes default test index.
701 703 """
702 704 try:
703 705 import rc_testdata
704 706 except ImportError:
705 707 raise ImportError('Failed to import rc_testdata, '
706 708 'please make sure this package is installed from requirements_test.txt')
707 709 rc_testdata.extract_search_index(
708 710 'vcs_search_index', os.path.dirname(config['search.location']))
709 711
710 712
711 713 def create_test_directory(test_path):
712 714 """
713 715 Create test directory if it doesn't exist.
714 716 """
715 717 if not os.path.isdir(test_path):
716 718 log.debug('Creating testdir %s', test_path)
717 719 os.makedirs(test_path)
718 720
719 721
720 722 def create_test_database(test_path, config):
721 723 """
722 724 Makes a fresh database.
723 725 """
724 726 from rhodecode.lib.db_manage import DbManage
725 727 from rhodecode.lib.utils2 import get_encryption_key
726 728
727 729 # PART ONE create db
728 730 dbconf = config['sqlalchemy.db1.url']
729 731 enc_key = get_encryption_key(config)
730 732
731 733 log.debug('making test db %s', dbconf)
732 734
733 735 dbmanage = DbManage(log_sql=False, dbconf=dbconf, root=config['here'],
734 736 tests=True, cli_args={'force_ask': True}, enc_key=enc_key)
735 737 dbmanage.create_tables(override=True)
736 738 dbmanage.set_db_version()
737 739 # for tests dynamically set new root paths based on generated content
738 740 dbmanage.create_settings(dbmanage.config_prompt(test_path))
739 741 dbmanage.create_default_user()
740 742 dbmanage.create_test_admin_and_users()
741 743 dbmanage.create_permissions()
742 744 dbmanage.populate_default_permissions()
743 745 Session().commit()
744 746
745 747
746 748 def create_test_repositories(test_path, config):
747 749 """
748 750 Creates test repositories in the temporary directory. Repositories are
749 751 extracted from archives within the rc_testdata package.
750 752 """
751 753 import rc_testdata
752 754 from rhodecode.tests import HG_REPO, GIT_REPO, SVN_REPO
753 755
754 756 log.debug('making test vcs repositories')
755 757
756 758 idx_path = config['search.location']
757 759 data_path = config['cache_dir']
758 760
759 761 # clean index and data
760 762 if idx_path and os.path.exists(idx_path):
761 763 log.debug('remove %s', idx_path)
762 764 shutil.rmtree(idx_path)
763 765
764 766 if data_path and os.path.exists(data_path):
765 767 log.debug('remove %s', data_path)
766 768 shutil.rmtree(data_path)
767 769
768 770 rc_testdata.extract_hg_dump('vcs_test_hg', jn(test_path, HG_REPO))
769 771 rc_testdata.extract_git_dump('vcs_test_git', jn(test_path, GIT_REPO))
770 772
771 773 # Note: Subversion is in the process of being integrated with the system,
772 774 # until we have a properly packed version of the test svn repository, this
773 775 # tries to copy over the repo from a package "rc_testdata"
774 776 svn_repo_path = rc_testdata.get_svn_repo_archive()
775 777 with tarfile.open(svn_repo_path) as tar:
776 778 tar.extractall(jn(test_path, SVN_REPO))
777 779
778 780
779 781 def password_changed(auth_user, session):
780 782 # Never report password change in case of default user or anonymous user.
781 783 if auth_user.username == User.DEFAULT_USER or auth_user.user_id is None:
782 784 return False
783 785
784 786 password_hash = md5(safe_bytes(auth_user.password)) if auth_user.password else None
785 787 rhodecode_user = session.get('rhodecode_user', {})
786 788 session_password_hash = rhodecode_user.get('password', '')
787 789 return password_hash != session_password_hash
788 790
789 791
790 792 def read_opensource_licenses():
791 793 global _license_cache
792 794
793 795 if not _license_cache:
794 796 licenses = pkg_resources.resource_string(
795 797 'rhodecode', 'config/licenses.json')
796 798 _license_cache = json.loads(licenses)
797 799
798 800 return _license_cache
799 801
800 802
801 803 def generate_platform_uuid():
802 804 """
803 805 Generates platform UUID based on it's name
804 806 """
805 807 import platform
806 808
807 809 try:
808 810 uuid_list = [platform.platform()]
809 811 return sha256_safe(':'.join(uuid_list))
810 812 except Exception as e:
811 813 log.error('Failed to generate host uuid: %s', e)
812 814 return 'UNDEFINED'
813 815
814 816
815 817 def send_test_email(recipients, email_body='TEST EMAIL'):
816 818 """
817 819 Simple code for generating test emails.
818 820 Usage::
819 821
820 822 from rhodecode.lib import utils
821 823 utils.send_test_email()
822 824 """
823 825 from rhodecode.lib.celerylib import tasks, run_task
824 826
825 827 email_body = email_body_plaintext = email_body
826 828 subject = f'SUBJECT FROM: {socket.gethostname()}'
827 829 tasks.send_email(recipients, subject, email_body_plaintext, email_body)
828 830
829 831
830 832 def call_service_api(ini_path, payload):
831 833 config = get_config(ini_path)
832 834 try:
833 835 host = config.get('app:main', 'app.service_api.host')
834 836 except NoOptionError:
835 837 raise ImproperlyConfiguredError(
836 838 "app.service_api.host is missing. "
837 839 "Please ensure that app.service_api.host and app.service_api.token are "
838 840 "defined inside of .ini configuration file."
839 841 )
840 842 try:
841 843 api_url = config.get('app:main', 'rhodecode.api.url')
842 844 except NoOptionError:
843 845 from rhodecode import api
844 846 log.debug('Cannot find rhodecode.api.url, setting API URL TO Default value')
845 847 api_url = api.DEFAULT_URL
846 848
847 849 payload.update({
848 850 'id': 'service',
849 851 'auth_token': config.get('app:main', 'app.service_api.token')
850 852 })
851 853
852 response = CurlSession().post(f'{host}{api_url}', json.dumps(payload))
854 response = CurlSession().post(urllib.parse.urljoin(host, api_url), json.dumps(payload))
853 855
854 856 if response.status_code != 200:
855 857 raise Exception("Service API responded with error")
856 858
857 859 return json.loads(response.content)['result']
General Comments 0
You need to be logged in to leave comments. Login now