##// END OF EJS Templates
fix(ssl): explicitly disable mercurial web_push ssl flag to prevent from errors about ssl required
super-admin -
r5537:7cab32ae default
parent child Browse files
Show More
@@ -1,707 +1,715 b''
1 1 # Copyright (C) 2010-2023 RhodeCode GmbH
2 2 #
3 3 # This program is free software: you can redistribute it and/or modify
4 4 # it under the terms of the GNU Affero General Public License, version 3
5 5 # (only), as published by the Free Software Foundation.
6 6 #
7 7 # This program is distributed in the hope that it will be useful,
8 8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 10 # GNU General Public License for more details.
11 11 #
12 12 # You should have received a copy of the GNU Affero General Public License
13 13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 14 #
15 15 # This program is dual-licensed. If you wish to learn more about the
16 16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18 18
19 19
20 20 import logging
21 21 import collections
22 22
23 23 import datetime
24 24 import formencode
25 25 import formencode.htmlfill
26 26
27 27 import rhodecode
28 28
29 29 from pyramid.httpexceptions import HTTPFound, HTTPNotFound
30 30 from pyramid.renderers import render
31 31 from pyramid.response import Response
32 32
33 33 from rhodecode.apps._base import BaseAppView
34 34 from rhodecode.apps._base.navigation import navigation_list
35 35 from rhodecode.apps.svn_support import config_keys
36 36 from rhodecode.lib import helpers as h
37 37 from rhodecode.lib.auth import (
38 38 LoginRequired, HasPermissionAllDecorator, CSRFRequired)
39 39 from rhodecode.lib.celerylib import tasks, run_task
40 40 from rhodecode.lib.str_utils import safe_str
41 41 from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_repo_store_path
42 42 from rhodecode.lib.utils2 import str2bool, AttributeDict
43 43 from rhodecode.lib.index import searcher_from_config
44 44
45 45 from rhodecode.model.db import RhodeCodeUi, Repository
46 46 from rhodecode.model.forms import (ApplicationSettingsForm,
47 47 ApplicationUiSettingsForm, ApplicationVisualisationForm,
48 48 LabsSettingsForm, IssueTrackerPatternsForm)
49 49 from rhodecode.model.permission import PermissionModel
50 50 from rhodecode.model.repo_group import RepoGroupModel
51 51
52 52 from rhodecode.model.scm import ScmModel
53 53 from rhodecode.model.notification import EmailNotificationModel
54 54 from rhodecode.model.meta import Session
55 55 from rhodecode.model.settings import (
56 56 IssueTrackerSettingsModel, VcsSettingsModel, SettingNotFound,
57 57 SettingsModel)
58 58
59 59
60 60 log = logging.getLogger(__name__)
61 61
62 62
63 63 class AdminSettingsView(BaseAppView):
64 64
65 65 def load_default_context(self):
66 66 c = self._get_local_tmpl_context()
67 67 c.labs_active = str2bool(
68 68 rhodecode.CONFIG.get('labs_settings_active', 'true'))
69 69 c.navlist = navigation_list(self.request)
70 70 return c
71 71
72 72 @classmethod
73 73 def _get_ui_settings(cls):
74 74 ret = RhodeCodeUi.query().all()
75 75
76 76 if not ret:
77 77 raise Exception('Could not get application ui settings !')
78 settings = {}
78 settings = {
79 # legacy param that needs to be kept
80 'web_push_ssl': False
81 }
79 82 for each in ret:
80 83 k = each.ui_key
81 84 v = each.ui_value
85 # skip some options if they are defined
86 if k in ['push_ssl']:
87 continue
88
82 89 if k == '/':
83 90 k = 'root_path'
84 91
85 92 if k in ['publish', 'enabled']:
86 93 v = str2bool(v)
87 94
88 95 if k.find('.') != -1:
89 96 k = k.replace('.', '_')
90 97
91 98 if each.ui_section in ['hooks', 'extensions']:
92 99 v = each.ui_active
93 100
94 101 settings[each.ui_section + '_' + k] = v
102
95 103 return settings
96 104
97 105 @classmethod
98 106 def _form_defaults(cls):
99 107 defaults = SettingsModel().get_all_settings()
100 108 defaults.update(cls._get_ui_settings())
101 109
102 110 defaults.update({
103 111 'new_svn_branch': '',
104 112 'new_svn_tag': '',
105 113 })
106 114 return defaults
107 115
108 116 @LoginRequired()
109 117 @HasPermissionAllDecorator('hg.admin')
110 118 def settings_vcs(self):
111 119 c = self.load_default_context()
112 120 c.active = 'vcs'
113 121 model = VcsSettingsModel()
114 122 c.svn_branch_patterns = model.get_global_svn_branch_patterns()
115 123 c.svn_tag_patterns = model.get_global_svn_tag_patterns()
116 124 c.svn_generate_config = rhodecode.ConfigGet().get_bool(config_keys.generate_config)
117 125 c.svn_config_path = rhodecode.ConfigGet().get_str(config_keys.config_file_path)
118 126 defaults = self._form_defaults()
119 127
120 128 model.create_largeobjects_dirs_if_needed(defaults['paths_root_path'])
121 129
122 130 data = render('rhodecode:templates/admin/settings/settings.mako',
123 131 self._get_template_context(c), self.request)
124 132 html = formencode.htmlfill.render(
125 133 data,
126 134 defaults=defaults,
127 135 encoding="UTF-8",
128 136 force_defaults=False
129 137 )
130 138 return Response(html)
131 139
132 140 @LoginRequired()
133 141 @HasPermissionAllDecorator('hg.admin')
134 142 @CSRFRequired()
135 143 def settings_vcs_update(self):
136 144 _ = self.request.translate
137 145 c = self.load_default_context()
138 146 c.active = 'vcs'
139 147
140 148 model = VcsSettingsModel()
141 149 c.svn_branch_patterns = model.get_global_svn_branch_patterns()
142 150 c.svn_tag_patterns = model.get_global_svn_tag_patterns()
143 151
144 152 c.svn_generate_config = rhodecode.ConfigGet().get_bool(config_keys.generate_config)
145 153 c.svn_config_path = rhodecode.ConfigGet().get_str(config_keys.config_file_path)
146 154 application_form = ApplicationUiSettingsForm(self.request.translate)()
147 155
148 156 try:
149 157 form_result = application_form.to_python(dict(self.request.POST))
150 158 except formencode.Invalid as errors:
151 159 h.flash(
152 160 _("Some form inputs contain invalid data."),
153 161 category='error')
154 162 data = render('rhodecode:templates/admin/settings/settings.mako',
155 163 self._get_template_context(c), self.request)
156 164 html = formencode.htmlfill.render(
157 165 data,
158 166 defaults=errors.value,
159 167 errors=errors.unpack_errors() or {},
160 168 prefix_error=False,
161 169 encoding="UTF-8",
162 170 force_defaults=False
163 171 )
164 172 return Response(html)
165 173
166 174 try:
167 175 model.update_global_hook_settings(form_result)
168 176
169 177 model.create_or_update_global_svn_settings(form_result)
170 178 model.create_or_update_global_hg_settings(form_result)
171 179 model.create_or_update_global_git_settings(form_result)
172 180 model.create_or_update_global_pr_settings(form_result)
173 181 except Exception:
174 182 log.exception("Exception while updating settings")
175 183 h.flash(_('Error occurred during updating '
176 184 'application settings'), category='error')
177 185 else:
178 186 Session().commit()
179 187 h.flash(_('Updated VCS settings'), category='success')
180 188 raise HTTPFound(h.route_path('admin_settings_vcs'))
181 189
182 190 data = render('rhodecode:templates/admin/settings/settings.mako',
183 191 self._get_template_context(c), self.request)
184 192 html = formencode.htmlfill.render(
185 193 data,
186 194 defaults=self._form_defaults(),
187 195 encoding="UTF-8",
188 196 force_defaults=False
189 197 )
190 198 return Response(html)
191 199
192 200 @LoginRequired()
193 201 @HasPermissionAllDecorator('hg.admin')
194 202 @CSRFRequired()
195 203 def settings_vcs_delete_svn_pattern(self):
196 204 delete_pattern_id = self.request.POST.get('delete_svn_pattern')
197 205 model = VcsSettingsModel()
198 206 try:
199 207 model.delete_global_svn_pattern(delete_pattern_id)
200 208 except SettingNotFound:
201 209 log.exception(
202 210 'Failed to delete svn_pattern with id %s', delete_pattern_id)
203 211 raise HTTPNotFound()
204 212
205 213 Session().commit()
206 214 return True
207 215
208 216 @LoginRequired()
209 217 @HasPermissionAllDecorator('hg.admin')
210 218 def settings_mapping(self):
211 219 c = self.load_default_context()
212 220 c.active = 'mapping'
213 221 c.storage_path = get_rhodecode_repo_store_path()
214 222 data = render('rhodecode:templates/admin/settings/settings.mako',
215 223 self._get_template_context(c), self.request)
216 224 html = formencode.htmlfill.render(
217 225 data,
218 226 defaults=self._form_defaults(),
219 227 encoding="UTF-8",
220 228 force_defaults=False
221 229 )
222 230 return Response(html)
223 231
224 232 @LoginRequired()
225 233 @HasPermissionAllDecorator('hg.admin')
226 234 @CSRFRequired()
227 235 def settings_mapping_update(self):
228 236 _ = self.request.translate
229 237 c = self.load_default_context()
230 238 c.active = 'mapping'
231 239 rm_obsolete = self.request.POST.get('destroy', False)
232 240 invalidate_cache = self.request.POST.get('invalidate', False)
233 241 log.debug('rescanning repo location with destroy obsolete=%s', rm_obsolete)
234 242
235 243 if invalidate_cache:
236 244 log.debug('invalidating all repositories cache')
237 245 for repo in Repository.get_all():
238 246 ScmModel().mark_for_invalidation(repo.repo_name, delete=True)
239 247
240 248 filesystem_repos = ScmModel().repo_scan()
241 249 added, removed = repo2db_mapper(filesystem_repos, rm_obsolete, force_hooks_rebuild=True)
242 250 PermissionModel().trigger_permission_flush()
243 251
244 252 def _repr(rm_repo):
245 253 return ', '.join(map(safe_str, rm_repo)) or '-'
246 254
247 255 h.flash(_('Repositories successfully '
248 256 'rescanned added: %s ; removed: %s') %
249 257 (_repr(added), _repr(removed)),
250 258 category='success')
251 259 raise HTTPFound(h.route_path('admin_settings_mapping'))
252 260
253 261 @LoginRequired()
254 262 @HasPermissionAllDecorator('hg.admin')
255 263 def settings_global(self):
256 264 c = self.load_default_context()
257 265 c.active = 'global'
258 266 c.personal_repo_group_default_pattern = RepoGroupModel()\
259 267 .get_personal_group_name_pattern()
260 268
261 269 data = render('rhodecode:templates/admin/settings/settings.mako',
262 270 self._get_template_context(c), self.request)
263 271 html = formencode.htmlfill.render(
264 272 data,
265 273 defaults=self._form_defaults(),
266 274 encoding="UTF-8",
267 275 force_defaults=False
268 276 )
269 277 return Response(html)
270 278
271 279 @LoginRequired()
272 280 @HasPermissionAllDecorator('hg.admin')
273 281 @CSRFRequired()
274 282 def settings_global_update(self):
275 283 _ = self.request.translate
276 284 c = self.load_default_context()
277 285 c.active = 'global'
278 286 c.personal_repo_group_default_pattern = RepoGroupModel()\
279 287 .get_personal_group_name_pattern()
280 288 application_form = ApplicationSettingsForm(self.request.translate)()
281 289 try:
282 290 form_result = application_form.to_python(dict(self.request.POST))
283 291 except formencode.Invalid as errors:
284 292 h.flash(
285 293 _("Some form inputs contain invalid data."),
286 294 category='error')
287 295 data = render('rhodecode:templates/admin/settings/settings.mako',
288 296 self._get_template_context(c), self.request)
289 297 html = formencode.htmlfill.render(
290 298 data,
291 299 defaults=errors.value,
292 300 errors=errors.unpack_errors() or {},
293 301 prefix_error=False,
294 302 encoding="UTF-8",
295 303 force_defaults=False
296 304 )
297 305 return Response(html)
298 306
299 307 settings = [
300 308 ('title', 'rhodecode_title', 'unicode'),
301 309 ('realm', 'rhodecode_realm', 'unicode'),
302 310 ('pre_code', 'rhodecode_pre_code', 'unicode'),
303 311 ('post_code', 'rhodecode_post_code', 'unicode'),
304 312 ('captcha_public_key', 'rhodecode_captcha_public_key', 'unicode'),
305 313 ('captcha_private_key', 'rhodecode_captcha_private_key', 'unicode'),
306 314 ('create_personal_repo_group', 'rhodecode_create_personal_repo_group', 'bool'),
307 315 ('personal_repo_group_pattern', 'rhodecode_personal_repo_group_pattern', 'unicode'),
308 316 ]
309 317
310 318 try:
311 319 for setting, form_key, type_ in settings:
312 320 sett = SettingsModel().create_or_update_setting(
313 321 setting, form_result[form_key], type_)
314 322 Session().add(sett)
315 323
316 324 Session().commit()
317 325 SettingsModel().invalidate_settings_cache()
318 326 h.flash(_('Updated application settings'), category='success')
319 327 except Exception:
320 328 log.exception("Exception while updating application settings")
321 329 h.flash(
322 330 _('Error occurred during updating application settings'),
323 331 category='error')
324 332
325 333 raise HTTPFound(h.route_path('admin_settings_global'))
326 334
327 335 @LoginRequired()
328 336 @HasPermissionAllDecorator('hg.admin')
329 337 def settings_visual(self):
330 338 c = self.load_default_context()
331 339 c.active = 'visual'
332 340
333 341 data = render('rhodecode:templates/admin/settings/settings.mako',
334 342 self._get_template_context(c), self.request)
335 343 html = formencode.htmlfill.render(
336 344 data,
337 345 defaults=self._form_defaults(),
338 346 encoding="UTF-8",
339 347 force_defaults=False
340 348 )
341 349 return Response(html)
342 350
343 351 @LoginRequired()
344 352 @HasPermissionAllDecorator('hg.admin')
345 353 @CSRFRequired()
346 354 def settings_visual_update(self):
347 355 _ = self.request.translate
348 356 c = self.load_default_context()
349 357 c.active = 'visual'
350 358 application_form = ApplicationVisualisationForm(self.request.translate)()
351 359 try:
352 360 form_result = application_form.to_python(dict(self.request.POST))
353 361 except formencode.Invalid as errors:
354 362 h.flash(
355 363 _("Some form inputs contain invalid data."),
356 364 category='error')
357 365 data = render('rhodecode:templates/admin/settings/settings.mako',
358 366 self._get_template_context(c), self.request)
359 367 html = formencode.htmlfill.render(
360 368 data,
361 369 defaults=errors.value,
362 370 errors=errors.unpack_errors() or {},
363 371 prefix_error=False,
364 372 encoding="UTF-8",
365 373 force_defaults=False
366 374 )
367 375 return Response(html)
368 376
369 377 try:
370 378 settings = [
371 379 ('show_public_icon', 'rhodecode_show_public_icon', 'bool'),
372 380 ('show_private_icon', 'rhodecode_show_private_icon', 'bool'),
373 381 ('stylify_metatags', 'rhodecode_stylify_metatags', 'bool'),
374 382 ('repository_fields', 'rhodecode_repository_fields', 'bool'),
375 383 ('dashboard_items', 'rhodecode_dashboard_items', 'int'),
376 384 ('admin_grid_items', 'rhodecode_admin_grid_items', 'int'),
377 385 ('show_version', 'rhodecode_show_version', 'bool'),
378 386 ('use_gravatar', 'rhodecode_use_gravatar', 'bool'),
379 387 ('markup_renderer', 'rhodecode_markup_renderer', 'unicode'),
380 388 ('gravatar_url', 'rhodecode_gravatar_url', 'unicode'),
381 389 ('clone_uri_tmpl', 'rhodecode_clone_uri_tmpl', 'unicode'),
382 390 ('clone_uri_id_tmpl', 'rhodecode_clone_uri_id_tmpl', 'unicode'),
383 391 ('clone_uri_ssh_tmpl', 'rhodecode_clone_uri_ssh_tmpl', 'unicode'),
384 392 ('support_url', 'rhodecode_support_url', 'unicode'),
385 393 ('show_revision_number', 'rhodecode_show_revision_number', 'bool'),
386 394 ('show_sha_length', 'rhodecode_show_sha_length', 'int'),
387 395 ]
388 396 for setting, form_key, type_ in settings:
389 397 sett = SettingsModel().create_or_update_setting(
390 398 setting, form_result[form_key], type_)
391 399 Session().add(sett)
392 400
393 401 Session().commit()
394 402 SettingsModel().invalidate_settings_cache()
395 403 h.flash(_('Updated visualisation settings'), category='success')
396 404 except Exception:
397 405 log.exception("Exception updating visualization settings")
398 406 h.flash(_('Error occurred during updating '
399 407 'visualisation settings'),
400 408 category='error')
401 409
402 410 raise HTTPFound(h.route_path('admin_settings_visual'))
403 411
404 412 @LoginRequired()
405 413 @HasPermissionAllDecorator('hg.admin')
406 414 def settings_issuetracker(self):
407 415 c = self.load_default_context()
408 416 c.active = 'issuetracker'
409 417 defaults = c.rc_config
410 418
411 419 entry_key = 'rhodecode_issuetracker_pat_'
412 420
413 421 c.issuetracker_entries = {}
414 422 for k, v in defaults.items():
415 423 if k.startswith(entry_key):
416 424 uid = k[len(entry_key):]
417 425 c.issuetracker_entries[uid] = None
418 426
419 427 for uid in c.issuetracker_entries:
420 428 c.issuetracker_entries[uid] = AttributeDict({
421 429 'pat': defaults.get('rhodecode_issuetracker_pat_' + uid),
422 430 'url': defaults.get('rhodecode_issuetracker_url_' + uid),
423 431 'pref': defaults.get('rhodecode_issuetracker_pref_' + uid),
424 432 'desc': defaults.get('rhodecode_issuetracker_desc_' + uid),
425 433 })
426 434
427 435 return self._get_template_context(c)
428 436
429 437 @LoginRequired()
430 438 @HasPermissionAllDecorator('hg.admin')
431 439 @CSRFRequired()
432 440 def settings_issuetracker_test(self):
433 441 error_container = []
434 442
435 443 urlified_commit = h.urlify_commit_message(
436 444 self.request.POST.get('test_text', ''),
437 445 'repo_group/test_repo1', error_container=error_container)
438 446 if error_container:
439 447 def converter(inp):
440 448 return h.html_escape(inp)
441 449
442 450 return 'ERRORS: ' + '\n'.join(map(converter, error_container))
443 451
444 452 return urlified_commit
445 453
446 454 @LoginRequired()
447 455 @HasPermissionAllDecorator('hg.admin')
448 456 @CSRFRequired()
449 457 def settings_issuetracker_update(self):
450 458 _ = self.request.translate
451 459 self.load_default_context()
452 460 settings_model = IssueTrackerSettingsModel()
453 461
454 462 try:
455 463 form = IssueTrackerPatternsForm(self.request.translate)()
456 464 data = form.to_python(self.request.POST)
457 465 except formencode.Invalid as errors:
458 466 log.exception('Failed to add new pattern')
459 467 error = errors
460 468 h.flash(_(f'Invalid issue tracker pattern: {error}'),
461 469 category='error')
462 470 raise HTTPFound(h.route_path('admin_settings_issuetracker'))
463 471
464 472 if data:
465 473 for uid in data.get('delete_patterns', []):
466 474 settings_model.delete_entries(uid)
467 475
468 476 for pattern in data.get('patterns', []):
469 477 for setting, value, type_ in pattern:
470 478 sett = settings_model.create_or_update_setting(
471 479 setting, value, type_)
472 480 Session().add(sett)
473 481
474 482 Session().commit()
475 483
476 484 SettingsModel().invalidate_settings_cache()
477 485 h.flash(_('Updated issue tracker entries'), category='success')
478 486 raise HTTPFound(h.route_path('admin_settings_issuetracker'))
479 487
480 488 @LoginRequired()
481 489 @HasPermissionAllDecorator('hg.admin')
482 490 @CSRFRequired()
483 491 def settings_issuetracker_delete(self):
484 492 _ = self.request.translate
485 493 self.load_default_context()
486 494 uid = self.request.POST.get('uid')
487 495 try:
488 496 IssueTrackerSettingsModel().delete_entries(uid)
489 497 except Exception:
490 498 log.exception('Failed to delete issue tracker setting %s', uid)
491 499 raise HTTPNotFound()
492 500
493 501 SettingsModel().invalidate_settings_cache()
494 502 h.flash(_('Removed issue tracker entry.'), category='success')
495 503
496 504 return {'deleted': uid}
497 505
498 506 @LoginRequired()
499 507 @HasPermissionAllDecorator('hg.admin')
500 508 def settings_email(self):
501 509 c = self.load_default_context()
502 510 c.active = 'email'
503 511 c.rhodecode_ini = rhodecode.CONFIG
504 512
505 513 data = render('rhodecode:templates/admin/settings/settings.mako',
506 514 self._get_template_context(c), self.request)
507 515 html = formencode.htmlfill.render(
508 516 data,
509 517 defaults=self._form_defaults(),
510 518 encoding="UTF-8",
511 519 force_defaults=False
512 520 )
513 521 return Response(html)
514 522
515 523 @LoginRequired()
516 524 @HasPermissionAllDecorator('hg.admin')
517 525 @CSRFRequired()
518 526 def settings_email_update(self):
519 527 _ = self.request.translate
520 528 c = self.load_default_context()
521 529 c.active = 'email'
522 530
523 531 test_email = self.request.POST.get('test_email')
524 532
525 533 if not test_email:
526 534 h.flash(_('Please enter email address'), category='error')
527 535 raise HTTPFound(h.route_path('admin_settings_email'))
528 536
529 537 email_kwargs = {
530 538 'date': datetime.datetime.now(),
531 539 'user': self._rhodecode_db_user
532 540 }
533 541
534 542 (subject, email_body, email_body_plaintext) = EmailNotificationModel().render_email(
535 543 EmailNotificationModel.TYPE_EMAIL_TEST, **email_kwargs)
536 544
537 545 recipients = [test_email] if test_email else None
538 546
539 547 run_task(tasks.send_email, recipients, subject,
540 548 email_body_plaintext, email_body)
541 549
542 550 h.flash(_('Send email task created'), category='success')
543 551 raise HTTPFound(h.route_path('admin_settings_email'))
544 552
545 553 @LoginRequired()
546 554 @HasPermissionAllDecorator('hg.admin')
547 555 def settings_hooks(self):
548 556 c = self.load_default_context()
549 557 c.active = 'hooks'
550 558
551 559 model = SettingsModel()
552 560 c.hooks = model.get_builtin_hooks()
553 561 c.custom_hooks = model.get_custom_hooks()
554 562
555 563 data = render('rhodecode:templates/admin/settings/settings.mako',
556 564 self._get_template_context(c), self.request)
557 565 html = formencode.htmlfill.render(
558 566 data,
559 567 defaults=self._form_defaults(),
560 568 encoding="UTF-8",
561 569 force_defaults=False
562 570 )
563 571 return Response(html)
564 572
565 573 @LoginRequired()
566 574 @HasPermissionAllDecorator('hg.admin')
567 575 @CSRFRequired()
568 576 def settings_hooks_update(self):
569 577 _ = self.request.translate
570 578 c = self.load_default_context()
571 579 c.active = 'hooks'
572 580 if c.visual.allow_custom_hooks_settings:
573 581 ui_key = self.request.POST.get('new_hook_ui_key')
574 582 ui_value = self.request.POST.get('new_hook_ui_value')
575 583
576 584 hook_id = self.request.POST.get('hook_id')
577 585 new_hook = False
578 586
579 587 model = SettingsModel()
580 588 try:
581 589 if ui_value and ui_key:
582 590 model.create_or_update_hook(ui_key, ui_value)
583 591 h.flash(_('Added new hook'), category='success')
584 592 new_hook = True
585 593 elif hook_id:
586 594 RhodeCodeUi.delete(hook_id)
587 595 Session().commit()
588 596
589 597 # check for edits
590 598 update = False
591 599 _d = self.request.POST.dict_of_lists()
592 600 for k, v in zip(_d.get('hook_ui_key', []),
593 601 _d.get('hook_ui_value_new', [])):
594 602 model.create_or_update_hook(k, v)
595 603 update = True
596 604
597 605 if update and not new_hook:
598 606 h.flash(_('Updated hooks'), category='success')
599 607 Session().commit()
600 608 except Exception:
601 609 log.exception("Exception during hook creation")
602 610 h.flash(_('Error occurred during hook creation'),
603 611 category='error')
604 612
605 613 raise HTTPFound(h.route_path('admin_settings_hooks'))
606 614
607 615 @LoginRequired()
608 616 @HasPermissionAllDecorator('hg.admin')
609 617 def settings_search(self):
610 618 c = self.load_default_context()
611 619 c.active = 'search'
612 620
613 621 c.searcher = searcher_from_config(self.request.registry.settings)
614 622 c.statistics = c.searcher.statistics(self.request.translate)
615 623
616 624 return self._get_template_context(c)
617 625
618 626 @LoginRequired()
619 627 @HasPermissionAllDecorator('hg.admin')
620 628 def settings_labs(self):
621 629 c = self.load_default_context()
622 630 if not c.labs_active:
623 631 raise HTTPFound(h.route_path('admin_settings'))
624 632
625 633 c.active = 'labs'
626 634 c.lab_settings = _LAB_SETTINGS
627 635
628 636 data = render('rhodecode:templates/admin/settings/settings.mako',
629 637 self._get_template_context(c), self.request)
630 638 html = formencode.htmlfill.render(
631 639 data,
632 640 defaults=self._form_defaults(),
633 641 encoding="UTF-8",
634 642 force_defaults=False
635 643 )
636 644 return Response(html)
637 645
638 646 @LoginRequired()
639 647 @HasPermissionAllDecorator('hg.admin')
640 648 @CSRFRequired()
641 649 def settings_labs_update(self):
642 650 _ = self.request.translate
643 651 c = self.load_default_context()
644 652 c.active = 'labs'
645 653
646 654 application_form = LabsSettingsForm(self.request.translate)()
647 655 try:
648 656 form_result = application_form.to_python(dict(self.request.POST))
649 657 except formencode.Invalid as errors:
650 658 h.flash(
651 659 _("Some form inputs contain invalid data."),
652 660 category='error')
653 661 data = render('rhodecode:templates/admin/settings/settings.mako',
654 662 self._get_template_context(c), self.request)
655 663 html = formencode.htmlfill.render(
656 664 data,
657 665 defaults=errors.value,
658 666 errors=errors.unpack_errors() or {},
659 667 prefix_error=False,
660 668 encoding="UTF-8",
661 669 force_defaults=False
662 670 )
663 671 return Response(html)
664 672
665 673 try:
666 674 session = Session()
667 675 for setting in _LAB_SETTINGS:
668 676 setting_name = setting.key[len('rhodecode_'):]
669 677 sett = SettingsModel().create_or_update_setting(
670 678 setting_name, form_result[setting.key], setting.type)
671 679 session.add(sett)
672 680
673 681 except Exception:
674 682 log.exception('Exception while updating lab settings')
675 683 h.flash(_('Error occurred during updating labs settings'),
676 684 category='error')
677 685 else:
678 686 Session().commit()
679 687 SettingsModel().invalidate_settings_cache()
680 688 h.flash(_('Updated Labs settings'), category='success')
681 689 raise HTTPFound(h.route_path('admin_settings_labs'))
682 690
683 691 data = render('rhodecode:templates/admin/settings/settings.mako',
684 692 self._get_template_context(c), self.request)
685 693 html = formencode.htmlfill.render(
686 694 data,
687 695 defaults=self._form_defaults(),
688 696 encoding="UTF-8",
689 697 force_defaults=False
690 698 )
691 699 return Response(html)
692 700
693 701
694 702 # :param key: name of the setting including the 'rhodecode_' prefix
695 703 # :param type: the RhodeCodeSetting type to use.
696 704 # :param group: the i18ned group in which we should dispaly this setting
697 705 # :param label: the i18ned label we should display for this setting
698 706 # :param help: the i18ned help we should dispaly for this setting
699 707 LabSetting = collections.namedtuple(
700 708 'LabSetting', ('key', 'type', 'group', 'label', 'help'))
701 709
702 710
703 711 # This list has to be kept in sync with the form
704 712 # rhodecode.model.forms.LabsSettingsForm.
705 713 _LAB_SETTINGS = [
706 714
707 715 ]
@@ -1,834 +1,835 b''
1 1 # Copyright (C) 2010-2023 RhodeCode GmbH
2 2 #
3 3 # This program is free software: you can redistribute it and/or modify
4 4 # it under the terms of the GNU Affero General Public License, version 3
5 5 # (only), as published by the Free Software Foundation.
6 6 #
7 7 # This program is distributed in the hope that it will be useful,
8 8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 10 # GNU General Public License for more details.
11 11 #
12 12 # You should have received a copy of the GNU Affero General Public License
13 13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 14 #
15 15 # This program is dual-licensed. If you wish to learn more about the
16 16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18 18
19 19 """
20 20 Utilities library for RhodeCode
21 21 """
22 22
23 23 import datetime
24 24
25 25 import decorator
26 26 import logging
27 27 import os
28 28 import re
29 29 import sys
30 30 import shutil
31 31 import socket
32 32 import tempfile
33 33 import traceback
34 34 import tarfile
35 35
36 36 from functools import wraps
37 37 from os.path import join as jn
38 38
39 39 import paste
40 40 import pkg_resources
41 41 from webhelpers2.text import collapse, strip_tags, convert_accented_entities, convert_misc_entities
42 42
43 43 from mako import exceptions
44 44
45 45 from rhodecode import ConfigGet
46 46 from rhodecode.lib.hash_utils import sha256_safe, md5, sha1
47 47 from rhodecode.lib.type_utils import AttributeDict
48 48 from rhodecode.lib.str_utils import safe_bytes, safe_str
49 49 from rhodecode.lib.vcs.backends.base import Config
50 50 from rhodecode.lib.vcs.exceptions import VCSError
51 51 from rhodecode.lib.vcs.utils.helpers import get_scm, get_scm_backend
52 52 from rhodecode.lib.ext_json import sjson as json
53 53 from rhodecode.model import meta
54 54 from rhodecode.model.db import (
55 55 Repository, User, RhodeCodeUi, UserLog, RepoGroup, UserGroup)
56 56 from rhodecode.model.meta import Session
57 57
58 58
59 59 log = logging.getLogger(__name__)
60 60
61 61 REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}__.*')
62 62
63 63 # String which contains characters that are not allowed in slug names for
64 64 # repositories or repository groups. It is properly escaped to use it in
65 65 # regular expressions.
66 66 SLUG_BAD_CHARS = re.escape(r'`?=[]\;\'"<>,/~!@#$%^&*()+{}|:')
67 67
68 68 # Regex that matches forbidden characters in repo/group slugs.
69 69 SLUG_BAD_CHAR_RE = re.compile(r'[{}\x00-\x08\x0b-\x0c\x0e-\x1f]'.format(SLUG_BAD_CHARS))
70 70
71 71 # Regex that matches allowed characters in repo/group slugs.
72 72 SLUG_GOOD_CHAR_RE = re.compile(r'[^{}]'.format(SLUG_BAD_CHARS))
73 73
74 74 # Regex that matches whole repo/group slugs.
75 75 SLUG_RE = re.compile(r'[^{}]+'.format(SLUG_BAD_CHARS))
76 76
77 77 _license_cache = None
78 78
79 79
80 80 def adopt_for_celery(func):
81 81 """
82 82 Decorator designed to adopt hooks (from rhodecode.lib.hooks_base)
83 83 for further usage as a celery tasks.
84 84 """
85 85 @wraps(func)
86 86 def wrapper(extras):
87 87 extras = AttributeDict(extras)
88 88 try:
89 89 # HooksResponse implements to_json method which must be used there.
90 90 return func(extras).to_json()
91 91 except Exception as e:
92 92 return {'status': 128, 'exception': type(e).__name__, 'exception_args': e.args}
93 93 return wrapper
94 94
95 95
96 96 def repo_name_slug(value):
97 97 """
98 98 Return slug of name of repository
99 99 This function is called on each creation/modification
100 100 of repository to prevent bad names in repo
101 101 """
102 102
103 103 replacement_char = '-'
104 104
105 105 slug = strip_tags(value)
106 106 slug = convert_accented_entities(slug)
107 107 slug = convert_misc_entities(slug)
108 108
109 109 slug = SLUG_BAD_CHAR_RE.sub('', slug)
110 110 slug = re.sub(r'[\s]+', '-', slug)
111 111 slug = collapse(slug, replacement_char)
112 112
113 113 return slug
114 114
115 115
116 116 #==============================================================================
117 117 # PERM DECORATOR HELPERS FOR EXTRACTING NAMES FOR PERM CHECKS
118 118 #==============================================================================
119 119 def get_repo_slug(request):
120 120 _repo = ''
121 121
122 122 if hasattr(request, 'db_repo_name'):
123 123 # if our requests has set db reference use it for name, this
124 124 # translates the example.com/_<id> into proper repo names
125 125 _repo = request.db_repo_name
126 126 elif getattr(request, 'matchdict', None):
127 127 # pyramid
128 128 _repo = request.matchdict.get('repo_name')
129 129
130 130 if _repo:
131 131 _repo = _repo.rstrip('/')
132 132 return _repo
133 133
134 134
135 135 def get_repo_group_slug(request):
136 136 _group = ''
137 137 if hasattr(request, 'db_repo_group'):
138 138 # if our requests has set db reference use it for name, this
139 139 # translates the example.com/_<id> into proper repo group names
140 140 _group = request.db_repo_group.group_name
141 141 elif getattr(request, 'matchdict', None):
142 142 # pyramid
143 143 _group = request.matchdict.get('repo_group_name')
144 144
145 145 if _group:
146 146 _group = _group.rstrip('/')
147 147 return _group
148 148
149 149
150 150 def get_user_group_slug(request):
151 151 _user_group = ''
152 152
153 153 if hasattr(request, 'db_user_group'):
154 154 _user_group = request.db_user_group.users_group_name
155 155 elif getattr(request, 'matchdict', None):
156 156 # pyramid
157 157 _user_group = request.matchdict.get('user_group_id')
158 158 _user_group_name = request.matchdict.get('user_group_name')
159 159 try:
160 160 if _user_group:
161 161 _user_group = UserGroup.get(_user_group)
162 162 elif _user_group_name:
163 163 _user_group = UserGroup.get_by_group_name(_user_group_name)
164 164
165 165 if _user_group:
166 166 _user_group = _user_group.users_group_name
167 167 except Exception:
168 168 log.exception('Failed to get user group by id and name')
169 169 # catch all failures here
170 170 return None
171 171
172 172 return _user_group
173 173
174 174
175 175 def get_filesystem_repos(path, recursive=False, skip_removed_repos=True):
176 176 """
177 177 Scans given path for repos and return (name,(type,path)) tuple
178 178
179 179 :param path: path to scan for repositories
180 180 :param recursive: recursive search and return names with subdirs in front
181 181 """
182 182
183 183 # remove ending slash for better results
184 184 path = path.rstrip(os.sep)
185 185 log.debug('now scanning in %s location recursive:%s...', path, recursive)
186 186
187 187 def _get_repos(p):
188 188 dirpaths = get_dirpaths(p)
189 189 if not _is_dir_writable(p):
190 190 log.warning('repo path without write access: %s', p)
191 191
192 192 for dirpath in dirpaths:
193 193 if os.path.isfile(os.path.join(p, dirpath)):
194 194 continue
195 195 cur_path = os.path.join(p, dirpath)
196 196
197 197 # skip removed repos
198 198 if skip_removed_repos and REMOVED_REPO_PAT.match(dirpath):
199 199 continue
200 200
201 201 #skip .<somethin> dirs
202 202 if dirpath.startswith('.'):
203 203 continue
204 204
205 205 try:
206 206 scm_info = get_scm(cur_path)
207 207 yield scm_info[1].split(path, 1)[-1].lstrip(os.sep), scm_info
208 208 except VCSError:
209 209 if not recursive:
210 210 continue
211 211 #check if this dir containts other repos for recursive scan
212 212 rec_path = os.path.join(p, dirpath)
213 213 if os.path.isdir(rec_path):
214 214 yield from _get_repos(rec_path)
215 215
216 216 return _get_repos(path)
217 217
218 218
219 219 def get_dirpaths(p: str) -> list:
220 220 try:
221 221 # OS-independable way of checking if we have at least read-only
222 222 # access or not.
223 223 dirpaths = os.listdir(p)
224 224 except OSError:
225 225 log.warning('ignoring repo path without read access: %s', p)
226 226 return []
227 227
228 228 # os.listpath has a tweak: If a unicode is passed into it, then it tries to
229 229 # decode paths and suddenly returns unicode objects itself. The items it
230 230 # cannot decode are returned as strings and cause issues.
231 231 #
232 232 # Those paths are ignored here until a solid solution for path handling has
233 233 # been built.
234 234 expected_type = type(p)
235 235
236 236 def _has_correct_type(item):
237 237 if type(item) is not expected_type:
238 238 log.error(
239 239 "Ignoring path %s since it cannot be decoded into str.",
240 240 # Using "repr" to make sure that we see the byte value in case
241 241 # of support.
242 242 repr(item))
243 243 return False
244 244 return True
245 245
246 246 dirpaths = [item for item in dirpaths if _has_correct_type(item)]
247 247
248 248 return dirpaths
249 249
250 250
251 251 def _is_dir_writable(path):
252 252 """
253 253 Probe if `path` is writable.
254 254
255 255 Due to trouble on Cygwin / Windows, this is actually probing if it is
256 256 possible to create a file inside of `path`, stat does not produce reliable
257 257 results in this case.
258 258 """
259 259 try:
260 260 with tempfile.TemporaryFile(dir=path):
261 261 pass
262 262 except OSError:
263 263 return False
264 264 return True
265 265
266 266
267 267 def is_valid_repo(repo_name, base_path, expect_scm=None, explicit_scm=None, config=None):
268 268 """
269 269 Returns True if given path is a valid repository False otherwise.
270 270 If expect_scm param is given also, compare if given scm is the same
271 271 as expected from scm parameter. If explicit_scm is given don't try to
272 272 detect the scm, just use the given one to check if repo is valid
273 273
274 274 :param repo_name:
275 275 :param base_path:
276 276 :param expect_scm:
277 277 :param explicit_scm:
278 278 :param config:
279 279
280 280 :return True: if given path is a valid repository
281 281 """
282 282 full_path = os.path.join(safe_str(base_path), safe_str(repo_name))
283 283 log.debug('Checking if `%s` is a valid path for repository. '
284 284 'Explicit type: %s', repo_name, explicit_scm)
285 285
286 286 try:
287 287 if explicit_scm:
288 288 detected_scms = [get_scm_backend(explicit_scm)(
289 289 full_path, config=config).alias]
290 290 else:
291 291 detected_scms = get_scm(full_path)
292 292
293 293 if expect_scm:
294 294 return detected_scms[0] == expect_scm
295 295 log.debug('path: %s is an vcs object:%s', full_path, detected_scms)
296 296 return True
297 297 except VCSError:
298 298 log.debug('path: %s is not a valid repo !', full_path)
299 299 return False
300 300
301 301
302 302 def is_valid_repo_group(repo_group_name, base_path, skip_path_check=False):
303 303 """
304 304 Returns True if a given path is a repository group, False otherwise
305 305
306 306 :param repo_group_name:
307 307 :param base_path:
308 308 """
309 309 full_path = os.path.join(safe_str(base_path), safe_str(repo_group_name))
310 310 log.debug('Checking if `%s` is a valid path for repository group',
311 311 repo_group_name)
312 312
313 313 # check if it's not a repo
314 314 if is_valid_repo(repo_group_name, base_path):
315 315 log.debug('Repo called %s exist, it is not a valid repo group', repo_group_name)
316 316 return False
317 317
318 318 try:
319 319 # we need to check bare git repos at higher level
320 320 # since we might match branches/hooks/info/objects or possible
321 321 # other things inside bare git repo
322 322 maybe_repo = os.path.dirname(full_path)
323 323 if maybe_repo == base_path:
324 324 # skip root level repo check; we know root location CANNOT BE a repo group
325 325 return False
326 326
327 327 scm_ = get_scm(maybe_repo)
328 328 log.debug('path: %s is a vcs object:%s, not valid repo group', full_path, scm_)
329 329 return False
330 330 except VCSError:
331 331 pass
332 332
333 333 # check if it's a valid path
334 334 if skip_path_check or os.path.isdir(full_path):
335 335 log.debug('path: %s is a valid repo group !', full_path)
336 336 return True
337 337
338 338 log.debug('path: %s is not a valid repo group !', full_path)
339 339 return False
340 340
341 341
342 342 def ask_ok(prompt, retries=4, complaint='[y]es or [n]o please!'):
343 343 while True:
344 344 ok = input(prompt)
345 345 if ok.lower() in ('y', 'ye', 'yes'):
346 346 return True
347 347 if ok.lower() in ('n', 'no', 'nop', 'nope'):
348 348 return False
349 349 retries = retries - 1
350 350 if retries < 0:
351 351 raise OSError
352 352 print(complaint)
353 353
354 354 # propagated from mercurial documentation
355 355 ui_sections = [
356 356 'alias', 'auth',
357 357 'decode/encode', 'defaults',
358 358 'diff', 'email',
359 359 'extensions', 'format',
360 360 'merge-patterns', 'merge-tools',
361 361 'hooks', 'http_proxy',
362 362 'smtp', 'patch',
363 363 'paths', 'profiling',
364 364 'server', 'trusted',
365 365 'ui', 'web', ]
366 366
367 367
368 368 def prepare_config_data(clear_session=True, repo=None):
369 369 """
370 370 Read the configuration data from the database, *.ini files and return configuration
371 371 tuples.
372 372 """
373 373 from rhodecode.model.settings import VcsSettingsModel
374 374
375 config = []
376
377 375 sa = meta.Session()
378 376 settings_model = VcsSettingsModel(repo=repo, sa=sa)
379 377
380 378 ui_settings = settings_model.get_ui_settings()
381 379
382 380 ui_data = []
381 config = [
382 ('web', 'push_ssl', 'false'),
383 ]
383 384 for setting in ui_settings:
384 385 # Todo: remove this section once transition to *.ini files will be completed
385 386 if setting.section in ('largefiles', 'vcs_git_lfs'):
386 387 if setting.key != 'enabled':
387 388 continue
388 389 if setting.active:
389 390 ui_data.append((setting.section, setting.key, setting.value))
390 391 config.append((
391 392 safe_str(setting.section), safe_str(setting.key),
392 393 safe_str(setting.value)))
393 394 if setting.key == 'push_ssl':
394 395 # force set push_ssl requirement to False this is deprecated, and we must force it to False
395 396 config.append((
396 397 safe_str(setting.section), safe_str(setting.key), False))
397 398 config_getter = ConfigGet()
398 399 config.append(('vcs_git_lfs', 'store_location', config_getter.get_str('vcs.git.lfs.storage_location')))
399 400 config.append(('largefiles', 'usercache', config_getter.get_str('vcs.hg.largefiles.storage_location')))
400 401 log.debug(
401 402 'settings ui from db@repo[%s]: %s',
402 403 repo,
403 404 ','.join(['[{}] {}={}'.format(*s) for s in ui_data]))
404 405 if clear_session:
405 406 meta.Session.remove()
406 407
407 408 # TODO: mikhail: probably it makes no sense to re-read hooks information.
408 409 # It's already there and activated/deactivated
409 410 skip_entries = []
410 411 enabled_hook_classes = get_enabled_hook_classes(ui_settings)
411 412 if 'pull' not in enabled_hook_classes:
412 413 skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PULL))
413 414 if 'push' not in enabled_hook_classes:
414 415 skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PUSH))
415 416 skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRETX_PUSH))
416 417 skip_entries.append(('hooks', RhodeCodeUi.HOOK_PUSH_KEY))
417 418
418 419 config = [entry for entry in config if entry[:2] not in skip_entries]
419 420
420 421 return config
421 422
422 423
423 424 def make_db_config(clear_session=True, repo=None):
424 425 """
425 426 Create a :class:`Config` instance based on the values in the database.
426 427 """
427 428 config = Config()
428 429 config_data = prepare_config_data(clear_session=clear_session, repo=repo)
429 430 for section, option, value in config_data:
430 431 config.set(section, option, value)
431 432 return config
432 433
433 434
434 435 def get_enabled_hook_classes(ui_settings):
435 436 """
436 437 Return the enabled hook classes.
437 438
438 439 :param ui_settings: List of ui_settings as returned
439 440 by :meth:`VcsSettingsModel.get_ui_settings`
440 441
441 442 :return: a list with the enabled hook classes. The order is not guaranteed.
442 443 :rtype: list
443 444 """
444 445 enabled_hooks = []
445 446 active_hook_keys = [
446 447 key for section, key, value, active in ui_settings
447 448 if section == 'hooks' and active]
448 449
449 450 hook_names = {
450 451 RhodeCodeUi.HOOK_PUSH: 'push',
451 452 RhodeCodeUi.HOOK_PULL: 'pull',
452 453 RhodeCodeUi.HOOK_REPO_SIZE: 'repo_size'
453 454 }
454 455
455 456 for key in active_hook_keys:
456 457 hook = hook_names.get(key)
457 458 if hook:
458 459 enabled_hooks.append(hook)
459 460
460 461 return enabled_hooks
461 462
462 463
463 464 def set_rhodecode_config(config):
464 465 """
465 466 Updates pyramid config with new settings from database
466 467
467 468 :param config:
468 469 """
469 470 from rhodecode.model.settings import SettingsModel
470 471 app_settings = SettingsModel().get_all_settings()
471 472
472 473 for k, v in list(app_settings.items()):
473 474 config[k] = v
474 475
475 476
476 477 def get_rhodecode_realm():
477 478 """
478 479 Return the rhodecode realm from database.
479 480 """
480 481 from rhodecode.model.settings import SettingsModel
481 482 realm = SettingsModel().get_setting_by_name('realm')
482 483 return safe_str(realm.app_settings_value)
483 484
484 485
485 486 def get_rhodecode_repo_store_path():
486 487 """
487 488 Returns the base path. The base path is the filesystem path which points
488 489 to the repository store.
489 490 """
490 491
491 492 import rhodecode
492 493 return rhodecode.CONFIG['repo_store.path']
493 494
494 495
495 496 def map_groups(path):
496 497 """
497 498 Given a full path to a repository, create all nested groups that this
498 499 repo is inside. This function creates parent-child relationships between
499 500 groups and creates default perms for all new groups.
500 501
501 502 :param paths: full path to repository
502 503 """
503 504 from rhodecode.model.repo_group import RepoGroupModel
504 505 sa = meta.Session()
505 506 groups = path.split(Repository.NAME_SEP)
506 507 parent = None
507 508 group = None
508 509
509 510 # last element is repo in nested groups structure
510 511 groups = groups[:-1]
511 512 rgm = RepoGroupModel(sa)
512 513 owner = User.get_first_super_admin()
513 514 for lvl, group_name in enumerate(groups):
514 515 group_name = '/'.join(groups[:lvl] + [group_name])
515 516 group = RepoGroup.get_by_group_name(group_name)
516 517 desc = '%s group' % group_name
517 518
518 519 # skip folders that are now removed repos
519 520 if REMOVED_REPO_PAT.match(group_name):
520 521 break
521 522
522 523 if group is None:
523 524 log.debug('creating group level: %s group_name: %s',
524 525 lvl, group_name)
525 526 group = RepoGroup(group_name, parent)
526 527 group.group_description = desc
527 528 group.user = owner
528 529 sa.add(group)
529 530 perm_obj = rgm._create_default_perms(group)
530 531 sa.add(perm_obj)
531 532 sa.flush()
532 533
533 534 parent = group
534 535 return group
535 536
536 537
537 538 def repo2db_mapper(initial_repo_list, remove_obsolete=False, force_hooks_rebuild=False):
538 539 """
539 540 maps all repos given in initial_repo_list, non existing repositories
540 541 are created, if remove_obsolete is True it also checks for db entries
541 542 that are not in initial_repo_list and removes them.
542 543
543 544 :param initial_repo_list: list of repositories found by scanning methods
544 545 :param remove_obsolete: check for obsolete entries in database
545 546 """
546 547 from rhodecode.model.repo import RepoModel
547 548 from rhodecode.model.repo_group import RepoGroupModel
548 549 from rhodecode.model.settings import SettingsModel
549 550
550 551 sa = meta.Session()
551 552 repo_model = RepoModel()
552 553 user = User.get_first_super_admin()
553 554 added = []
554 555
555 556 # creation defaults
556 557 defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
557 558 enable_statistics = defs.get('repo_enable_statistics')
558 559 enable_locking = defs.get('repo_enable_locking')
559 560 enable_downloads = defs.get('repo_enable_downloads')
560 561 private = defs.get('repo_private')
561 562
562 563 for name, repo in list(initial_repo_list.items()):
563 564 group = map_groups(name)
564 565 str_name = safe_str(name)
565 566 db_repo = repo_model.get_by_repo_name(str_name)
566 567
567 568 # found repo that is on filesystem not in RhodeCode database
568 569 if not db_repo:
569 570 log.info('repository `%s` not found in the database, creating now', name)
570 571 added.append(name)
571 572 desc = (repo.description
572 573 if repo.description != 'unknown'
573 574 else '%s repository' % name)
574 575
575 576 db_repo = repo_model._create_repo(
576 577 repo_name=name,
577 578 repo_type=repo.alias,
578 579 description=desc,
579 580 repo_group=getattr(group, 'group_id', None),
580 581 owner=user,
581 582 enable_locking=enable_locking,
582 583 enable_downloads=enable_downloads,
583 584 enable_statistics=enable_statistics,
584 585 private=private,
585 586 state=Repository.STATE_CREATED
586 587 )
587 588 sa.commit()
588 589 # we added that repo just now, and make sure we updated server info
589 590 if db_repo.repo_type == 'git':
590 591 git_repo = db_repo.scm_instance()
591 592 # update repository server-info
592 593 log.debug('Running update server info')
593 594 git_repo._update_server_info(force=True)
594 595
595 596 db_repo.update_commit_cache(recursive=False)
596 597
597 598 config = db_repo._config
598 599 config.set('extensions', 'largefiles', '')
599 600 repo = db_repo.scm_instance(config=config)
600 601 repo.install_hooks(force=force_hooks_rebuild)
601 602
602 603 removed = []
603 604 if remove_obsolete:
604 605 # remove from database those repositories that are not in the filesystem
605 606 for repo in sa.query(Repository).all():
606 607 if repo.repo_name not in list(initial_repo_list.keys()):
607 608 log.debug("Removing non-existing repository found in db `%s`",
608 609 repo.repo_name)
609 610 try:
610 611 RepoModel(sa).delete(repo, forks='detach', fs_remove=False)
611 612 sa.commit()
612 613 removed.append(repo.repo_name)
613 614 except Exception:
614 615 # don't hold further removals on error
615 616 log.error(traceback.format_exc())
616 617 sa.rollback()
617 618
618 619 def splitter(full_repo_name):
619 620 _parts = full_repo_name.rsplit(RepoGroup.url_sep(), 1)
620 621 gr_name = None
621 622 if len(_parts) == 2:
622 623 gr_name = _parts[0]
623 624 return gr_name
624 625
625 626 initial_repo_group_list = [splitter(x) for x in
626 627 list(initial_repo_list.keys()) if splitter(x)]
627 628
628 629 # remove from database those repository groups that are not in the
629 630 # filesystem due to parent child relationships we need to delete them
630 631 # in a specific order of most nested first
631 632 all_groups = [x.group_name for x in sa.query(RepoGroup).all()]
632 633 def nested_sort(gr):
633 634 return len(gr.split('/'))
634 635 for group_name in sorted(all_groups, key=nested_sort, reverse=True):
635 636 if group_name not in initial_repo_group_list:
636 637 repo_group = RepoGroup.get_by_group_name(group_name)
637 638 if (repo_group.children.all() or
638 639 not RepoGroupModel().check_exist_filesystem(
639 640 group_name=group_name, exc_on_failure=False)):
640 641 continue
641 642
642 643 log.info(
643 644 'Removing non-existing repository group found in db `%s`',
644 645 group_name)
645 646 try:
646 647 RepoGroupModel(sa).delete(group_name, fs_remove=False)
647 648 sa.commit()
648 649 removed.append(group_name)
649 650 except Exception:
650 651 # don't hold further removals on error
651 652 log.exception(
652 653 'Unable to remove repository group `%s`',
653 654 group_name)
654 655 sa.rollback()
655 656 raise
656 657
657 658 return added, removed
658 659
659 660
660 661 def load_rcextensions(root_path):
661 662 import rhodecode
662 663 from rhodecode.config import conf
663 664
664 665 path = os.path.join(root_path)
665 666 sys.path.append(path)
666 667
667 668 try:
668 669 rcextensions = __import__('rcextensions')
669 670 except ImportError:
670 671 if os.path.isdir(os.path.join(path, 'rcextensions')):
671 672 log.warning('Unable to load rcextensions from %s', path)
672 673 rcextensions = None
673 674
674 675 if rcextensions:
675 676 log.info('Loaded rcextensions from %s...', rcextensions)
676 677 rhodecode.EXTENSIONS = rcextensions
677 678
678 679 # Additional mappings that are not present in the pygments lexers
679 680 conf.LANGUAGES_EXTENSIONS_MAP.update(
680 681 getattr(rhodecode.EXTENSIONS, 'EXTRA_MAPPINGS', {}))
681 682
682 683
683 684 def get_custom_lexer(extension):
684 685 """
685 686 returns a custom lexer if it is defined in rcextensions module, or None
686 687 if there's no custom lexer defined
687 688 """
688 689 import rhodecode
689 690 from pygments import lexers
690 691
691 692 # custom override made by RhodeCode
692 693 if extension in ['mako']:
693 694 return lexers.get_lexer_by_name('html+mako')
694 695
695 696 # check if we didn't define this extension as other lexer
696 697 extensions = rhodecode.EXTENSIONS and getattr(rhodecode.EXTENSIONS, 'EXTRA_LEXERS', None)
697 698 if extensions and extension in rhodecode.EXTENSIONS.EXTRA_LEXERS:
698 699 _lexer_name = rhodecode.EXTENSIONS.EXTRA_LEXERS[extension]
699 700 return lexers.get_lexer_by_name(_lexer_name)
700 701
701 702
702 703 #==============================================================================
703 704 # TEST FUNCTIONS AND CREATORS
704 705 #==============================================================================
705 706 def create_test_index(repo_location, config):
706 707 """
707 708 Makes default test index.
708 709 """
709 710 try:
710 711 import rc_testdata
711 712 except ImportError:
712 713 raise ImportError('Failed to import rc_testdata, '
713 714 'please make sure this package is installed from requirements_test.txt')
714 715 rc_testdata.extract_search_index(
715 716 'vcs_search_index', os.path.dirname(config['search.location']))
716 717
717 718
718 719 def create_test_directory(test_path):
719 720 """
720 721 Create test directory if it doesn't exist.
721 722 """
722 723 if not os.path.isdir(test_path):
723 724 log.debug('Creating testdir %s', test_path)
724 725 os.makedirs(test_path)
725 726
726 727
727 728 def create_test_database(test_path, config):
728 729 """
729 730 Makes a fresh database.
730 731 """
731 732 from rhodecode.lib.db_manage import DbManage
732 733 from rhodecode.lib.utils2 import get_encryption_key
733 734
734 735 # PART ONE create db
735 736 dbconf = config['sqlalchemy.db1.url']
736 737 enc_key = get_encryption_key(config)
737 738
738 739 log.debug('making test db %s', dbconf)
739 740
740 741 dbmanage = DbManage(log_sql=False, dbconf=dbconf, root=config['here'],
741 742 tests=True, cli_args={'force_ask': True}, enc_key=enc_key)
742 743 dbmanage.create_tables(override=True)
743 744 dbmanage.set_db_version()
744 745 # for tests dynamically set new root paths based on generated content
745 746 dbmanage.create_settings(dbmanage.config_prompt(test_path))
746 747 dbmanage.create_default_user()
747 748 dbmanage.create_test_admin_and_users()
748 749 dbmanage.create_permissions()
749 750 dbmanage.populate_default_permissions()
750 751 Session().commit()
751 752
752 753
753 754 def create_test_repositories(test_path, config):
754 755 """
755 756 Creates test repositories in the temporary directory. Repositories are
756 757 extracted from archives within the rc_testdata package.
757 758 """
758 759 import rc_testdata
759 760 from rhodecode.tests import HG_REPO, GIT_REPO, SVN_REPO
760 761
761 762 log.debug('making test vcs repositories')
762 763
763 764 idx_path = config['search.location']
764 765 data_path = config['cache_dir']
765 766
766 767 # clean index and data
767 768 if idx_path and os.path.exists(idx_path):
768 769 log.debug('remove %s', idx_path)
769 770 shutil.rmtree(idx_path)
770 771
771 772 if data_path and os.path.exists(data_path):
772 773 log.debug('remove %s', data_path)
773 774 shutil.rmtree(data_path)
774 775
775 776 rc_testdata.extract_hg_dump('vcs_test_hg', jn(test_path, HG_REPO))
776 777 rc_testdata.extract_git_dump('vcs_test_git', jn(test_path, GIT_REPO))
777 778
778 779 # Note: Subversion is in the process of being integrated with the system,
779 780 # until we have a properly packed version of the test svn repository, this
780 781 # tries to copy over the repo from a package "rc_testdata"
781 782 svn_repo_path = rc_testdata.get_svn_repo_archive()
782 783 with tarfile.open(svn_repo_path) as tar:
783 784 tar.extractall(jn(test_path, SVN_REPO))
784 785
785 786
786 787 def password_changed(auth_user, session):
787 788 # Never report password change in case of default user or anonymous user.
788 789 if auth_user.username == User.DEFAULT_USER or auth_user.user_id is None:
789 790 return False
790 791
791 792 password_hash = md5(safe_bytes(auth_user.password)) if auth_user.password else None
792 793 rhodecode_user = session.get('rhodecode_user', {})
793 794 session_password_hash = rhodecode_user.get('password', '')
794 795 return password_hash != session_password_hash
795 796
796 797
797 798 def read_opensource_licenses():
798 799 global _license_cache
799 800
800 801 if not _license_cache:
801 802 licenses = pkg_resources.resource_string(
802 803 'rhodecode', 'config/licenses.json')
803 804 _license_cache = json.loads(licenses)
804 805
805 806 return _license_cache
806 807
807 808
808 809 def generate_platform_uuid():
809 810 """
810 811 Generates platform UUID based on it's name
811 812 """
812 813 import platform
813 814
814 815 try:
815 816 uuid_list = [platform.platform()]
816 817 return sha256_safe(':'.join(uuid_list))
817 818 except Exception as e:
818 819 log.error('Failed to generate host uuid: %s', e)
819 820 return 'UNDEFINED'
820 821
821 822
822 823 def send_test_email(recipients, email_body='TEST EMAIL'):
823 824 """
824 825 Simple code for generating test emails.
825 826 Usage::
826 827
827 828 from rhodecode.lib import utils
828 829 utils.send_test_email()
829 830 """
830 831 from rhodecode.lib.celerylib import tasks, run_task
831 832
832 833 email_body = email_body_plaintext = email_body
833 834 subject = f'SUBJECT FROM: {socket.gethostname()}'
834 835 tasks.send_email(recipients, subject, email_body_plaintext, email_body)
General Comments 0
You need to be logged in to leave comments. Login now