# HG changeset patch # User RhodeCode Admin # Date 2024-11-13 12:39:21 # Node ID c9e499e7c60bbd2c7af1a30ec3041e048c1d1eec # Parent bdbdb63f8abb0192131d5207680c0f5534ba797b feat(remap and rescan): added more relient remap and removal option, and also split the logic to either add or cleanup diff --git a/rhodecode/api/tests/test_cleanup_repos.py b/rhodecode/api/tests/test_cleanup_repos.py new file mode 100644 --- /dev/null +++ b/rhodecode/api/tests/test_cleanup_repos.py @@ -0,0 +1,44 @@ +# Copyright (C) 2010-2024 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +import mock +import pytest + +from rhodecode.model.scm import ScmModel +from rhodecode.api.tests.utils import ( + build_data, api_call, assert_ok, assert_error, crash) + + +@pytest.mark.usefixtures("testuser_api", "app") +class TestCleanupRepos(object): + def test_api_cleanup_repos(self): + id_, params = build_data(self.apikey, 'cleanup_repos') + response = api_call(self.app, params) + + expected = {'removed': [], 'errors': []} + assert_ok(id_, expected, given=response.body) + + def test_api_cleanup_repos_error(self): + + id_, params = build_data(self.apikey, 'cleanup_repos', ) + + with mock.patch('rhodecode.lib.utils.repo2db_cleanup', side_effect=crash): + response = api_call(self.app, params) + + expected = 'Error occurred during repo storage cleanup action' + assert_error(id_, expected, given=response.body) diff --git a/rhodecode/api/tests/test_rescan_repos.py b/rhodecode/api/tests/test_rescan_repos.py --- a/rhodecode/api/tests/test_rescan_repos.py +++ b/rhodecode/api/tests/test_rescan_repos.py @@ -19,7 +19,6 @@ import mock import pytest -from rhodecode.model.scm import ScmModel from rhodecode.api.tests.utils import ( build_data, api_call, assert_ok, assert_error, crash) @@ -30,13 +29,14 @@ class TestRescanRepos(object): id_, params = build_data(self.apikey, 'rescan_repos') response = api_call(self.app, params) - expected = {'added': [], 'removed': []} + expected = {'added': [], 'errors': []} assert_ok(id_, expected, given=response.body) - @mock.patch.object(ScmModel, 'repo_scan', crash) - def test_api_rescann_error(self): + def test_api_rescan_repos_error(self): id_, params = build_data(self.apikey, 'rescan_repos', ) - response = api_call(self.app, params) + + with mock.patch('rhodecode.lib.utils.repo2db_mapper', side_effect=crash): + response = api_call(self.app, params) expected = 'Error occurred during rescan repositories action' assert_error(id_, expected, given=response.body) diff --git a/rhodecode/api/views/server_api.py b/rhodecode/api/views/server_api.py --- a/rhodecode/api/views/server_api.py +++ b/rhodecode/api/views/server_api.py @@ -18,14 +18,13 @@ import logging import itertools -import base64 from rhodecode.api import ( jsonrpc_method, JSONRPCError, JSONRPCForbidden, find_methods) from rhodecode.api.utils import ( Optional, OAttr, has_superadmin_permission, get_user_or_error) -from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_repo_store_path +from rhodecode.lib.utils import get_rhodecode_repo_store_path from rhodecode.lib import system_info from rhodecode.lib import user_sessions from rhodecode.lib import exc_tracking @@ -33,9 +32,7 @@ from rhodecode.lib.ext_json import json from rhodecode.lib.utils2 import safe_int from rhodecode.model.db import UserIpMap from rhodecode.model.scm import ScmModel -from rhodecode.apps.file_store import utils as store_utils -from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \ - FileOverSizeException + log = logging.getLogger(__name__) @@ -158,13 +155,10 @@ def get_ip(request, apiuser, userid=Opti @jsonrpc_method() -def rescan_repos(request, apiuser, remove_obsolete=Optional(False)): +def rescan_repos(request, apiuser): """ Triggers a rescan of the specified repositories. - - * If the ``remove_obsolete`` option is set, it also deletes repositories - that are found in the database but not on the file system, so called - "clean zombies". + It returns list of added repositories, and errors during scan. This command can only be run using an |authtoken| with admin rights to the specified repository. @@ -173,9 +167,6 @@ def rescan_repos(request, apiuser, remov :param apiuser: This is filled automatically from the |authtoken|. :type apiuser: AuthUser - :param remove_obsolete: Deletes repositories from the database that - are not found on the filesystem. - :type remove_obsolete: Optional(``True`` | ``False``) Example output: @@ -184,7 +175,7 @@ def rescan_repos(request, apiuser, remov id : result : { 'added': [,...] - 'removed': [,...] + 'errors': [,...] } error : null @@ -199,28 +190,24 @@ def rescan_repos(request, apiuser, remov } """ + from rhodecode.lib.utils import repo2db_mapper # re-import for testing patches + if not has_superadmin_permission(apiuser): raise JSONRPCForbidden() try: - rm_obsolete = Optional.extract(remove_obsolete) - added, removed = repo2db_mapper(ScmModel().repo_scan(), - remove_obsolete=rm_obsolete, force_hooks_rebuild=True) - return {'added': added, 'removed': removed} + added, errors = repo2db_mapper(ScmModel().repo_scan(), force_hooks_rebuild=True) + return {'added': added, 'errors': errors} except Exception: - log.exception('Failed to run repo rescann') + log.exception('Failed to run repo rescan') raise JSONRPCError( 'Error occurred during rescan repositories action' ) @jsonrpc_method() -def cleanup_repos(request, apiuser, remove_obsolete=Optional(False)): +def cleanup_repos(request, apiuser): """ - Triggers a rescan of the specified repositories. - - * If the ``remove_obsolete`` option is set, it also deletes repositories - that are found in the database but not on the file system, so called - "clean zombies". + Triggers a cleanup of non-existing repositories or repository groups in filesystem. This command can only be run using an |authtoken| with admin rights to the specified repository. @@ -229,9 +216,6 @@ def cleanup_repos(request, apiuser, remo :param apiuser: This is filled automatically from the |authtoken|. :type apiuser: AuthUser - :param remove_obsolete: Deletes repositories from the database that - are not found on the filesystem. - :type remove_obsolete: Optional(``True`` | ``False``) Example output: @@ -239,8 +223,8 @@ def cleanup_repos(request, apiuser, remo id : result : { - 'added': [,...] - 'removed': [,...] + 'removed': [,...] + 'errors': [,...] } error : null @@ -251,22 +235,22 @@ def cleanup_repos(request, apiuser, remo id : result : null error : { - 'Error occurred during rescan repositories action' + 'Error occurred during repo storage cleanup action' } """ + from rhodecode.lib.utils import repo2db_cleanup # re-import for testing patches + if not has_superadmin_permission(apiuser): raise JSONRPCForbidden() try: - rm_obsolete = Optional.extract(remove_obsolete) - added, removed = repo2db_mapper(ScmModel().repo_scan(), - remove_obsolete=rm_obsolete, force_hooks_rebuild=True) - return {'added': added, 'removed': removed} + removed, errors = repo2db_cleanup() + return {'removed': removed, 'errors': errors} except Exception: - log.exception('Failed to run repo rescann') + log.exception('Failed to run repo storage cleanup') raise JSONRPCError( - 'Error occurred during rescan repositories action' + 'Error occurred during repo storage cleanup action' ) diff --git a/rhodecode/apps/admin/__init__.py b/rhodecode/apps/admin/__init__.py --- a/rhodecode/apps/admin/__init__.py +++ b/rhodecode/apps/admin/__init__.py @@ -361,12 +361,21 @@ def admin_routes(config): renderer='rhodecode:templates/admin/settings/settings.mako') config.add_route( - name='admin_settings_mapping_update', - pattern='/settings/mapping/update') + name='admin_settings_mapping_create', + pattern='/settings/mapping/create') config.add_view( AdminSettingsView, - attr='settings_mapping_update', - route_name='admin_settings_mapping_update', request_method='POST', + attr='settings_mapping_create', + route_name='admin_settings_mapping_create', request_method='POST', + renderer='rhodecode:templates/admin/settings/settings.mako') + + config.add_route( + name='admin_settings_mapping_cleanup', + pattern='/settings/mapping/cleanup') + config.add_view( + AdminSettingsView, + attr='settings_mapping_cleanup', + route_name='admin_settings_mapping_cleanup', request_method='POST', renderer='rhodecode:templates/admin/settings/settings.mako') config.add_route( diff --git a/rhodecode/apps/admin/tests/test_admin_repos.py b/rhodecode/apps/admin/tests/test_admin_repos.py --- a/rhodecode/apps/admin/tests/test_admin_repos.py +++ b/rhodecode/apps/admin/tests/test_admin_repos.py @@ -110,9 +110,11 @@ class TestAdminRepos(object): repo_type=backend.alias, repo_description=description, csrf_token=csrf_token)) - - self.assert_repository_is_created_correctly( - repo_name, description, backend) + try: + self.assert_repository_is_created_correctly(repo_name, description, backend) + finally: + RepoModel().delete(numeric_repo) + Session().commit() @pytest.mark.parametrize("suffix", ['', '_ąćę'], ids=['', 'non-ascii']) def test_create_in_group( diff --git a/rhodecode/apps/admin/views/settings.py b/rhodecode/apps/admin/views/settings.py --- a/rhodecode/apps/admin/views/settings.py +++ b/rhodecode/apps/admin/views/settings.py @@ -38,7 +38,7 @@ from rhodecode.lib.auth import ( LoginRequired, HasPermissionAllDecorator, CSRFRequired) from rhodecode.lib.celerylib import tasks, run_task from rhodecode.lib.str_utils import safe_str -from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_repo_store_path +from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_repo_store_path, repo2db_cleanup from rhodecode.lib.utils2 import str2bool, AttributeDict from rhodecode.lib.index import searcher_from_config @@ -233,13 +233,12 @@ class AdminSettingsView(BaseAppView): @LoginRequired() @HasPermissionAllDecorator('hg.admin') @CSRFRequired() - def settings_mapping_update(self): + def settings_mapping_create(self): _ = self.request.translate c = self.load_default_context() c.active = 'mapping' - rm_obsolete = self.request.POST.get('destroy', False) invalidate_cache = self.request.POST.get('invalidate', False) - log.debug('rescanning repo location with destroy obsolete=%s', rm_obsolete) + log.debug('rescanning repo location') if invalidate_cache: log.debug('invalidating all repositories cache') @@ -247,16 +246,34 @@ class AdminSettingsView(BaseAppView): ScmModel().mark_for_invalidation(repo.repo_name, delete=True) filesystem_repos = ScmModel().repo_scan() - added, removed = repo2db_mapper(filesystem_repos, rm_obsolete, force_hooks_rebuild=True) + added, errors = repo2db_mapper(filesystem_repos, force_hooks_rebuild=True) PermissionModel().trigger_permission_flush() def _repr(rm_repo): return ', '.join(map(safe_str, rm_repo)) or '-' - h.flash(_('Repositories successfully ' - 'rescanned added: %s ; removed: %s') % - (_repr(added), _repr(removed)), - category='success') + if errors: + h.flash(_('Errors during scan: {}').format(_repr(errors), ), category='error') + + h.flash(_('Repositories successfully scanned: Added: {}').format(_repr(added)), category='success') + raise HTTPFound(h.route_path('admin_settings_mapping')) + + @LoginRequired() + @HasPermissionAllDecorator('hg.admin') + @CSRFRequired() + def settings_mapping_cleanup(self): + _ = self.request.translate + c = self.load_default_context() + c.active = 'mapping' + log.debug('rescanning repo location') + + removed, errors = repo2db_cleanup() + PermissionModel().trigger_permission_flush() + + def _repr(rm_repo): + return ', '.join(map(safe_str, rm_repo)) or '-' + + h.flash(_('Repositories successfully scanned: Errors: {}, Added: {}').format(errors, _repr(removed)), category='success') raise HTTPFound(h.route_path('admin_settings_mapping')) @LoginRequired() diff --git a/rhodecode/lib/utils.py b/rhodecode/lib/utils.py --- a/rhodecode/lib/utils.py +++ b/rhodecode/lib/utils.py @@ -582,23 +582,19 @@ def map_groups(path): return group -def repo2db_mapper(initial_repo_list, remove_obsolete=False, force_hooks_rebuild=False): +def repo2db_mapper(initial_repo_list, force_hooks_rebuild=False): """ - maps all repos given in initial_repo_list, non existing repositories - are created, if remove_obsolete is True it also checks for db entries - that are not in initial_repo_list and removes them. - - :param initial_repo_list: list of repositories found by scanning methods - :param remove_obsolete: check for obsolete entries in database + maps all repos given in initial_repo_list, non-existing repositories + are created """ from rhodecode.model.repo import RepoModel - from rhodecode.model.repo_group import RepoGroupModel from rhodecode.model.settings import SettingsModel sa = meta.Session() repo_model = RepoModel() user = User.get_first_super_admin() added = [] + errors = [] # creation defaults defs = SettingsModel().get_default_repo_settings(strip_prefix=True) @@ -616,9 +612,7 @@ def repo2db_mapper(initial_repo_list, re if not db_repo: log.info('repository `%s` not found in the database, creating now', name) added.append(name) - desc = (repo.description - if repo.description != 'unknown' - else '%s repository' % name) + desc = repo.description if repo.description != 'unknown' else f'{name} repository' db_repo = repo_model._create_repo( repo_name=name, @@ -633,76 +627,115 @@ def repo2db_mapper(initial_repo_list, re state=Repository.STATE_CREATED ) sa.commit() + + try: + config = db_repo._config + config.set('extensions', 'largefiles', '') + scm_repo = db_repo.scm_instance(config=config) + except Exception: + log.error(traceback.format_exc()) + errors.append(f'getting vcs instance for {name} failed') + continue + + try: + db_repo.update_commit_cache(recursive=False) + except Exception: + log.error(traceback.format_exc()) + errors.append(f'update_commit_cache for {name} failed') + continue + + try: + scm_repo.install_hooks(force=force_hooks_rebuild) + except Exception: + log.error(traceback.format_exc()) + errors.append(f'install_hooks for {name} failed') + continue + + try: # we added that repo just now, and make sure we updated server info if db_repo.repo_type == 'git': - git_repo = db_repo.scm_instance() # update repository server-info log.debug('Running update server info') - git_repo._update_server_info(force=True) - - db_repo.update_commit_cache(recursive=False) + scm_repo._update_server_info(force=True) + except Exception: + log.error(traceback.format_exc()) + errors.append(f'update_server_info for {name} failed') + continue - config = db_repo._config - config.set('extensions', 'largefiles', '') - repo = db_repo.scm_instance(config=config) - repo.install_hooks(force=force_hooks_rebuild) + return added, errors +def repo2db_cleanup(skip_repos=None, skip_groups=None): + from rhodecode.model.repo import RepoModel + from rhodecode.model.repo_group import RepoGroupModel + + sa = meta.Session() removed = [] - if remove_obsolete: - # remove from database those repositories that are not in the filesystem - for repo in sa.query(Repository).all(): - if repo.repo_name not in list(initial_repo_list.keys()): - log.debug("Removing non-existing repository found in db `%s`", - repo.repo_name) - try: - RepoModel(sa).delete(repo, forks='detach', fs_remove=False) - sa.commit() - removed.append(repo.repo_name) - except Exception: - # don't hold further removals on error - log.error(traceback.format_exc()) - sa.rollback() + errors = [] + + + all_repos = Repository.execute( + Repository.select(Repository)\ + .order_by(Repository.repo_name) + ).scalars() - def splitter(full_repo_name): - _parts = full_repo_name.rsplit(RepoGroup.url_sep(), 1) - gr_name = None - if len(_parts) == 2: - gr_name = _parts[0] - return gr_name + # remove from database those repositories that are not in the filesystem + for db_repo in all_repos: + db_repo_name = db_repo.repo_name + if skip_repos and db_repo_name in skip_repos: + log.debug('Skipping repo `%s`', db_repo_name) + continue + try: + instance = db_repo.scm_instance() + except Exception: + instance = None - initial_repo_group_list = [splitter(x) for x in - list(initial_repo_list.keys()) if splitter(x)] + if not instance: + log.debug("Removing non-existing repository found in db `%s`", db_repo_name) + try: + RepoModel(sa).delete(db_repo, forks='detach', fs_remove=False, call_events=False) + sa.commit() + removed.append(db_repo_name) + except Exception: + # don't hold further removals on error + log.error(traceback.format_exc()) + sa.rollback() + errors.append(db_repo_name) - # remove from database those repository groups that are not in the - # filesystem due to parent child relationships we need to delete them - # in a specific order of most nested first - all_groups = [x.group_name for x in sa.query(RepoGroup).all()] - def nested_sort(gr): - return len(gr.split('/')) - for group_name in sorted(all_groups, key=nested_sort, reverse=True): - if group_name not in initial_repo_group_list: - repo_group = RepoGroup.get_by_group_name(group_name) - if (repo_group.children.all() or - not RepoGroupModel().check_exist_filesystem( - group_name=group_name, exc_on_failure=False)): - continue + # remove from database those repository groups that are not in the + # filesystem due to parent child relationships we need to delete them + # in a specific order of most nested first + all_groups = RepoGroup.execute( + RepoGroup.select(RepoGroup.group_name)\ + .order_by(RepoGroup.group_name) + ).scalars().all() + + def nested_sort(gr): + return len(gr.split('/')) - log.info( - 'Removing non-existing repository group found in db `%s`', - group_name) - try: - RepoGroupModel(sa).delete(group_name, fs_remove=False) - sa.commit() - removed.append(group_name) - except Exception: - # don't hold further removals on error - log.exception( - 'Unable to remove repository group `%s`', - group_name) - sa.rollback() - raise + for group_name in sorted(all_groups, key=nested_sort, reverse=True): + if skip_groups and group_name in skip_groups: + log.debug('Skipping repo group `%s`', group_name) + continue + + repo_group = RepoGroup.get_by_group_name(group_name) + + if repo_group.children.all() or not RepoGroupModel().check_exist_filesystem(group_name=group_name, exc_on_failure=False): + continue + + log.info('Removing non-existing repository group found in db `%s`', group_name) - return added, removed + try: + RepoGroupModel(sa).delete(group_name, fs_remove=False, call_events=False) + sa.commit() + removed.append(group_name) + except Exception: + # don't hold further removals on error + log.exception('Unable to remove repository group `%s`',group_name) + sa.rollback() + errors.append(group_name) + + return removed, errors + def deep_reload_package(package_name): """ @@ -829,7 +862,7 @@ def create_test_repositories(test_path, raise ImportError('Failed to import rc_testdata, ' 'please make sure this package is installed from requirements_test.txt') - from rhodecode.tests import HG_REPO, GIT_REPO, SVN_REPO + from rhodecode.bootstrap import HG_REPO, GIT_REPO, SVN_REPO log.debug('making test vcs repositories at %s', test_path) diff --git a/rhodecode/lib/vcs/backends/__init__.py b/rhodecode/lib/vcs/backends/__init__.py --- a/rhodecode/lib/vcs/backends/__init__.py +++ b/rhodecode/lib/vcs/backends/__init__.py @@ -58,9 +58,10 @@ def get_vcs_instance(repo_path, *args, * raise VCSError(f"Given path {repo_path} is not a directory") except VCSError: log.exception( - 'Perhaps this repository is in db and not in ' - 'filesystem run rescan repositories with ' - '"destroy old data" option from admin panel') + 'Perhaps this repository is in db and not in filesystem.' + 'Run cleanup filesystem option from admin settings under Remap and rescan' + ) + return None return backend(repo_path=repo_path, *args, **kwargs) diff --git a/rhodecode/model/repo.py b/rhodecode/model/repo.py --- a/rhodecode/model/repo.py +++ b/rhodecode/model/repo.py @@ -745,7 +745,7 @@ class RepoModel(BaseModel): log.error(traceback.format_exc()) raise - def delete(self, repo, forks=None, pull_requests=None, artifacts=None, fs_remove=True, cur_user=None): + def delete(self, repo, forks=None, pull_requests=None, artifacts=None, fs_remove=True, cur_user=None, call_events=True): """ Delete given repository, forks parameter defines what do do with attached forks. Throws AttachedForksError if deleted repo has attached @@ -760,47 +760,54 @@ class RepoModel(BaseModel): if not cur_user: cur_user = getattr(get_current_rhodecode_user(), 'username', None) repo = self._get_repo(repo) - if repo: - if forks == 'detach': - for r in repo.forks: - r.fork = None - self.sa.add(r) - elif forks == 'delete': - for r in repo.forks: - self.delete(r, forks='delete') - elif [f for f in repo.forks]: - raise AttachedForksError() + if not repo: + return False - # check for pull requests - pr_sources = repo.pull_requests_source - pr_targets = repo.pull_requests_target - if pull_requests != 'delete' and (pr_sources or pr_targets): - raise AttachedPullRequestsError() + if forks == 'detach': + for r in repo.forks: + r.fork = None + self.sa.add(r) + elif forks == 'delete': + for r in repo.forks: + self.delete(r, forks='delete') + elif [f for f in repo.forks]: + raise AttachedForksError() + + # check for pull requests + pr_sources = repo.pull_requests_source + pr_targets = repo.pull_requests_target + if pull_requests != 'delete' and (pr_sources or pr_targets): + raise AttachedPullRequestsError() - artifacts_objs = repo.artifacts - if artifacts == 'delete': - for a in artifacts_objs: - self.sa.delete(a) - elif [a for a in artifacts_objs]: - raise AttachedArtifactsError() + artifacts_objs = repo.artifacts + if artifacts == 'delete': + for a in artifacts_objs: + self.sa.delete(a) + elif [a for a in artifacts_objs]: + raise AttachedArtifactsError() - old_repo_dict = repo.get_dict() + old_repo_dict = repo.get_dict() + if call_events: events.trigger(events.RepoPreDeleteEvent(repo)) - try: - self.sa.delete(repo) - if fs_remove: - self._delete_filesystem_repo(repo) - else: - log.debug('skipping removal from filesystem') - old_repo_dict.update({ - 'deleted_by': cur_user, - 'deleted_on': time.time(), - }) + + try: + self.sa.delete(repo) + if fs_remove: + self._delete_filesystem_repo(repo) + else: + log.debug('skipping removal from filesystem') + old_repo_dict.update({ + 'deleted_by': cur_user, + 'deleted_on': time.time(), + }) + if call_events: hooks_base.delete_repository(**old_repo_dict) events.trigger(events.RepoDeleteEvent(repo)) - except Exception: - log.error(traceback.format_exc()) - raise + except Exception: + log.error(traceback.format_exc()) + raise + + return True def grant_user_permission(self, repo, user, perm): """ diff --git a/rhodecode/model/repo_group.py b/rhodecode/model/repo_group.py --- a/rhodecode/model/repo_group.py +++ b/rhodecode/model/repo_group.py @@ -156,7 +156,7 @@ class RepoGroupModel(BaseModel): def check_exist_filesystem(self, group_name, exc_on_failure=True): create_path = os.path.join(self.repos_path, group_name) - log.debug('creating new group in %s', create_path) + log.debug('checking FS presence for repo group in %s', create_path) if os.path.isdir(create_path): if exc_on_failure: @@ -573,10 +573,11 @@ class RepoGroupModel(BaseModel): log.error(traceback.format_exc()) raise - def delete(self, repo_group, force_delete=False, fs_remove=True): + def delete(self, repo_group, force_delete=False, fs_remove=True, call_events=True): repo_group = self._get_repo_group(repo_group) if not repo_group: return False + repo_group_name = repo_group.group_name try: self.sa.delete(repo_group) if fs_remove: @@ -585,13 +586,15 @@ class RepoGroupModel(BaseModel): log.debug('skipping removal from filesystem') # Trigger delete event. - events.trigger(events.RepoGroupDeleteEvent(repo_group)) - return True + if call_events: + events.trigger(events.RepoGroupDeleteEvent(repo_group)) except Exception: - log.error('Error removing repo_group %s', repo_group) + log.error('Error removing repo_group %s', repo_group_name) raise + return True + def grant_user_permission(self, repo_group, user, perm): """ Grant permission for user on given repository group, or update diff --git a/rhodecode/subscribers.py b/rhodecode/subscribers.py --- a/rhodecode/subscribers.py +++ b/rhodecode/subscribers.py @@ -121,7 +121,7 @@ def scan_repositories_if_enabled(event): from rhodecode.lib.utils import repo2db_mapper scm = ScmModel() repositories = scm.repo_scan(scm.repos_path) - repo2db_mapper(repositories, remove_obsolete=False) + repo2db_mapper(repositories) def write_metadata_if_needed(event): diff --git a/rhodecode/templates/admin/settings/settings_mapping.mako b/rhodecode/templates/admin/settings/settings_mapping.mako --- a/rhodecode/templates/admin/settings/settings_mapping.mako +++ b/rhodecode/templates/admin/settings/settings_mapping.mako @@ -1,33 +1,45 @@ -${h.secure_form(h.route_path('admin_settings_mapping_update'), request=request)} +
-

${_('Import New Groups or Repositories')}

+

${_('Import new repository groups and repositories')}

- + ${h.secure_form(h.route_path('admin_settings_mapping_create'), request=request)}

- ${_('This function will scann all data under the current storage path location at')} ${c.storage_path} + ${_('This function will scan all data under the current storage path location at')} ${c.storage_path}
+ ${_('Each folder will be imported as a new repository group, and each repository found will be also imported to root level or corresponding repository group')}

- ${h.checkbox('destroy',True)} - -
- ${_('In case a repository or a group was deleted from the filesystem and it still exists in the database, check this option to remove obsolete data from the database.')} - -
${h.checkbox('invalidate',True)}
${_('Each cache data for repositories will be cleaned with this option selected. Use this to reload data and clear cache keys.')}
- ${h.submit('rescan',_('Rescan Filesystem'),class_="btn")} + ${h.submit('rescan',_('Scan filesystem'),class_="btn")}
- + ${h.end_form()}
-${h.end_form()} +
+
+

${_('Cleanup removed Repository Groups or Repositories')}

+
+
+ ${h.secure_form(h.route_path('admin_settings_mapping_cleanup'), request=request)} +

+ ${_('This function will scan all data under the current storage path location at')} ${c.storage_path} + ${_('Then it will remove all repository groups and repositories that are no longer present in the filesystem.')} +

+ +
+ ${h.submit('rescan',_('Cleanup filesystem'),class_="btn btn-danger")} +
+ ${h.end_form()} +
+
+ diff --git a/rhodecode/tests/fixtures/fixture_utils.py b/rhodecode/tests/fixtures/fixture_utils.py --- a/rhodecode/tests/fixtures/fixture_utils.py +++ b/rhodecode/tests/fixtures/fixture_utils.py @@ -1558,6 +1558,7 @@ def stub_integration_settings(): @pytest.fixture() def repo_integration_stub(request, repo_stub, StubIntegrationType, stub_integration_settings): + repo_id = repo_stub.repo_id integration = IntegrationModel().create( StubIntegrationType, settings=stub_integration_settings, @@ -1571,6 +1572,7 @@ def repo_integration_stub(request, repo_ @request.addfinalizer def cleanup(): IntegrationModel().delete(integration) + RepoModel().delete(repo_id) return integration diff --git a/rhodecode/tests/integrations/test_integration.py b/rhodecode/tests/integrations/test_integration.py --- a/rhodecode/tests/integrations/test_integration.py +++ b/rhodecode/tests/integrations/test_integration.py @@ -20,23 +20,21 @@ import time import pytest from rhodecode import events +from rhodecode.model.repo import RepoModel from rhodecode.tests.fixtures.rc_fixture import Fixture from rhodecode.model.db import Session, Integration from rhodecode.model.integration import IntegrationModel class TestDeleteScopesDeletesIntegrations(object): - def test_delete_repo_with_integration_deletes_integration( - self, repo_integration_stub): - - Session().delete(repo_integration_stub.repo) + def test_delete_repo_with_integration_deletes_integration(self, repo_integration_stub): + RepoModel().delete(repo_integration_stub.repo) Session().commit() Session().expire_all() integration = Integration.get(repo_integration_stub.integration_id) assert integration is None - def test_delete_repo_group_with_integration_deletes_integration( - self, repogroup_integration_stub): + def test_delete_repo_group_with_integration_deletes_integration(self, repogroup_integration_stub): Session().delete(repogroup_integration_stub.repo_group) Session().commit() @@ -52,7 +50,7 @@ def counter(): global count val = count count += 1 - return '{}_{}'.format(val, time.time()) + return f'{val}_{time.time()}' @pytest.fixture() diff --git a/rhodecode/tests/lib/test_utils.py b/rhodecode/tests/lib/test_utils.py --- a/rhodecode/tests/lib/test_utils.py +++ b/rhodecode/tests/lib/test_utils.py @@ -18,20 +18,25 @@ import multiprocessing import os +import shutil import mock import py import pytest +import rhodecode from rhodecode.lib import caching_query from rhodecode.lib import utils from rhodecode.lib.str_utils import safe_bytes from rhodecode.model import settings from rhodecode.model import db from rhodecode.model import meta +from rhodecode.model.meta import Session from rhodecode.model.repo import RepoModel from rhodecode.model.repo_group import RepoGroupModel +from rhodecode.model.scm import ScmModel from rhodecode.model.settings import UiSetting, SettingsModel +from rhodecode.tests.fixtures.fixture_pyramid import rhodecode_factory from rhodecode.tests.fixtures.rc_fixture import Fixture from rhodecode_tools.lib.hash_utils import md5_safe from rhodecode.lib.ext_json import json @@ -230,7 +235,7 @@ def platform_encodes_filenames(): return path_with_latin1 != read_path -def test_repo2db_mapper_groups(repo_groups): +def test_repo2db_cleaner_removes_zombie_groups(repo_groups): session = meta.Session() zombie_group, parent_group, child_group = repo_groups zombie_path = os.path.join( @@ -238,10 +243,9 @@ def test_repo2db_mapper_groups(repo_grou os.rmdir(zombie_path) # Avoid removing test repos when calling repo2db_mapper - repo_list = { - repo.repo_name: 'test' for repo in session.query(db.Repository).all() - } - utils.repo2db_mapper(repo_list, remove_obsolete=True) + repo_list = [repo.repo_name for repo in session.query(db.Repository).all()] + + utils.repo2db_cleanup(skip_repos=repo_list) groups_in_db = session.query(db.RepoGroup).all() assert child_group in groups_in_db @@ -249,20 +253,68 @@ def test_repo2db_mapper_groups(repo_grou assert zombie_path not in groups_in_db -def test_repo2db_mapper_enables_largefiles(backend): + +@pytest.mark.backends("hg", "git", "svn") +def test_repo2db_cleaner_removes_zombie_repos(backend): repo = backend.create_repo() - repo_list = {repo.repo_name: 'test'} - with mock.patch('rhodecode.model.db.Repository.scm_instance') as scm_mock: - utils.repo2db_mapper(repo_list, remove_obsolete=False) - _, kwargs = scm_mock.call_args - assert kwargs['config'].get('extensions', 'largefiles') == '' + zombie_path = repo.repo_full_path + shutil.rmtree(zombie_path) + + removed, errors = utils.repo2db_cleanup() + assert len(removed) == 1 + assert not errors -@pytest.mark.backends("git", "svn") +def test_repo2db_mapper_adds_new_repos(request, backend): + repo = backend.create_repo() + cleanup_repos = [] + cleanup_groups = [] + for num in range(5): + copy_repo_name = f'{repo.repo_name}-{num}' + copy_repo_path = f'{repo.repo_full_path}-{num}' + + shutil.copytree(repo.repo_full_path, copy_repo_path) + cleanup_repos.append(copy_repo_name) + + for gr_num in range(5): + gr_name = f'my_gr_{gr_num}' + dest_gr = os.path.join(os.path.dirname(repo.repo_full_path), gr_name) + os.makedirs(dest_gr, exist_ok=True) + + copy_repo_name = f'{gr_name}/{repo.repo_name}-{gr_num}' + copy_repo_path = f'{dest_gr}/{repo.repo_name}-{gr_num}' + + shutil.copytree(repo.repo_full_path, copy_repo_path) + cleanup_repos.append(copy_repo_name) + cleanup_groups.append(gr_name) + + repo_list = ScmModel().repo_scan() + + added, errors = utils.repo2db_mapper(repo_list) + Session().commit() + assert not errors + + assert len(added) == 10 + + @request.addfinalizer + def cleanup(): + for _repo in cleanup_repos: + del_result = RepoModel().delete(_repo, call_events=False) + Session().commit() + assert del_result is True + + for _repo_group in cleanup_groups: + del_result = RepoGroupModel().delete(_repo_group, force_delete=True, call_events=False) + Session().commit() + assert del_result is True + + def test_repo2db_mapper_installs_hooks_for_repos_in_db(backend): repo = backend.create_repo() repo_list = {repo.repo_name: 'test'} - utils.repo2db_mapper(repo_list, remove_obsolete=False) + added, errors = utils.repo2db_mapper(repo_list) + assert not errors + assert repo.scm_instance().get_hooks_info() == {'pre_version': rhodecode.__version__, 'post_version': rhodecode.__version__} @pytest.mark.backends("git", "svn") @@ -271,7 +323,9 @@ def test_repo2db_mapper_installs_hooks_f RepoModel().delete(repo, fs_remove=False) meta.Session().commit() repo_list = {repo.repo_name: repo.scm_instance()} - utils.repo2db_mapper(repo_list, remove_obsolete=False) + added, errors = utils.repo2db_mapper(repo_list) + assert not errors + assert len(added) == 1 class TestPasswordChanged(object): @@ -453,7 +507,7 @@ class TestGetEnabledHooks(object): def test_obfuscate_url_pw(): from rhodecode.lib.utils2 import obfuscate_url_pw - engine = u'/home/repos/malmö' + engine = '/home/repos/malmö' assert obfuscate_url_pw(engine) diff --git a/rhodecode/tests/routes.py b/rhodecode/tests/routes.py --- a/rhodecode/tests/routes.py +++ b/rhodecode/tests/routes.py @@ -182,7 +182,8 @@ def get_url_defs(): "admin_settings_vcs_svn_pattern_delete": ADMIN_PREFIX + "/settings/vcs/svn_pattern_delete", "admin_settings_mapping": ADMIN_PREFIX + "/settings/mapping", - "admin_settings_mapping_update": ADMIN_PREFIX + "/settings/mapping/update", + "admin_settings_mapping_create": ADMIN_PREFIX + "/settings/mapping/create", + "admin_settings_mapping_cleanup": ADMIN_PREFIX + "/settings/mapping/cleanup", "admin_settings_visual": ADMIN_PREFIX + "/settings/visual", "admin_settings_visual_update": ADMIN_PREFIX + "/settings/visual/update", "admin_settings_issuetracker": ADMIN_PREFIX + "/settings/issue-tracker",