feat(remap and rescan): added more relient remap and removal option, and also split the logic to either add or cleanup

rhodecode/api/tests/test_cleanup_repos.py

@@ -0,0 +1,44 b''
+# Copyright (C) 2010-2024 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import mock
+import pytest
+
+from rhodecode.model.scm import ScmModel
+from rhodecode.api.tests.utils import (
+    build_data, api_call, assert_ok, assert_error, crash)
+
+
[e-mail]("testuser_api", "app")
+class TestCleanupRepos(object):
+    def test_api_cleanup_repos(self):
+        id_, params = build_data(self.apikey, 'cleanup_repos')
+        response = api_call(self.app, params)
+
+        expected = {'removed': [], 'errors': []}
+        assert_ok(id_, expected, given=response.body)
+
+    def test_api_cleanup_repos_error(self):
+
+        id_, params = build_data(self.apikey, 'cleanup_repos', )
+
+        with mock.patch('rhodecode.lib.utils.repo2db_cleanup', side_effect=crash):
+            response = api_call(self.app, params)
+
+        expected = 'Error occurred during repo storage cleanup action'
+        assert_error(id_, expected, given=response.body)

rhodecode/api/tests/test_rescan_repos.py

@@ -19,7 +19,6 b''
 import mock
 import pytest
 
-from rhodecode.model.scm import ScmModel
 from rhodecode.api.tests.utils import (
     build_data, api_call, assert_ok, assert_error, crash)
 
@@ -30,12 +29,13 b' class TestRescanRepos(object):'
         id_, params = build_data(self.apikey, 'rescan_repos')
         response = api_call(self.app, params)
 
-        expected = {'added': [], 'removed': []}
+        expected = {'added': [], 'errors': []}
         assert_ok(id_, expected, given=response.body)
 
-    @mock.patch.object(ScmModel, 'repo_scan', crash)
-    def test_api_rescann_error(self):
+    def test_api_rescan_repos_error(self):
         id_, params = build_data(self.apikey, 'rescan_repos', )
+
+        with mock.patch('rhodecode.lib.utils.repo2db_mapper', side_effect=crash):
         response = api_call(self.app, params)
 
         expected = 'Error occurred during rescan repositories action'

rhodecode/api/views/server_api.py

@@ -18,14 +18,13 b''
 
 import logging
 import itertools
-import base64
 
 from rhodecode.api import (
     jsonrpc_method, JSONRPCError, JSONRPCForbidden, find_methods)
 
 from rhodecode.api.utils import (
     Optional, OAttr, has_superadmin_permission, get_user_or_error)
-from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_repo_store_path
+from rhodecode.lib.utils import get_rhodecode_repo_store_path
 from rhodecode.lib import system_info
 from rhodecode.lib import user_sessions
 from rhodecode.lib import exc_tracking
@@ -33,9 +32,7 b' from rhodecode.lib.ext_json import json'
 from rhodecode.lib.utils2 import safe_int
 from rhodecode.model.db import UserIpMap
 from rhodecode.model.scm import ScmModel
-from rhodecode.apps.file_store import utils as store_utils
-from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \
-    FileOverSizeException
+
 
 log = logging.getLogger(__name__)
 
@@ -158,13 +155,10 b' def get_ip(request, apiuser, userid=Opti'
 
 
 @jsonrpc_method()
-def rescan_repos(request, apiuser, remove_obsolete=Optional(False)):
+def rescan_repos(request, apiuser):
     """
     Triggers a rescan of the specified repositories.
-
-    * If the ``remove_obsolete`` option is set, it also deletes repositories
-      that are found in the database but not on the file system, so called
-      "clean zombies".
+    It returns list of added repositories, and errors during scan.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
@@ -173,9 +167,6 b' def rescan_repos(request, apiuser, remov'
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
-    :param remove_obsolete: Deletes repositories from the database that
-        are not found on the filesystem.
-    :type remove_obsolete: Optional(``True`` | ``False``)
 
     Example output:
 
@@ -184,7 +175,7 b' def rescan_repos(request, apiuser, remov'
       id : <id_given_in_input>
       result : {
         'added': [<added repository name>,...]
-        'removed': [<removed repository name>,...]
+        'errors': [<error_list>,...]
       }
       error :  null
 
@@ -199,28 +190,24 b' def rescan_repos(request, apiuser, remov'
       }
 
     """
+    from rhodecode.lib.utils import repo2db_mapper # re-import for testing patches
+
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     try:
-        rm_obsolete = Optional.extract(remove_obsolete)
-        added, removed = repo2db_mapper(ScmModel().repo_scan(),
-                                        remove_obsolete=rm_obsolete, force_hooks_rebuild=True)
-        return {'added': added, 'removed': removed}
+        added, errors = repo2db_mapper(ScmModel().repo_scan(), force_hooks_rebuild=True)
+        return {'added': added, 'errors': errors}
     except Exception:
-        log.exception('Failed to run repo rescann')
+        log.exception('Failed to run repo rescan')
         raise JSONRPCError(
             'Error occurred during rescan repositories action'
         )
 
 @jsonrpc_method()
-def cleanup_repos(request, apiuser, remove_obsolete=Optional(False)):
+def cleanup_repos(request, apiuser):
     """
-    Triggers a rescan of the specified repositories.
-
-    * If the ``remove_obsolete`` option is set, it also deletes repositories
-      that are found in the database but not on the file system, so called
-      "clean zombies".
+    Triggers a cleanup of non-existing repositories or repository groups in filesystem.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
@@ -229,9 +216,6 b' def cleanup_repos(request, apiuser, remo'
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
-    :param remove_obsolete: Deletes repositories from the database that
-        are not found on the filesystem.
-    :type remove_obsolete: Optional(``True`` | ``False``)
 
     Example output:
 
@@ -239,8 +223,8 b' def cleanup_repos(request, apiuser, remo'
 
       id : <id_given_in_input>
       result : {
-        'added': [<added repository name>,...]
-        'removed': [<removed repository name>,...]
+        'removed': [<removed repository name or repository group name>,...]
+        'errors': [<error list of failures to remove>,...]
       }
       error :  null
 
@@ -251,22 +235,22 b' def cleanup_repos(request, apiuser, remo'
       id : <id_given_in_input>
       result : null
       error :  {
-        'Error occurred during rescan repositories action'
+        'Error occurred during repo storage cleanup action'
       }
 
     """
+    from rhodecode.lib.utils import repo2db_cleanup  # re-import for testing patches
+
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     try:
-        rm_obsolete = Optional.extract(remove_obsolete)
-        added, removed = repo2db_mapper(ScmModel().repo_scan(),
-                                        remove_obsolete=rm_obsolete, force_hooks_rebuild=True)
-        return {'added': added, 'removed': removed}
+        removed, errors = repo2db_cleanup()
+        return {'removed': removed, 'errors': errors}
     except Exception:
-        log.exception('Failed to run repo rescann')
+        log.exception('Failed to run repo storage cleanup')
         raise JSONRPCError(
-            'Error occurred during rescan repositories action'
+            'Error occurred during repo storage cleanup action'
         )
 
 

rhodecode/apps/admin/__init__.py

@@ -361,12 +361,21 b' def admin_routes(config):'
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
-        name='admin_settings_mapping_update',
-        pattern='/settings/mapping/update')
+        name='admin_settings_mapping_create',
+        pattern='/settings/mapping/create')
     config.add_view(
         AdminSettingsView,
-        attr='settings_mapping_update',
-        route_name='admin_settings_mapping_update', request_method='POST',
+        attr='settings_mapping_create',
+        route_name='admin_settings_mapping_create', request_method='POST',
+        renderer='rhodecode:templates/admin/settings/settings.mako')
+
+    config.add_route(
+        name='admin_settings_mapping_cleanup',
+        pattern='/settings/mapping/cleanup')
+    config.add_view(
+        AdminSettingsView,
+        attr='settings_mapping_cleanup',
+        route_name='admin_settings_mapping_cleanup', request_method='POST',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(

rhodecode/apps/admin/tests/test_admin_repos.py

@@ -110,9 +110,11 b' class TestAdminRepos(object):'
                 repo_type=backend.alias,
                 repo_description=description,
                 csrf_token=csrf_token))
-
-        self.assert_repository_is_created_correctly(
-            repo_name, description, backend)
+        try:
+            self.assert_repository_is_created_correctly(repo_name, description, backend)
+        finally:
+            RepoModel().delete(numeric_repo)
+            Session().commit()
 
     @pytest.mark.parametrize("suffix", ['', '_ąćę'], ids=['', 'non-ascii'])
     def test_create_in_group(

rhodecode/apps/admin/views/settings.py

@@ -38,7 +38,7 b' from rhodecode.lib.auth import ('
     LoginRequired, HasPermissionAllDecorator, CSRFRequired)
 from rhodecode.lib.celerylib import tasks, run_task
 from rhodecode.lib.str_utils import safe_str
-from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_repo_store_path
+from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_repo_store_path, repo2db_cleanup
 from rhodecode.lib.utils2 import str2bool, AttributeDict
 from rhodecode.lib.index import searcher_from_config
 
@@ -233,13 +233,12 b' class AdminSettingsView(BaseAppView):'
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
-    def settings_mapping_update(self):
+    def settings_mapping_create(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'mapping'
-        rm_obsolete = self.request.POST.get('destroy', False)
         invalidate_cache = self.request.POST.get('invalidate', False)
-        log.debug('rescanning repo location with destroy obsolete=%s', rm_obsolete)
+        log.debug('rescanning repo location')
 
         if invalidate_cache:
             log.debug('invalidating all repositories cache')
@@ -247,16 +246,34 b' class AdminSettingsView(BaseAppView):'
                 ScmModel().mark_for_invalidation(repo.repo_name, delete=True)
 
         filesystem_repos = ScmModel().repo_scan()
-        added, removed = repo2db_mapper(filesystem_repos, rm_obsolete, force_hooks_rebuild=True)
+        added, errors = repo2db_mapper(filesystem_repos, force_hooks_rebuild=True)
         PermissionModel().trigger_permission_flush()
 
         def _repr(rm_repo):
             return ', '.join(map(safe_str, rm_repo)) or '-'
 
-        h.flash(_('Repositories successfully '
-                  'rescanned added: %s ; removed: %s') %
-                (_repr(added), _repr(removed)),
-                category='success')
+        if errors:
+            h.flash(_('Errors during scan: {}').format(_repr(errors), ), category='error')
+
+        h.flash(_('Repositories successfully scanned: Added: {}').format(_repr(added)), category='success')
+        raise HTTPFound(h.route_path('admin_settings_mapping'))
+
+    @LoginRequired()
+    @HasPermissionAllDecorator('hg.admin')
+    @CSRFRequired()
+    def settings_mapping_cleanup(self):
+        _ = self.request.translate
+        c = self.load_default_context()
+        c.active = 'mapping'
+        log.debug('rescanning repo location')
+
+        removed, errors = repo2db_cleanup()
+        PermissionModel().trigger_permission_flush()
+
+        def _repr(rm_repo):
+            return ', '.join(map(safe_str, rm_repo)) or '-'
+
+        h.flash(_('Repositories successfully scanned: Errors: {}, Added: {}').format(errors, _repr(removed)), category='success')
         raise HTTPFound(h.route_path('admin_settings_mapping'))
 
     @LoginRequired()

rhodecode/lib/utils.py

@@ -582,23 +582,19 b' def map_groups(path):'
     return group
 
 
-def repo2db_mapper(initial_repo_list, remove_obsolete=False, force_hooks_rebuild=False):
+def repo2db_mapper(initial_repo_list, force_hooks_rebuild=False):
     """
-    maps all repos given in initial_repo_list, non existing repositories
-    are created, if remove_obsolete is True it also checks for db entries
-    that are not in initial_repo_list and removes them.
-
-    :param initial_repo_list: list of repositories found by scanning methods
-    :param remove_obsolete: check for obsolete entries in database
+    maps all repos given in initial_repo_list, non-existing repositories
+    are created
     """
     from rhodecode.model.repo import RepoModel
-    from rhodecode.model.repo_group import RepoGroupModel
     from rhodecode.model.settings import SettingsModel
 
     sa = meta.Session()
     repo_model = RepoModel()
     user = User.get_first_super_admin()
     added = []
+    errors = []
 
     # creation defaults
     defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
@@ -616,9 +612,7 b' def repo2db_mapper(initial_repo_list, re'
         if not db_repo:
             log.info('repository `%s` not found in the database, creating now', name)
             added.append(name)
-            desc = (repo.description
-                    if repo.description != 'unknown'
-                    else '%s repository' % name)
+            desc = repo.description if repo.description != 'unknown' else f'{name} repository'
 
             db_repo = repo_model._create_repo(
                 repo_name=name,
@@ -633,76 +627,115 b' def repo2db_mapper(initial_repo_list, re'
                 state=Repository.STATE_CREATED
             )
             sa.commit()
-            # we added that repo just now, and make sure we updated server info
-            if db_repo.repo_type == 'git':
-                git_repo = db_repo.scm_instance()
-                # update repository server-info
-                log.debug('Running update server info')
-                git_repo._update_server_info(force=True)
 
-            db_repo.update_commit_cache(recursive=False)
-
+        try:
         config = db_repo._config
         config.set('extensions', 'largefiles', '')
-        repo = db_repo.scm_instance(config=config)
-        repo.install_hooks(force=force_hooks_rebuild)
+            scm_repo = db_repo.scm_instance(config=config)
+        except Exception:
+            log.error(traceback.format_exc())
+            errors.append(f'getting vcs instance for {name} failed')
+            continue
+
+        try:
+            db_repo.update_commit_cache(recursive=False)
+        except Exception:
+            log.error(traceback.format_exc())
+            errors.append(f'update_commit_cache for {name} failed')
+            continue
+
+        try:
+            scm_repo.install_hooks(force=force_hooks_rebuild)
+        except Exception:
+            log.error(traceback.format_exc())
+            errors.append(f'install_hooks for {name} failed')
+            continue
 
+        try:
+            # we added that repo just now, and make sure we updated server info
+            if db_repo.repo_type == 'git':
+                # update repository server-info
+                log.debug('Running update server info')
+                scm_repo._update_server_info(force=True)
+        except Exception:
+            log.error(traceback.format_exc())
+            errors.append(f'update_server_info for {name} failed')
+            continue
+
+    return added, errors
+
+def repo2db_cleanup(skip_repos=None, skip_groups=None):
+    from rhodecode.model.repo import RepoModel
+    from rhodecode.model.repo_group import RepoGroupModel
+
+    sa = meta.Session()
     removed = []
-    if remove_obsolete:
+    errors = []
+
+
+    all_repos = Repository.execute(
+        Repository.select(Repository)\
+            .order_by(Repository.repo_name)
+    ).scalars()
+
         # remove from database those repositories that are not in the filesystem
-        for repo in sa.query(Repository).all():
-            if repo.repo_name not in list(initial_repo_list.keys()):
-                log.debug("Removing non-existing repository found in db `%s`",
-                          repo.repo_name)
+    for db_repo in all_repos:
+        db_repo_name = db_repo.repo_name
+        if skip_repos and db_repo_name in skip_repos:
+            log.debug('Skipping repo `%s`', db_repo_name)
+            continue
                 try:
-                    RepoModel(sa).delete(repo, forks='detach', fs_remove=False)
+            instance = db_repo.scm_instance()
+        except Exception:
+            instance = None
+
+        if not instance:
+            log.debug("Removing non-existing repository found in db `%s`", db_repo_name)
+            try:
+                RepoModel(sa).delete(db_repo, forks='detach', fs_remove=False, call_events=False)
                     sa.commit()
-                    removed.append(repo.repo_name)
+                removed.append(db_repo_name)
                 except Exception:
                     # don't hold further removals on error
                     log.error(traceback.format_exc())
                     sa.rollback()
-
-        def splitter(full_repo_name):
-            _parts = full_repo_name.rsplit(RepoGroup.url_sep(), 1)
-            gr_name = None
-            if len(_parts) == 2:
-                gr_name = _parts[0]
-            return gr_name
-
-        initial_repo_group_list = [splitter(x) for x in
-                                   list(initial_repo_list.keys()) if splitter(x)]
+                errors.append(db_repo_name)
 
         # remove from database those repository groups that are not in the
         # filesystem due to parent child relationships we need to delete them
         # in a specific order of most nested first
-        all_groups = [x.group_name for x in sa.query(RepoGroup).all()]
+    all_groups = RepoGroup.execute(
+        RepoGroup.select(RepoGroup.group_name)\
+            .order_by(RepoGroup.group_name)
+    ).scalars().all()
+
         def nested_sort(gr):
             return len(gr.split('/'))
+
         for group_name in sorted(all_groups, key=nested_sort, reverse=True):
-            if group_name not in initial_repo_group_list:
-                repo_group = RepoGroup.get_by_group_name(group_name)
-                if (repo_group.children.all() or
-                    not RepoGroupModel().check_exist_filesystem(
-                        group_name=group_name, exc_on_failure=False)):
+        if skip_groups and group_name in skip_groups:
+            log.debug('Skipping repo group `%s`', group_name)
                     continue
 
-                log.info(
-                    'Removing non-existing repository group found in db `%s`',
-                    group_name)
+        repo_group = RepoGroup.get_by_group_name(group_name)
+
+        if repo_group.children.all() or not RepoGroupModel().check_exist_filesystem(group_name=group_name, exc_on_failure=False):
+            continue
+
+        log.info('Removing non-existing repository group found in db `%s`', group_name)
+
                 try:
-                    RepoGroupModel(sa).delete(group_name, fs_remove=False)
+            RepoGroupModel(sa).delete(group_name, fs_remove=False, call_events=False)
                     sa.commit()
                     removed.append(group_name)
                 except Exception:
                     # don't hold further removals on error
-                    log.exception(
-                        'Unable to remove repository group `%s`',
-                        group_name)
+            log.exception('Unable to remove repository group `%s`',group_name)
                     sa.rollback()
-                    raise
+            errors.append(group_name)
 
-    return added, removed
+    return removed, errors
+
 
 def deep_reload_package(package_name):
     """
@@ -829,7 +862,7 b' def create_test_repositories(test_path, '
         raise ImportError('Failed to import rc_testdata, '
                           'please make sure this package is installed from requirements_test.txt')
 
-    from rhodecode.tests import HG_REPO, GIT_REPO, SVN_REPO
+    from rhodecode.bootstrap import HG_REPO, GIT_REPO, SVN_REPO
 
     log.debug('making test vcs repositories at %s', test_path)
 

rhodecode/lib/vcs/backends/__init__.py

@@ -58,9 +58,10 b' def get_vcs_instance(repo_path, *args, *'
                 raise VCSError(f"Given path {repo_path} is not a directory")
     except VCSError:
         log.exception(
-            'Perhaps this repository is in db and not in '
-            'filesystem run rescan repositories with '
-            '"destroy old data" option from admin panel')
+            'Perhaps this repository is in db and not in filesystem.'
+            'Run cleanup filesystem option from admin settings under Remap and rescan'
+            )
+
         return None
 
     return backend(repo_path=repo_path, *args, **kwargs)

rhodecode/model/repo.py

@@ -745,7 +745,7 b' class RepoModel(BaseModel):'
                 log.error(traceback.format_exc())
                 raise
 
-    def delete(self, repo, forks=None, pull_requests=None, artifacts=None, fs_remove=True, cur_user=None):
+    def delete(self, repo, forks=None, pull_requests=None, artifacts=None, fs_remove=True, cur_user=None, call_events=True):
         """
         Delete given repository, forks parameter defines what do do with
         attached forks. Throws AttachedForksError if deleted repo has attached
@@ -760,7 +760,9 b' class RepoModel(BaseModel):'
         if not cur_user:
             cur_user = getattr(get_current_rhodecode_user(), 'username', None)
         repo = self._get_repo(repo)
-        if repo:
+        if not repo:
+            return False
+
             if forks == 'detach':
                 for r in repo.forks:
                     r.fork = None
@@ -785,7 +787,9 b' class RepoModel(BaseModel):'
                 raise AttachedArtifactsError()
 
             old_repo_dict = repo.get_dict()
+        if call_events:
             events.trigger(events.RepoPreDeleteEvent(repo))
+
             try:
                 self.sa.delete(repo)
                 if fs_remove:
@@ -796,12 +800,15 b' class RepoModel(BaseModel):'
                     'deleted_by': cur_user,
                     'deleted_on': time.time(),
                 })
+            if call_events:
                 hooks_base.delete_repository(**old_repo_dict)
                 events.trigger(events.RepoDeleteEvent(repo))
             except Exception:
                 log.error(traceback.format_exc())
                 raise
 
+        return True
+
     def grant_user_permission(self, repo, user, perm):
         """
         Grant permission for user on given repository, or update existing one

rhodecode/model/repo_group.py

@@ -156,7 +156,7 b' class RepoGroupModel(BaseModel):'
 
     def check_exist_filesystem(self, group_name, exc_on_failure=True):
         create_path = os.path.join(self.repos_path, group_name)
-        log.debug('creating new group in %s', create_path)
+        log.debug('checking FS presence for repo group in %s', create_path)
 
         if os.path.isdir(create_path):
             if exc_on_failure:
@@ -573,10 +573,11 b' class RepoGroupModel(BaseModel):'
             log.error(traceback.format_exc())
             raise
 
-    def delete(self, repo_group, force_delete=False, fs_remove=True):
+    def delete(self, repo_group, force_delete=False, fs_remove=True, call_events=True):
         repo_group = self._get_repo_group(repo_group)
         if not repo_group:
             return False
+        repo_group_name = repo_group.group_name
         try:
             self.sa.delete(repo_group)
             if fs_remove:
@@ -585,13 +586,15 b' class RepoGroupModel(BaseModel):'
                 log.debug('skipping removal from filesystem')
 
             # Trigger delete event.
+            if call_events:
             events.trigger(events.RepoGroupDeleteEvent(repo_group))
-            return True
 
         except Exception:
-            log.error('Error removing repo_group %s', repo_group)
+            log.error('Error removing repo_group %s', repo_group_name)
             raise
 
+        return True
+
     def grant_user_permission(self, repo_group, user, perm):
         """
         Grant permission for user on given repository group, or update

rhodecode/subscribers.py

@@ -121,7 +121,7 b' def scan_repositories_if_enabled(event):'
         from rhodecode.lib.utils import repo2db_mapper
         scm = ScmModel()
         repositories = scm.repo_scan(scm.repos_path)
-        repo2db_mapper(repositories, remove_obsolete=False)
+        repo2db_mapper(repositories)
 
 
 def write_metadata_if_needed(event):

rhodecode/templates/admin/settings/settings_mapping.mako

@@ -1,33 +1,45 b''
-${h.secure_form(h.route_path('admin_settings_mapping_update'), request=request)}
+
 
 <div class="panel panel-default">
     <div class="panel-heading">
-        <h3 class="panel-title">${_('Import New Groups or Repositories')}</h3>
+        <h3 class="panel-title">${_('Import new repository groups and repositories')}</h3>
     </div>
     <div class="panel-body">
-
+    ${h.secure_form(h.route_path('admin_settings_mapping_create'), request=request)}
         <p>
-            ${_('This function will scann all data under the current storage path location at')} <code>${c.storage_path}</code>
+            ${_('This function will scan all data under the current storage path location at')} <code>${c.storage_path}</code><br/>
+            ${_('Each folder will be imported as a new repository group, and each repository found will be also imported to root level or corresponding repository group')}
         </p>
 
         <div class="checkbox">
-            ${h.checkbox('destroy',True)}
-            <label for="destroy">${_('Destroy old data')}</label>
-        </div>
-        <span class="help-block">${_('In case a repository or a group was deleted from the filesystem and it still exists in the database, check this option to remove obsolete data from the database.')}</span>
-
-        <div class="checkbox">
             ${h.checkbox('invalidate',True)}
             <label for="invalidate"> ${_('Invalidate cache for all repositories')}</label>
         </div>
         <span class="help-block">${_('Each cache data for repositories will be cleaned with this option selected. Use this to reload data and clear cache keys.')}</span>
 
         <div class="buttons">
-            ${h.submit('rescan',_('Rescan Filesystem'),class_="btn")}
+            ${h.submit('rescan',_('Scan filesystem'),class_="btn")}
         </div>
-
+    ${h.end_form()}
     </div>
 </div>
 
 
+<div class="panel panel-default">
+    <div class="panel-heading">
+        <h3 class="panel-title">${_('Cleanup removed Repository Groups or Repositories')}</h3>
+    </div>
+    <div class="panel-body">
+    ${h.secure_form(h.route_path('admin_settings_mapping_cleanup'), request=request)}
+        <p>
+            ${_('This function will scan all data under the current storage path location at')} <code>${c.storage_path}</code>
+            ${_('Then it will remove all repository groups and repositories that are no longer present in the filesystem.')}
+        </p>
+
+        <div class="buttons">
+            ${h.submit('rescan',_('Cleanup filesystem'),class_="btn btn-danger")}
+        </div>
 ${h.end_form()}
+    </div>
+</div>
+

rhodecode/tests/fixtures/fixture_utils.py

@@ -1558,6 +1558,7 b' def stub_integration_settings():'
 
 @pytest.fixture()
 def repo_integration_stub(request, repo_stub, StubIntegrationType, stub_integration_settings):
+    repo_id = repo_stub.repo_id
     integration = IntegrationModel().create(
         StubIntegrationType,
         settings=stub_integration_settings,
@@ -1571,6 +1572,7 b' def repo_integration_stub(request, repo_'
     @request.addfinalizer
     def cleanup():
         IntegrationModel().delete(integration)
+        RepoModel().delete(repo_id)
 
     return integration
 

rhodecode/tests/integrations/test_integration.py

@@ -20,23 +20,21 b' import time'
 import pytest
 
 from rhodecode import events
+from rhodecode.model.repo import RepoModel
 from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.model.db import Session, Integration
 from rhodecode.model.integration import IntegrationModel
 
 
 class TestDeleteScopesDeletesIntegrations(object):
-    def test_delete_repo_with_integration_deletes_integration(
-            self, repo_integration_stub):
-
-        Session().delete(repo_integration_stub.repo)
+    def test_delete_repo_with_integration_deletes_integration(self, repo_integration_stub):
+        RepoModel().delete(repo_integration_stub.repo)
         Session().commit()
         Session().expire_all()
         integration = Integration.get(repo_integration_stub.integration_id)
         assert integration is None
 
-    def test_delete_repo_group_with_integration_deletes_integration(
-            self, repogroup_integration_stub):
+    def test_delete_repo_group_with_integration_deletes_integration(self, repogroup_integration_stub):
 
         Session().delete(repogroup_integration_stub.repo_group)
         Session().commit()
@@ -52,7 +50,7 b' def counter():'
     global count
     val = count
     count += 1
-    return '{}_{}'.format(val, time.time())
+    return f'{val}_{time.time()}'
 
 
 @pytest.fixture()

rhodecode/tests/lib/test_utils.py

@@ -18,20 +18,25 b''
 
 import multiprocessing
 import os
+import shutil
 
 import mock
 import py
 import pytest
 
+import rhodecode
 from rhodecode.lib import caching_query
 from rhodecode.lib import utils
 from rhodecode.lib.str_utils import safe_bytes
 from rhodecode.model import settings
 from rhodecode.model import db
 from rhodecode.model import meta
+from rhodecode.model.meta import Session
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.repo_group import RepoGroupModel
+from rhodecode.model.scm import ScmModel
 from rhodecode.model.settings import UiSetting, SettingsModel
+from rhodecode.tests.fixtures.fixture_pyramid import rhodecode_factory
 from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode_tools.lib.hash_utils import md5_safe
 from rhodecode.lib.ext_json import json
@@ -230,7 +235,7 b' def platform_encodes_filenames():'
     return path_with_latin1 != read_path
 
 
-def test_repo2db_mapper_groups(repo_groups):
+def test_repo2db_cleaner_removes_zombie_groups(repo_groups):
     session = meta.Session()
     zombie_group, parent_group, child_group = repo_groups
     zombie_path = os.path.join(
@@ -238,10 +243,9 b' def test_repo2db_mapper_groups(repo_grou'
     os.rmdir(zombie_path)
 
     # Avoid removing test repos when calling repo2db_mapper
-    repo_list = {
-        repo.repo_name: 'test' for repo in session.query(db.Repository).all()
-    }
-    utils.repo2db_mapper(repo_list, remove_obsolete=True)
+    repo_list = [repo.repo_name for repo in session.query(db.Repository).all()]
+
+    utils.repo2db_cleanup(skip_repos=repo_list)
 
     groups_in_db = session.query(db.RepoGroup).all()
     assert child_group in groups_in_db
@@ -249,20 +253,68 b' def test_repo2db_mapper_groups(repo_grou'
     assert zombie_path not in groups_in_db
 
 
-def test_repo2db_mapper_enables_largefiles(backend):
+
[e-mail]("hg", "git", "svn")
+def test_repo2db_cleaner_removes_zombie_repos(backend):
     repo = backend.create_repo()
-    repo_list = {repo.repo_name: 'test'}
-    with mock.patch('rhodecode.model.db.Repository.scm_instance') as scm_mock:
-        utils.repo2db_mapper(repo_list, remove_obsolete=False)
-        _, kwargs = scm_mock.call_args
-        assert kwargs['config'].get('extensions', 'largefiles') == ''
+    zombie_path = repo.repo_full_path
+    shutil.rmtree(zombie_path)
+
+    removed, errors = utils.repo2db_cleanup()
+    assert len(removed) == 1
+    assert not errors
 
 
[e-mail]("git", "svn")
+def test_repo2db_mapper_adds_new_repos(request, backend):
+    repo = backend.create_repo()
+    cleanup_repos = []
+    cleanup_groups = []
+    for num in range(5):
+        copy_repo_name = f'{repo.repo_name}-{num}'
+        copy_repo_path = f'{repo.repo_full_path}-{num}'
+
+        shutil.copytree(repo.repo_full_path, copy_repo_path)
+        cleanup_repos.append(copy_repo_name)
+
+    for gr_num in range(5):
+        gr_name = f'my_gr_{gr_num}'
+        dest_gr = os.path.join(os.path.dirname(repo.repo_full_path), gr_name)
+        os.makedirs(dest_gr, exist_ok=True)
+
+        copy_repo_name = f'{gr_name}/{repo.repo_name}-{gr_num}'
+        copy_repo_path = f'{dest_gr}/{repo.repo_name}-{gr_num}'
+
+        shutil.copytree(repo.repo_full_path, copy_repo_path)
+        cleanup_repos.append(copy_repo_name)
+        cleanup_groups.append(gr_name)
+
+    repo_list = ScmModel().repo_scan()
+
+    added, errors = utils.repo2db_mapper(repo_list)
+    Session().commit()
+    assert not errors
+
+    assert len(added) == 10
+
+    @request.addfinalizer
+    def cleanup():
+        for _repo in cleanup_repos:
+            del_result = RepoModel().delete(_repo, call_events=False)
+            Session().commit()
+            assert del_result is True
+
+        for _repo_group in cleanup_groups:
+            del_result = RepoGroupModel().delete(_repo_group, force_delete=True, call_events=False)
+            Session().commit()
+            assert del_result is True
+
+
 def test_repo2db_mapper_installs_hooks_for_repos_in_db(backend):
     repo = backend.create_repo()
     repo_list = {repo.repo_name: 'test'}
-    utils.repo2db_mapper(repo_list, remove_obsolete=False)
+    added, errors = utils.repo2db_mapper(repo_list)
+    assert not errors
+    assert repo.scm_instance().get_hooks_info() == {'pre_version': rhodecode.__version__, 'post_version': rhodecode.__version__}
 
 
 @pytest.mark.backends("git", "svn")
@@ -271,7 +323,9 b' def test_repo2db_mapper_installs_hooks_f'
     RepoModel().delete(repo, fs_remove=False)
     meta.Session().commit()
     repo_list = {repo.repo_name: repo.scm_instance()}
-    utils.repo2db_mapper(repo_list, remove_obsolete=False)
+    added, errors = utils.repo2db_mapper(repo_list)
+    assert not errors
+    assert len(added) == 1
 
 
 class TestPasswordChanged(object):
@@ -453,7 +507,7 b' class TestGetEnabledHooks(object):'
 
 def test_obfuscate_url_pw():
     from rhodecode.lib.utils2 import obfuscate_url_pw
-    engine = u'/home/repos/malmö'
+    engine = '/home/repos/malmö'
     assert obfuscate_url_pw(engine)
 
 

rhodecode/tests/routes.py

@@ -182,7 +182,8 b' def get_url_defs():'
         "admin_settings_vcs_svn_pattern_delete": ADMIN_PREFIX
         + "/settings/vcs/svn_pattern_delete",
         "admin_settings_mapping": ADMIN_PREFIX + "/settings/mapping",
-        "admin_settings_mapping_update": ADMIN_PREFIX + "/settings/mapping/update",
+        "admin_settings_mapping_create": ADMIN_PREFIX + "/settings/mapping/create",
+        "admin_settings_mapping_cleanup": ADMIN_PREFIX + "/settings/mapping/cleanup",
         "admin_settings_visual": ADMIN_PREFIX + "/settings/visual",
         "admin_settings_visual_update": ADMIN_PREFIX + "/settings/visual/update",
         "admin_settings_issuetracker": ADMIN_PREFIX + "/settings/issue-tracker",