# HG changeset patch # User Serhii Ilin # Date 2024-03-01 07:39:14 # Node ID 99a9110049dcf8481e6b6de6187d5fdbf00c40e9 # Parent cede61e3bb955da2601085222347cfeeaf5a9852 feat(repo_path-config): moved main storage location path into ini file. Fixes: RCCE-61 diff --git a/configs/development.ini b/configs/development.ini --- a/configs/development.ini +++ b/configs/development.ini @@ -261,8 +261,8 @@ auth_ret_code_detection = false ; codes don't break the transactions while 4XX codes do lock_ret_code = 423 -; allows to change the repository location in settings page -allow_repo_location_change = true +; Filesystem location were repositories should be stored +repo_store.path = /var/opt/rhodecode_repo_store ; allows to setup custom hooks in settings page allow_custom_hooks_settings = true diff --git a/configs/production.ini b/configs/production.ini --- a/configs/production.ini +++ b/configs/production.ini @@ -212,8 +212,8 @@ auth_ret_code_detection = false ; codes don't break the transactions while 4XX codes do lock_ret_code = 423 -; allows to change the repository location in settings page -allow_repo_location_change = true +; Filesystem location were repositories should be stored +repo_store.path = /var/opt/rhodecode_repo_store ; allows to setup custom hooks in settings page allow_custom_hooks_settings = true diff --git a/rhodecode/api/views/server_api.py b/rhodecode/api/views/server_api.py --- a/rhodecode/api/views/server_api.py +++ b/rhodecode/api/views/server_api.py @@ -25,7 +25,7 @@ from rhodecode.api import ( from rhodecode.api.utils import ( Optional, OAttr, has_superadmin_permission, get_user_or_error) -from rhodecode.lib.utils import repo2db_mapper +from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_repo_store_path from rhodecode.lib import system_info from rhodecode.lib import user_sessions from rhodecode.lib import exc_tracking @@ -33,7 +33,6 @@ from rhodecode.lib.ext_json import json from rhodecode.lib.utils2 import safe_int from rhodecode.model.db import UserIpMap from rhodecode.model.scm import ScmModel -from rhodecode.model.settings import VcsSettingsModel from rhodecode.apps.file_store import utils from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \ FileOverSizeException @@ -103,7 +102,7 @@ def get_repo_store(request, apiuser): if not has_superadmin_permission(apiuser): raise JSONRPCForbidden() - path = VcsSettingsModel().get_repos_location() + path = get_rhodecode_repo_store_path() return {"path": path} diff --git a/rhodecode/apps/admin/views/settings.py b/rhodecode/apps/admin/views/settings.py --- a/rhodecode/apps/admin/views/settings.py +++ b/rhodecode/apps/admin/views/settings.py @@ -38,7 +38,7 @@ from rhodecode.lib.auth import ( LoginRequired, HasPermissionAllDecorator, CSRFRequired) from rhodecode.lib.celerylib import tasks, run_task from rhodecode.lib.str_utils import safe_str -from rhodecode.lib.utils import repo2db_mapper +from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_repo_store_path from rhodecode.lib.utils2 import str2bool, AttributeDict from rhodecode.lib.index import searcher_from_config @@ -167,9 +167,6 @@ class AdminSettingsView(BaseAppView): return Response(html) try: - if c.visual.allow_repo_location_change: - model.update_global_path_setting(form_result['paths_root_path']) - model.update_global_ssl_setting(form_result['web_push_ssl']) model.update_global_hook_settings(form_result) @@ -217,7 +214,7 @@ class AdminSettingsView(BaseAppView): def settings_mapping(self): c = self.load_default_context() c.active = 'mapping' - c.storage_path = VcsSettingsModel().get_repos_location() + c.storage_path = get_rhodecode_repo_store_path() data = render('rhodecode:templates/admin/settings/settings.mako', self._get_template_context(c), self.request) html = formencode.htmlfill.render( diff --git a/rhodecode/apps/svn_support/utils.py b/rhodecode/apps/svn_support/utils.py --- a/rhodecode/apps/svn_support/utils.py +++ b/rhodecode/apps/svn_support/utils.py @@ -22,7 +22,7 @@ import os from pyramid.renderers import render from rhodecode.events import trigger -from rhodecode.lib.utils import get_rhodecode_realm, get_rhodecode_base_path +from rhodecode.lib.utils import get_rhodecode_realm, get_rhodecode_repo_store_path from rhodecode.lib.utils2 import str2bool from rhodecode.model.db import RepoGroup @@ -38,7 +38,7 @@ def write_mod_dav_svn_config(settings): file_path = settings[config_keys.config_file_path] config = _render_mod_dav_svn_config( use_ssl=use_ssl, - parent_path_root=get_rhodecode_base_path(), + parent_path_root=get_rhodecode_repo_store_path(), list_parent_path=settings[config_keys.list_parent_path], location_root=settings[config_keys.location_root], repo_groups=RepoGroup.get_all_repo_groups(), diff --git a/rhodecode/config/config_maker.py b/rhodecode/config/config_maker.py --- a/rhodecode/config/config_maker.py +++ b/rhodecode/config/config_maker.py @@ -117,6 +117,8 @@ def sanitize_settings_and_apply_defaults settings_maker.make_setting('vcs.methods.cache', True, parser='bool') + # repo_store path + settings_maker.make_setting('repo_store.path', '/var/opt/rhodecode_repo_store') # Support legacy values of vcs.scm_app_implementation. Legacy # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'. diff --git a/rhodecode/config/environment.py b/rhodecode/config/environment.py --- a/rhodecode/config/environment.py +++ b/rhodecode/config/environment.py @@ -81,7 +81,6 @@ def load_pyramid_environment(global_conf rhodecode.PYRAMID_SETTINGS = settings_merged rhodecode.CONFIG = settings_merged rhodecode.CONFIG['default_user_id'] = utils.get_default_user_id() - rhodecode.CONFIG['default_base_path'] = utils.get_default_base_path() if vcs_server_enabled: connect_vcs(vcs_server_uri, utils.get_vcs_server_protocol(settings)) diff --git a/rhodecode/config/utils.py b/rhodecode/config/utils.py --- a/rhodecode/config/utils.py +++ b/rhodecode/config/utils.py @@ -102,15 +102,3 @@ def get_default_user_id(): user_id = result.first()[0] return user_id - - -def get_default_base_path(): - from sqlalchemy import text - from rhodecode.model import meta - - engine = meta.get_engine() - with meta.SA_Session(engine) as session: - result = session.execute(text("SELECT ui_value from rhodecode_ui where ui_key = '/'")) - base_path = result.first()[0] - - return base_path diff --git a/rhodecode/lib/base.py b/rhodecode/lib/base.py --- a/rhodecode/lib/base.py +++ b/rhodecode/lib/base.py @@ -347,8 +347,6 @@ def attach_context_attributes(context, r context.ssh_key_generator_enabled = str2bool( config.get('ssh.enable_ui_key_generator', 'true')) - context.visual.allow_repo_location_change = str2bool( - config.get('allow_repo_location_change', True)) context.visual.allow_custom_hooks_settings = str2bool( config.get('allow_custom_hooks_settings', True)) context.debug_style = str2bool(config.get('debug_style', False)) diff --git a/rhodecode/lib/middleware/simplevcs.py b/rhodecode/lib/middleware/simplevcs.py --- a/rhodecode/lib/middleware/simplevcs.py +++ b/rhodecode/lib/middleware/simplevcs.py @@ -156,17 +156,10 @@ class SimpleVCS(object): @property def base_path(self): - settings_path = self.repo_vcs_config.get(*VcsSettingsModel.PATH_SETTING) - - if not settings_path: - settings_path = self.global_vcs_config.get(*VcsSettingsModel.PATH_SETTING) + settings_path = self.config.get('repo_store.path') if not settings_path: - # try, maybe we passed in explicitly as config option - settings_path = self.config.get('base_path') - - if not settings_path: - raise ValueError('FATAL: base_path is empty') + raise ValueError('FATAL: repo_store.path is empty') return settings_path def set_repo_names(self, environ): diff --git a/rhodecode/lib/system_info.py b/rhodecode/lib/system_info.py --- a/rhodecode/lib/system_info.py +++ b/rhodecode/lib/system_info.py @@ -331,8 +331,8 @@ def cpu(): @register_sysinfo def storage(): from rhodecode.lib.helpers import format_byte_size_binary - from rhodecode.model.settings import VcsSettingsModel - path = VcsSettingsModel().get_repos_location() + from rhodecode.lib.utils import get_rhodecode_repo_store_path + path = get_rhodecode_repo_store_path() value = dict(percent=0, used=0, total=0, path=path, text='') state = STATE_OK_DEFAULT @@ -364,8 +364,8 @@ def storage(): @register_sysinfo def storage_inodes(): - from rhodecode.model.settings import VcsSettingsModel - path = VcsSettingsModel().get_repos_location() + from rhodecode.lib.utils import get_rhodecode_repo_store_path + path = get_rhodecode_repo_store_path() value = dict(percent=0.0, free=0, used=0, total=0, path=path, text='') state = STATE_OK_DEFAULT @@ -442,11 +442,10 @@ def storage_archives(): @register_sysinfo def storage_gist(): from rhodecode.model.gist import GIST_STORE_LOC - from rhodecode.model.settings import VcsSettingsModel - from rhodecode.lib.utils import safe_str + from rhodecode.lib.utils import safe_str, get_rhodecode_repo_store_path from rhodecode.lib.helpers import format_byte_size_binary path = safe_str(os.path.join( - VcsSettingsModel().get_repos_location(), GIST_STORE_LOC)) + get_rhodecode_repo_store_path(), GIST_STORE_LOC)) # gist storage value = dict(percent=0, used=0, total=0, items=0, path=path, text='') diff --git a/rhodecode/lib/utils.py b/rhodecode/lib/utils.py --- a/rhodecode/lib/utils.py +++ b/rhodecode/lib/utils.py @@ -472,14 +472,14 @@ def get_rhodecode_realm(): return safe_str(realm.app_settings_value) -def get_rhodecode_base_path(): +def get_rhodecode_repo_store_path(): """ Returns the base path. The base path is the filesystem path which points to the repository store. """ import rhodecode - return rhodecode.CONFIG['default_base_path'] + return rhodecode.CONFIG['repo_store.path'] def map_groups(path): diff --git a/rhodecode/model/__init__.py b/rhodecode/model/__init__.py --- a/rhodecode/model/__init__.py +++ b/rhodecode/model/__init__.py @@ -21,6 +21,7 @@ import logging import rhodecode from rhodecode.model import meta, db +from rhodecode.lib.utils import get_rhodecode_repo_store_path from rhodecode.lib.utils2 import obfuscate_url_pw, get_encryption_key log = logging.getLogger(__name__) @@ -138,3 +139,11 @@ class BaseModel(object): Returns all instances of what is defined in `cls` class variable """ return cls.cls.getAll() + + @property + def repos_path(self): + """ + Gets the repositories root path from *ini file + """ + + return get_rhodecode_repo_store_path() diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py --- a/rhodecode/model/db.py +++ b/rhodecode/model/db.py @@ -1942,8 +1942,8 @@ class Repository(Base, BaseModel): :param cls: """ - from rhodecode.lib.utils import get_rhodecode_base_path - return get_rhodecode_base_path() + from rhodecode.lib.utils import get_rhodecode_repo_store_path + return get_rhodecode_repo_store_path() @classmethod def get_all_repos(cls, user_id=Optional(None), group_id=Optional(None), @@ -2009,16 +2009,13 @@ class Repository(Base, BaseModel): def groups_and_repo(self): return self.groups_with_parents, self - @LazyProperty + @property def repo_path(self): """ Returns base full path for that repository means where it actually exists on a filesystem """ - q = Session().query(RhodeCodeUi).filter( - RhodeCodeUi.ui_key == self.NAME_SEP) - q = q.options(FromCache("sql_cache_short", "repository_repo_path")) - return q.one().ui_value + return self.base_path() @property def repo_full_path(self): diff --git a/rhodecode/model/forms.py b/rhodecode/model/forms.py --- a/rhodecode/model/forms.py +++ b/rhodecode/model/forms.py @@ -434,10 +434,6 @@ def ApplicationUiSettingsForm(localizer) class _ApplicationUiSettingsForm(_BaseVcsSettingsForm): web_push_ssl = v.StringBoolean(if_missing=False) - paths_root_path = All( - v.ValidPath(localizer), - v.UnicodeString(strip=True, min=1, not_empty=True) - ) largefiles_usercache = All( v.ValidPath(localizer), v.UnicodeString(strip=True, min=2, not_empty=True)) diff --git a/rhodecode/model/repo.py b/rhodecode/model/repo.py --- a/rhodecode/model/repo.py +++ b/rhodecode/model/repo.py @@ -83,14 +83,6 @@ class RepoModel(BaseModel): return repo_to_perm - @LazyProperty - def repos_path(self): - """ - Gets the repositories root path from database - """ - settings_model = VcsSettingsModel(sa=self.sa) - return settings_model.get_repos_location() - def get(self, repo_id): repo = self.sa.query(Repository) \ .filter(Repository.repo_id == repo_id) diff --git a/rhodecode/model/repo_group.py b/rhodecode/model/repo_group.py --- a/rhodecode/model/repo_group.py +++ b/rhodecode/model/repo_group.py @@ -62,15 +62,6 @@ class RepoGroupModel(BaseModel): def get_repo_group(self, repo_group): return self._get_repo_group(repo_group) - @LazyProperty - def repos_path(self): - """ - Gets the repositories root path from database - """ - - settings_model = VcsSettingsModel(sa=self.sa) - return settings_model.get_repos_location() - def get_by_group_name(self, repo_group_name, cache=None): repo = self.sa.query(RepoGroup) \ .filter(RepoGroup.group_name == repo_group_name) diff --git a/rhodecode/model/scm.py b/rhodecode/model/scm.py --- a/rhodecode/model/scm.py +++ b/rhodecode/model/scm.py @@ -189,15 +189,6 @@ class ScmModel(BaseModel): Generic Scm Model """ - @LazyProperty - def repos_path(self): - """ - Gets the repositories root path from database - """ - - settings_model = VcsSettingsModel(sa=self.sa) - return settings_model.get_repos_location() - def repo_scan(self, repos_path=None): """ Listing of repositories in given path. This path should not be a diff --git a/rhodecode/model/settings.py b/rhodecode/model/settings.py --- a/rhodecode/model/settings.py +++ b/rhodecode/model/settings.py @@ -811,9 +811,6 @@ class VcsSettingsModel(object): else: return self.get_repo_general_settings() - def get_repos_location(self): - return self.global_settings.get_ui_by_key('/').ui_value - def _filter_ui_settings(self, settings): filtered_settings = [ s for s in settings if self._should_keep_setting(s)] diff --git a/rhodecode/subscribers.py b/rhodecode/subscribers.py --- a/rhodecode/subscribers.py +++ b/rhodecode/subscribers.py @@ -116,8 +116,9 @@ def scan_repositories_if_enabled(event): import_on_startup = settings['startup.import_repos'] if vcs_server_enabled and import_on_startup: from rhodecode.model.scm import ScmModel - from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_base_path - repositories = ScmModel().repo_scan(get_rhodecode_base_path()) + from rhodecode.lib.utils import repo2db_mapper + scm = ScmModel() + repositories = scm.repo_scan(scm.repos_path) repo2db_mapper(repositories, remove_obsolete=False) diff --git a/rhodecode/templates/admin/settings/settings_vcs.mako b/rhodecode/templates/admin/settings/settings_vcs.mako --- a/rhodecode/templates/admin/settings/settings_vcs.mako +++ b/rhodecode/templates/admin/settings/settings_vcs.mako @@ -6,8 +6,7 @@ suffix='', svn_tag_patterns=c.svn_tag_patterns, svn_branch_patterns=c.svn_branch_patterns, - display_globals=True, - allow_repo_location_change=c.visual.allow_repo_location_change + display_globals=True )}
${h.submit('save',_('Save settings'),class_="btn")} diff --git a/rhodecode/templates/base/vcs_settings.mako b/rhodecode/templates/base/vcs_settings.mako --- a/rhodecode/templates/base/vcs_settings.mako +++ b/rhodecode/templates/base/vcs_settings.mako @@ -3,7 +3,7 @@ ## <%namespace name="vcss" file="/base/vcssettings.mako"/> ## ${vcss.vcs_settings_fields()} -<%def name="vcs_settings_fields(suffix='', svn_branch_patterns=None, svn_tag_patterns=None, repo_type=None, display_globals=False, allow_repo_location_change=False, **kwargs)"> +<%def name="vcs_settings_fields(suffix='', svn_branch_patterns=None, svn_tag_patterns=None, repo_type=None, display_globals=False, **kwargs)"> % if display_globals:
@@ -23,34 +23,6 @@
% endif - % if display_globals: -
-
-

${_('Main Storage Location')}

-
-
-
-
- %if allow_repo_location_change: - ${h.text('paths_root_path',size=59,readonly="readonly", class_="disabled")} - -
- - %else: - ${_('Repository location change is disabled. You can enable this by changing the `allow_repo_location_change` inside .ini file.')} - ## form still requires this but we cannot internally change it anyway - ${h.hidden('paths_root_path',size=30,readonly="readonly", class_="disabled")} - %endif -
-
-
- ${_('Filesystem location where repositories should be stored. After changing this value a restart and rescan of the repository folder are required.')} -
-
-
- % endif - % if display_globals or repo_type in ['git', 'hg']:
diff --git a/rhodecode/tests/__init__.py b/rhodecode/tests/__init__.py --- a/rhodecode/tests/__init__.py +++ b/rhodecode/tests/__init__.py @@ -27,6 +27,7 @@ import urllib.parse import pytest +import rhodecode from rhodecode.model.db import User from rhodecode.lib import auth from rhodecode.lib import helpers as h @@ -53,8 +54,11 @@ log = logging.getLogger(__name__) # SOME GLOBALS FOR TESTS TEST_DIR = tempfile.gettempdir() +if os.getenv("RC_TEST"): + rhodecode.CONFIG['repo_store.path'] = TEST_DIR -TESTS_TMP_PATH = jn(TEST_DIR, 'rc_test_{}'.format(next(tempfile._RandomNameSequence()))) +# Once repo_store.path in test config will be changed update TEST_TMP_PATH accordingly +TESTS_TMP_PATH = jn(TEST_DIR, 'rc_test_a0hmmrva') TEST_USER_ADMIN_LOGIN = 'test_admin' TEST_USER_ADMIN_PASS = 'test12' TEST_USER_ADMIN_EMAIL = 'test_admin@mail.com' @@ -111,7 +115,7 @@ def get_new_dir(title): hex_str = sha1_safe(f'{os.getpid()} {time.time()}') name_parts.append(hex_str) name = '-'.join(name_parts) - path = os.path.join(TEST_DIR, name) + path = jn(TEST_DIR, name) return get_normalized_path(path) diff --git a/rhodecode/tests/lib/middleware/test_simplesvn.py b/rhodecode/tests/lib/middleware/test_simplesvn.py --- a/rhodecode/tests/lib/middleware/test_simplesvn.py +++ b/rhodecode/tests/lib/middleware/test_simplesvn.py @@ -23,16 +23,16 @@ import pytest from unittest.mock import patch, Mock, MagicMock from rhodecode.lib.middleware.simplesvn import SimpleSvn, SimpleSvnApp -from rhodecode.lib.utils import get_rhodecode_base_path +from rhodecode.lib.utils import get_rhodecode_repo_store_path from rhodecode.tests import SVN_REPO, TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS class TestSimpleSvn(object): @pytest.fixture(autouse=True) def simple_svn(self, baseapp, request_stub): - base_path = get_rhodecode_base_path() + base_path = get_rhodecode_repo_store_path() self.app = SimpleSvn( - config={'auth_ret_code': '', 'base_path': base_path}, + config={'auth_ret_code': '', 'repo_store.path': base_path}, registry=request_stub.registry) def test_get_config(self): @@ -126,7 +126,7 @@ class TestSimpleSvnApp(object): def setup_method(self, method): # note(marcink): this is hostname from docker compose used for testing... self.host = 'http://svn:8090' - base_path = get_rhodecode_base_path() + base_path = get_rhodecode_repo_store_path() self.app = SimpleSvnApp( config={'subversion_http_server_url': self.host, 'base_path': base_path}) diff --git a/rhodecode/tests/models/settings/test_vcs_settings.py b/rhodecode/tests/models/settings/test_vcs_settings.py --- a/rhodecode/tests/models/settings/test_vcs_settings.py +++ b/rhodecode/tests/models/settings/test_vcs_settings.py @@ -1004,21 +1004,6 @@ class TestGetSvnPatterns(object): settings_mock.assert_called_once_with(*args) -class TestGetReposLocation(object): - def test_returns_repos_location(self, repo_stub): - model = VcsSettingsModel() - - result_mock = mock.Mock() - result_mock.ui_value = '/tmp' - - with mock.patch.object(model, 'global_settings') as settings_mock: - settings_mock.get_ui_by_key.return_value = result_mock - result = model.get_repos_location() - - settings_mock.get_ui_by_key.assert_called_once_with('/') - assert result == '/tmp' - - class TestCreateOrUpdateRepoSettings(object): FORM_DATA = { 'inherit_global_settings': False, diff --git a/rhodecode/tests/models/test_repos.py b/rhodecode/tests/models/test_repos.py --- a/rhodecode/tests/models/test_repos.py +++ b/rhodecode/tests/models/test_repos.py @@ -121,20 +121,20 @@ class TestRepoModel(object): def test_create_filesystem_repo_installs_hooks(self, tmpdir, backend): repo = backend.create_repo() repo_name = repo.repo_name - model = RepoModel() - repo_location = tempfile.mkdtemp() - model.repos_path = repo_location - repo = model._create_filesystem_repo( - repo_name, backend.alias, repo_group='', clone_uri=None) + with mock.patch('rhodecode.model.repo.RepoModel.repos_path', + new_callable=mock.PropertyMock) as mocked_models_property: + mocked_models_property.return_value = tempfile.mkdtemp() + repo = RepoModel()._create_filesystem_repo( + repo_name, backend.alias, repo_group='', clone_uri=None) - hooks = { - 'svn': ('pre-commit', 'post-commit'), - 'git': ('pre-receive', 'post-receive'), - } - for hook in hooks[backend.alias]: - with open(os.path.join(repo.path, 'hooks', hook)) as f: - data = f.read() - assert 'RC_HOOK_VER' in data + hooks = { + 'svn': ('pre-commit', 'post-commit'), + 'git': ('pre-receive', 'post-receive'), + } + for hook in hooks[backend.alias]: + with open(os.path.join(repo.path, 'hooks', hook)) as f: + data = f.read() + assert 'RC_HOOK_VER' in data @pytest.mark.parametrize("use_global_config, repo_name_passed", [ (True, False), diff --git a/rhodecode/tests/rhodecode.ini b/rhodecode/tests/rhodecode.ini --- a/rhodecode/tests/rhodecode.ini +++ b/rhodecode/tests/rhodecode.ini @@ -207,9 +207,6 @@ auth_ret_code_detection = false ; codes don't break the transactions while 4XX codes do lock_ret_code = 423 -; allows to change the repository location in settings page -allow_repo_location_change = true - ; allows to setup custom hooks in settings page allow_custom_hooks_settings = true