merge: Resolved conflicts

docs/release-notes/release-notes-5.0.1.rst

@@ -0,0 +1,43 b''
+|RCE| 5.0.1 |RNS|
+-----------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2024-05-20
+
+
+New Features
+^^^^^^^^^^^^
+
+
+
+General
+^^^^^^^
+
+
+
+Security
+^^^^^^^^
+
+
+
+Performance
+^^^^^^^^^^^
+
+
+
+
+Fixes
+^^^^^
+
+- Fixed Celery serialization issues
+- Fixed Celery startup problems signaling
+- Fixed SVN hooks binary dir paths which in certain scenarios resulted in empty values forbidding hooks to execute
+- Fixed annotation bug for files without new lines or mixed newlines
+
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- RhodeCode 5.0.1 is unscheduled bugfix release to address some of the issues found during 4.X -> 5.X migration

docs/release-notes/release-notes-5.0.2.rst

@@ -0,0 +1,39 b''
+|RCE| 5.0.2 |RNS|
+-----------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2024-05-29
+
+
+New Features
+^^^^^^^^^^^^
+
+
+
+General
+^^^^^^^
+
+
+
+Security
+^^^^^^^^
+
+
+
+Performance
+^^^^^^^^^^^
+
+
+
+
+Fixes
+^^^^^
+
+- Fixed problems with saving branch permissions
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- RhodeCode 5.0.2 is unscheduled bugfix release to address some of the issues found during 4.X -> 5.X migration

docs/release-notes/release-notes-5.0.3.rst

@@ -0,0 +1,39 b''
+|RCE| 5.0.3 |RNS|
+-----------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2024-06-17
+
+
+New Features
+^^^^^^^^^^^^
+
+
+
+General
+^^^^^^^
+
+
+
+Security
+^^^^^^^^
+
+
+
+Performance
+^^^^^^^^^^^
+
+
+
+
+Fixes
+^^^^^
+
+- Fixed problems nested ldap groups
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- RhodeCode 5.0.3 is unscheduled bugfix release to address some of the issues found during 4.X -> 5.X migration

docs/release-notes/release-notes-5.1.0.rst

@@ -0,0 +1,59 b''
+|RCE| 5.1.0 |RNS|
+-----------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2024-07-18
+
+
+New Features
+^^^^^^^^^^^^
+
+- We've introduced 2FA for users. Now alongside the external auth 2fa support RhodeCode allows to enable 2FA for users
+  2FA options will be available for each user individually, or enforced via authentication plugins like ldap, or internal.
+- Email based log-in. RhodeCode now allows to log-in using email as well as username for main authentication type.
+- Ability to replace a file using web UI. Now one can replace an existing file from the web-ui.
+- GIT LFS Sync automation. Remote push/pull commands now can also sync GIT LFS objects.
+- Added ability to remove or close branches from the web ui
+- Added ability to delete a branch automatically after merging PR for git repositories
+- Added support for S3 based archive_cache based that allows storing cached archives in S3 compatible object store.
+
+
+General
+^^^^^^^
+
+- Upgraded all dependency libraries to their latest available versions
+- Repository storage is no longer controlled via DB settings, but .ini file. This allows easier automated deployments.
+- Bumped mercurial to 6.7.4
+- Mercurial: enable httppostarguments for better support of large repositories with lots of heads.
+- Added explicit db-migrate step to update hooks for 5.X release.
+
+
+Security
+^^^^^^^^
+
+
+
+Performance
+^^^^^^^^^^^
+
+- Introduced a full rewrite of ssh backend for performance. The result is 2-5x speed improvement for operation with ssh.
+  enable new ssh wrapper by setting: `ssh.wrapper_cmd = /home/rhodecode/venv/bin/rc-ssh-wrapper-v2`
+- Introduced a new hooks subsystem that is more scalable and faster, enable it by settings: `vcs.hooks.protocol = celery`
+
+
+Fixes
+^^^^^
+
+- Archives: Zip archive download breaks when a gitmodules file is present
+- Branch permissions: fixed bug preventing to specify own rules from 4.X install
+- SVN: refactored svn events, thus fixing support for it in dockerized env
+- Fixed empty server url in PR link after push from cli
+
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- RhodeCode 5.1.0 is a mayor feature release after big 5.0.0 python3 migration. Happy to ship a first time feature
+  rich release

rhodecode/api/tests/test_service_api.py

@@ -0,0 +1,55 b''
+
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import pytest
+
+from rhodecode.api.tests.utils import (
+    build_data, api_call)
+
+
[e-mail]("app")
+class TestServiceApi:
+
+    def test_service_api_with_wrong_secret(self):
+        id, payload = build_data("wrong_api_key", 'service_get_repo_name_by_id')
+        response = api_call(self.app, payload)
+
+        assert 'Invalid API KEY' == response.json['error']
+
+    def test_service_api_with_legit_secret(self):
+        id, payload = build_data(self.app.app.config.get_settings()['app.service_api.token'],
+                                 'service_get_repo_name_by_id', repo_id='1')
+        response = api_call(self.app, payload)
+        assert not response.json['error']
+
+    def test_service_api_not_a_part_of_public_api_suggestions(self):
+        id, payload = build_data("secret", 'some_random_guess_method')
+        response = api_call(self.app, payload)
+        assert 'service_' not in response.json['error']
+
+    def test_service_get_data_for_ssh_wrapper_output(self):
+        id, payload = build_data(
+            self.app.app.config.get_settings()['app.service_api.token'],
+            'service_get_data_for_ssh_wrapper',
+            user_id=1,
+            repo_name='vcs_test_git')
+        response = api_call(self.app, payload)
+
+        assert ['branch_permissions', 'repo_permissions', 'repos_path', 'user_id', 'username']\
+               == list(response.json['result'].keys())

rhodecode/api/views/service_api.py

@@ -0,0 +1,125 b''
+# Copyright (C) 2011-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import logging
+import datetime
+from collections import defaultdict
+
+from sqlalchemy import Table
+from rhodecode.api import jsonrpc_method, SERVICE_API_IDENTIFIER
+
+
+log = logging.getLogger(__name__)
+
+
+@jsonrpc_method()
+def service_get_data_for_ssh_wrapper(request, apiuser, user_id, repo_name, key_id=None):
+    from rhodecode.model.db import User
+    from rhodecode.model.scm import ScmModel
+    from rhodecode.model.meta import raw_query_executor, Base
+
+    if key_id:
+        table = Table('user_ssh_keys', Base.metadata, autoload=False)
+        atime = datetime.datetime.utcnow()
+        stmt = (
+            table.update()
+                .where(table.c.ssh_key_id == key_id)
+                .values(accessed_on=atime)
+        )
+
+        res_count = None
+        with raw_query_executor() as session:
+            result = session.execute(stmt)
+            if result.rowcount:
+                res_count = result.rowcount
+
+        if res_count:
+            log.debug(f'Update key id:{key_id} access time')
+    db_user = User.get(user_id)
+    if not db_user:
+        return None
+    auth_user = db_user.AuthUser()
+
+    return {
+        'user_id': db_user.user_id,
+        'username': db_user.username,
+        'repo_permissions': auth_user.permissions['repositories'],
+        "branch_permissions": auth_user.get_branch_permissions(repo_name),
+        "repos_path": ScmModel().repos_path
+    }
+
+
+@jsonrpc_method()
+def service_get_repo_name_by_id(request, apiuser, repo_id):
+    from rhodecode.model.repo import RepoModel
+    by_id_match = RepoModel().get_repo_by_id(repo_id)
+    if by_id_match:
+        repo_name = by_id_match.repo_name
+        return {
+            'repo_name': repo_name
+        }
+    return None
+
+
+@jsonrpc_method()
+def service_mark_for_invalidation(request, apiuser, repo_name):
+    from rhodecode.model.scm import ScmModel
+    ScmModel().mark_for_invalidation(repo_name)
+    return {'msg': "Applied"}
+
+
+@jsonrpc_method()
+def service_config_to_hgrc(request, apiuser, cli_flags, repo_name):
+    from rhodecode.model.db import RhodeCodeUi
+    from rhodecode.model.settings import VcsSettingsModel
+
+    ui_sections = defaultdict(list)
+    ui = VcsSettingsModel(repo=repo_name).get_ui_settings(section=None, key=None)
+
+    default_hooks = [
+        ('pretxnchangegroup.ssh_auth', 'python:vcsserver.hooks.pre_push_ssh_auth'),
+        ('pretxnchangegroup.ssh', 'python:vcsserver.hooks.pre_push_ssh'),
+        ('changegroup.ssh', 'python:vcsserver.hooks.post_push_ssh'),
+
+        ('preoutgoing.ssh', 'python:vcsserver.hooks.pre_pull_ssh'),
+        ('outgoing.ssh', 'python:vcsserver.hooks.post_pull_ssh'),
+    ]
+
+    for k, v in default_hooks:
+        ui_sections['hooks'].append((k, v))
+
+    for entry in ui:
+        if not entry.active:
+            continue
+        sec = entry.section
+        key = entry.key
+
+        if sec in cli_flags:
+            # we want only custom hooks, so we skip builtins
+            if sec == 'hooks' and key in RhodeCodeUi.HOOKS_BUILTIN:
+                continue
+
+            ui_sections[sec].append([key, entry.value])
+
+    flags = []
+    for _sec, key_val in ui_sections.items():
+        flags.append(' ')
+        flags.append(f'[{_sec}]')
+        for key, val in key_val:
+            flags.append(f'{key}= {val}')
+    return {'flags': flags}

rhodecode/apps/login/tests/test_2fa.py

@@ -0,0 +1,67 b''
+import pytest
+import mock
+
+from rhodecode.lib.type_utils import AttributeDict
+from rhodecode.model.meta import Session
+from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.routes import route_path
+from rhodecode.model.settings import SettingsModel
+
+fixture = Fixture()
+
+
[e-mail]('app')
+class Test2FA(object):
+    @classmethod
+    def setup_class(cls):
+        cls.password = 'valid-one'
+
+    def test_redirect_to_2fa_setup_if_enabled_for_user(self, user_util):
+        user = user_util.create_user(password=self.password)
+        user.has_enabled_2fa = True
+        self.app.post(
+            route_path('login'),
+            {'username': user.username,
+             'password': self.password})
+
+        response = self.app.get('/')
+        assert response.status_code == 302
+        assert response.location.endswith(route_path('setup_2fa'))
+
+    def test_redirect_to_2fa_check_if_2fa_configured(self, user_util):
+        user = user_util.create_user(password=self.password)
+        user.has_enabled_2fa = True
+        user.init_secret_2fa()
+        Session().add(user)
+        Session().commit()
+        self.app.post(
+            route_path('login'),
+            {'username': user.username,
+             'password': self.password})
+        response = self.app.get('/')
+        assert response.status_code == 302
+        assert response.location.endswith(route_path('check_2fa'))
+
+    def test_2fa_recovery_codes_works_only_once(self, user_util):
+        user = user_util.create_user(password=self.password)
+        user.has_enabled_2fa = True
+        user.init_secret_2fa()
+        recovery_code_to_check = user.init_2fa_recovery_codes()[0]
+        Session().add(user)
+        Session().commit()
+        self.app.post(
+            route_path('login'),
+            {'username': user.username,
+             'password': self.password})
+        response = self.app.post(route_path('check_2fa'), {'totp': recovery_code_to_check})
+        assert response.status_code == 302
+        response = self.app.post(route_path('check_2fa'), {'totp': recovery_code_to_check})
+        response.mustcontain('Code is invalid. Try again!')
+
+    def test_2fa_state_when_forced_by_admin(self, user_util):
+        user = user_util.create_user(password=self.password)
+        user.has_enabled_2fa = False
+        with mock.patch.object(
+                SettingsModel, 'get_setting_by_name', lambda *a, **kw: AttributeDict(app_settings_value=True)):
+
+            assert user.has_enabled_2fa

.bumpversion.cfg

@@ -1,5 +1,5 b''
 [bumpversion]
-current_version = 5.0.3
+current_version = 5.1.0
 message = release: Bump version {current_version} to {new_version}
 
 [bumpversion:file:rhodecode/VERSION]

Makefile

@@ -1,192 +1,171 b''
 # required for pushd to work..
 SHELL = /bin/bash
 
 
 # set by: PATH_TO_OUTDATED_PACKAGES=/some/path/outdated_packages.py
 OUTDATED_PACKAGES = ${PATH_TO_OUTDATED_PACKAGES}
 
 .PHONY: clean
 ## Cleanup compiled and cache py files
 clean:
 	make test-clean
 	find . -type f \( -iname '*.c' -o -iname '*.pyc' -o -iname '*.so' -o -iname '*.orig' \) -exec rm '{}' ';'
 	find . -type d -name "build" -prune -exec rm -rf '{}' ';'
 
 
 .PHONY: test
 ## run test-clean and tests
 test:
 	make test-clean
 	make test-only
 
 
 .PHONY: test-clean
 ## run test-clean and tests
 test-clean:
 	rm -rf coverage.xml htmlcov junit.xml pylint.log result
 	find . -type d -name "__pycache__" -prune -exec rm -rf '{}' ';'
 	find . -type f \( -iname '.coverage.*' \) -exec rm '{}' ';'
 
 
 .PHONY: test-only
 ## Run tests only without cleanup
 test-only:
 	PYTHONHASHSEED=random \
 	py.test -x -vv -r xw -p no:sugar \
 	--cov-report=term-missing --cov-report=html \
 	--cov=rhodecode rhodecode
 
 
-.PHONY: test-only-mysql
-## run tests against mysql
-test-only-mysql:
-	PYTHONHASHSEED=random \
-	py.test -x -vv -r xw -p no:sugar \
-	--cov-report=term-missing --cov-report=html \
-    --ini-config-override='{"app:main": {"sqlalchemy.db1.url": "mysql://root:qweqwe@localhost/rhodecode_test?charset=utf8"}}' \
-    --cov=rhodecode rhodecode
-
-
-.PHONY: test-only-postgres
-## run tests against postgres
-test-only-postgres:
-	PYTHONHASHSEED=random \
-	py.test -x -vv -r xw -p no:sugar \
-	--cov-report=term-missing --cov-report=html \
-    --ini-config-override='{"app:main": {"sqlalchemy.db1.url": "postgresql://postgres:qweqwe@localhost/rhodecode_test"}}' \
-    --cov=rhodecode rhodecode
-
-.PHONY: ruff-check
-## run a ruff analysis
-ruff-check:
-	ruff check --ignore F401 --ignore I001 --ignore E402 --ignore E501 --ignore F841 --exclude rhodecode/lib/dbmigrate --exclude .eggs  --exclude .dev .
-
-
 .PHONY: docs
 ## build docs
 docs:
 	(cd docs; docker run --rm -v $(PWD):/project --workdir=/project/docs sphinx-doc-build-rc make clean html SPHINXOPTS="-W")
 
 
 .PHONY: docs-clean
 ## Cleanup docs
 docs-clean:
 	(cd docs; docker run --rm -v $(PWD):/project --workdir=/project/docs sphinx-doc-build-rc make clean)
 
 
 .PHONY: docs-cleanup
 ## Cleanup docs
 docs-cleanup:
 	(cd docs; docker run --rm -v $(PWD):/project --workdir=/project/docs sphinx-doc-build-rc make cleanup)
 
 
 .PHONY: web-build
 ## Build JS packages static/js
 web-build:
 	docker run -it --rm -v $(PWD):/project --workdir=/project rhodecode/static-files-build:16 -c "npm install && /project/node_modules/.bin/grunt"
 	# run static file check
 	./rhodecode/tests/scripts/static-file-check.sh rhodecode/public/
 	rm -rf node_modules
 
+.PHONY: ruff-check
+## run a ruff analysis
+ruff-check:
+	ruff check --ignore F401 --ignore I001 --ignore E402 --ignore E501 --ignore F841 --exclude rhodecode/lib/dbmigrate --exclude .eggs  --exclude .dev .
 
 .PHONY: pip-packages
 ## Show outdated packages
 pip-packages:
 	python ${OUTDATED_PACKAGES}
 
 
 .PHONY: build
 ## Build sdist/egg
 build:
 	python -m build
 
 
 .PHONY: dev-sh
 ## make dev-sh
 dev-sh:
 	sudo echo "deb [trusted=yes] https://apt.fury.io/rsteube/ /" | sudo tee -a "/etc/apt/sources.list.d/fury.list"
 	sudo apt-get update
 	sudo apt-get install -y zsh carapace-bin
 	rm -rf /home/rhodecode/.oh-my-zsh
 	curl https://raw.githubusercontent.com/robbyrussell/oh-my-zsh/master/tools/install.sh | sh
-	echo "source <(carapace _carapace)" > /home/rhodecode/.zsrc
-	PROMPT='%(?.%F{green}√.%F{red}?%?)%f %B%F{240}%1~%f%b %# ' zsh
+	@echo "source <(carapace _carapace)" > /home/rhodecode/.zsrc
+	@echo "${RC_DEV_CMD_HELP}"
+	@PROMPT='%(?.%F{green}√.%F{red}?%?)%f %B%F{240}%1~%f%b %# ' zsh
 
 
 .PHONY: dev-cleanup
 ## Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
 dev-cleanup:
 	pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
 	rm -rf /tmp/*
 
 
 .PHONY: dev-env
 ## make dev-env based on the requirements files and install develop of packages
+## Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
 dev-env:
+	sudo -u root chown rhodecode:rhodecode /home/rhodecode/.cache/pip/
 	pip install build virtualenv
 	pushd ../rhodecode-vcsserver/ && make dev-env && popd
 	pip wheel --wheel-dir=/home/rhodecode/.cache/pip/wheels -r requirements.txt -r requirements_rc_tools.txt -r requirements_test.txt -r requirements_debug.txt
 	pip install --no-index --find-links=/home/rhodecode/.cache/pip/wheels -r requirements.txt -r requirements_rc_tools.txt -r requirements_test.txt -r requirements_debug.txt
 	pip install -e .
 
 
 .PHONY: sh
 ## shortcut for make dev-sh dev-env
 sh:
 	make dev-env
 	make dev-sh
 
 
-.PHONY: dev-srv
-## run develop server instance, docker exec -it $(docker ps -q --filter 'name=dev-enterprise-ce') /bin/bash
-dev-srv:
-	pserve --reload .dev/dev.ini
+## Allows changes of workers e.g make dev-srv-g workers=2
+workers?=1
 
-
-.PHONY: dev-srv-g
-## run gunicorn multi process workers
-dev-srv-g:
-	gunicorn --paste .dev/dev.ini --bind=0.0.0.0:10020 --config=.dev/gunicorn_config.py --timeout=120 --reload
+.PHONY: dev-srv
+## run gunicorn web server with reloader, use workers=N to set multiworker mode
+dev-srv:
+	gunicorn --paste=.dev/dev.ini --bind=0.0.0.0:10020 --config=.dev/gunicorn_config.py --timeout=120 --reload --workers=$(workers)
 
 
 # Default command on calling make
 .DEFAULT_GOAL := show-help
 
 .PHONY: show-help
 show-help:
 	@echo "$$(tput bold)Available rules:$$(tput sgr0)"
 	@echo
 	@sed -n -e "/^## / { \
 		h; \
 		s/.*//; \
 		:doc" \
 		-e "H; \
 		n; \
 		s/^## //; \
 		t doc" \
 		-e "s/:.*//; \
 		G; \
 		s/\\n## /---/; \
 		s/\\n/ /g; \
 		p; \
 	}" ${MAKEFILE_LIST} \
 	| LC_ALL='C' sort --ignore-case \
 	| awk -F '---' \
 		-v ncol=$$(tput cols) \
 		-v indent=19 \
 		-v col_on="$$(tput setaf 6)" \
 		-v col_off="$$(tput sgr0)" \
 	'{ \
 		printf "%s%*s%s ", col_on, -indent, $$1, col_off; \
 		n = split($$2, words, " "); \
 		line_length = ncol - indent; \
 		for (i = 1; i <= n; i++) { \
 			line_length -= length(words[i]) + 1; \
 			if (line_length <= 0) { \
 				line_length = ncol - indent - length(words[i]) - 1; \
 				printf "\n%*s ", -indent, " "; \
 			} \
 			printf "%s ", words[i]; \
 		} \
 		printf "\n"; \
 	}'

configs/development.ini

@@ -1,865 +1,856 b''
 
 ; #########################################
 ; RHODECODE COMMUNITY EDITION CONFIGURATION
 ; #########################################
 
 [DEFAULT]
 ; Debug flag sets all loggers to debug, and enables request tracking
 debug = true
 
 ; ########################################################################
 ; EMAIL CONFIGURATION
 ; These settings will be used by the RhodeCode mailing system
 ; ########################################################################
 
 ; prefix all emails subjects with given prefix, helps filtering out emails
 #email_prefix = [RhodeCode]
 
 ; email FROM address all mails will be sent
 #app_email_from = rhodecode-noreply@localhost
 
 #smtp_server = mail.server.com
 #smtp_username =
 #smtp_password =
 #smtp_port =
 #smtp_use_tls = false
 #smtp_use_ssl = true
 
 [server:main]
 ; COMMON HOST/IP CONFIG, This applies mostly to develop setup,
 ; Host port for gunicorn are controlled by gunicorn_conf.py
 host = 127.0.0.1
 port = 10020
 
-; ##################################################
-; WAITRESS WSGI SERVER - Recommended for Development
-; ##################################################
-
-; use server type
-use = egg:waitress#main
-
-; number of worker threads
-threads = 5
-
-; MAX BODY SIZE 100GB
-max_request_body_size = 107374182400
-
-; Use poll instead of select, fixes file descriptors limits problems.
-; May not work on old windows systems.
-asyncore_use_poll = true
-
 
 ; ###########################
 ; GUNICORN APPLICATION SERVER
 ; ###########################
 
-; run with gunicorn --paste rhodecode.ini --config gunicorn_conf.py
+; run with gunicorn  --config gunicorn_conf.py --paste rhodecode.ini
 
 ; Module to use, this setting shouldn't be changed
-#use = egg:gunicorn#main
+use = egg:gunicorn#main
 
 ; Prefix middleware for RhodeCode.
 ; recommended when using proxy setup.
 ; allows to set RhodeCode under a prefix in server.
 ; eg https://server.com/custom_prefix. Enable `filter-with =` option below as well.
 ; And set your prefix like: `prefix = /custom_prefix`
 ; be sure to also set beaker.session.cookie_path = /custom_prefix if you need
 ; to make your cookies only work on prefix url
 [filter:proxy-prefix]
 use = egg:PasteDeploy#prefix
 prefix = /
 
 [app:main]
 ; The %(here)s variable will be replaced with the absolute path of parent directory
 ; of this file
 ; Each option in the app:main can be override by an environmental variable
 ;
 ;To override an option:
 ;
 ;RC_<KeyName>
 ;Everything should be uppercase, . and - should be replaced by _.
 ;For example, if you have these configuration settings:
 ;rc_cache.repo_object.backend = foo
 ;can be overridden by
 ;export RC_CACHE_REPO_OBJECT_BACKEND=foo
 
 use = egg:rhodecode-enterprise-ce
 
 ; enable proxy prefix middleware, defined above
 #filter-with = proxy-prefix
 
 ; #############
 ; DEBUG OPTIONS
 ; #############
 
 pyramid.reload_templates = true
 
 # During development the we want to have the debug toolbar enabled
 pyramid.includes =
     pyramid_debugtoolbar
 
 debugtoolbar.hosts = 0.0.0.0/0
 debugtoolbar.exclude_prefixes =
     /css
     /fonts
     /images
     /js
 
 ## RHODECODE PLUGINS ##
 rhodecode.includes =
     rhodecode.api
 
 
 # api prefix url
 rhodecode.api.url = /_admin/api
 
 ; enable debug style page
 debug_style = true
 
 ; #################
 ; END DEBUG OPTIONS
 ; #################
 
 ; encryption key used to encrypt social plugin tokens,
 ; remote_urls with credentials etc, if not set it defaults to
 ; `beaker.session.secret`
 #rhodecode.encrypted_values.secret =
 
 ; decryption strict mode (enabled by default). It controls if decryption raises
 ; `SignatureVerificationError` in case of wrong key, or damaged encryption data.
 #rhodecode.encrypted_values.strict = false
 
 ; Pick algorithm for encryption. Either fernet (more secure) or aes (default)
 ; fernet is safer, and we strongly recommend switching to it.
 ; Due to backward compatibility aes is used as default.
 #rhodecode.encrypted_values.algorithm = fernet
 
 ; Return gzipped responses from RhodeCode (static files/application)
 gzip_responses = false
 
 ; Auto-generate javascript routes file on startup
 generate_js_files = false
 
 ; System global default language.
 ; All available languages: en (default), be, de, es, fr, it, ja, pl, pt, ru, zh
 lang = en
 
 ; Perform a full repository scan and import on each server start.
 ; Settings this to true could lead to very long startup time.
 startup.import_repos = false
 
 ; URL at which the application is running. This is used for Bootstrapping
 ; requests in context when no web request is available. Used in ishell, or
 ; SSH calls. Set this for events to receive proper url for SSH calls.
 app.base_url = http://rhodecode.local
 
+; Host at which the Service API is running.
+app.service_api.host = http://rhodecode.local:10020
+
+; Secret for Service API authentication.
+app.service_api.token =
+
 ; Unique application ID. Should be a random unique string for security.
 app_instance_uuid = rc-production
 
 ; Cut off limit for large diffs (size in bytes). If overall diff size on
 ; commit, or pull request exceeds this limit this diff will be displayed
 ; partially. E.g 512000 == 512Kb
 cut_off_limit_diff = 512000
 
 ; Cut off limit for large files inside diffs (size in bytes). Each individual
 ; file inside diff which exceeds this limit will be displayed partially.
 ;  E.g 128000 == 128Kb
 cut_off_limit_file = 128000
 
 ; Use cached version of vcs repositories everywhere. Recommended to be `true`
 vcs_full_cache = true
 
 ; Force https in RhodeCode, fixes https redirects, assumes it's always https.
 ; Normally this is controlled by proper flags sent from http server such as Nginx or Apache
 force_https = false
 
 ; use Strict-Transport-Security headers
 use_htsts = false
 
 ; Set to true if your repos are exposed using the dumb protocol
 git_update_server_info = false
 
 ; RSS/ATOM feed options
 rss_cut_off_limit = 256000
 rss_items_per_page = 10
 rss_include_diff = false
 
 ; gist URL alias, used to create nicer urls for gist. This should be an
 ; url that does rewrites to _admin/gists/{gistid}.
 ; example: http://gist.rhodecode.org/{gistid}. Empty means use the internal
 ; RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/{gistid}
 gist_alias_url =
 
 ; List of views (using glob pattern syntax) that AUTH TOKENS could be
 ; used for access.
 ; Adding ?auth_token=TOKEN_HASH to the url authenticates this request as if it
 ; came from the the logged in user who own this authentication token.
 ; Additionally @TOKEN syntax can be used to bound the view to specific
 ; authentication token. Such view would be only accessible when used together
 ; with this authentication token
 ; list of all views can be found under `/_admin/permissions/auth_token_access`
 ; The list should be "," separated and on a single line.
 ; Most common views to enable:
 
 #    RepoCommitsView:repo_commit_download
 #    RepoCommitsView:repo_commit_patch
 #    RepoCommitsView:repo_commit_raw
 #    RepoCommitsView:repo_commit_raw@TOKEN
 #    RepoFilesView:repo_files_diff
 #    RepoFilesView:repo_archivefile
 #    RepoFilesView:repo_file_raw
 #    GistView:*
 api_access_controllers_whitelist =
 
 ; Default encoding used to convert from and to unicode
 ; can be also a comma separated list of encoding in case of mixed encodings
 default_encoding = UTF-8
 
 ; instance-id prefix
 ; a prefix key for this instance used for cache invalidation when running
 ; multiple instances of RhodeCode, make sure it's globally unique for
 ; all running RhodeCode instances. Leave empty if you don't use it
 instance_id =
 
 ; Fallback authentication plugin. Set this to a plugin ID to force the usage
 ; of an authentication plugin also if it is disabled by it's settings.
 ; This could be useful if you are unable to log in to the system due to broken
 ; authentication settings. Then you can enable e.g. the internal RhodeCode auth
 ; module to log in again and fix the settings.
 ; Available builtin plugin IDs (hash is part of the ID):
 ; egg:rhodecode-enterprise-ce#rhodecode
 ; egg:rhodecode-enterprise-ce#pam
 ; egg:rhodecode-enterprise-ce#ldap
 ; egg:rhodecode-enterprise-ce#jasig_cas
 ; egg:rhodecode-enterprise-ce#headers
 ; egg:rhodecode-enterprise-ce#crowd
 
 #rhodecode.auth_plugin_fallback = egg:rhodecode-enterprise-ce#rhodecode
 
 ; Flag to control loading of legacy plugins in py:/path format
 auth_plugin.import_legacy_plugins = true
 
 ; alternative return HTTP header for failed authentication. Default HTTP
 ; response is 401 HTTPUnauthorized. Currently HG clients have troubles with
 ; handling that causing a series of failed authentication calls.
 ; Set this variable to 403 to return HTTPForbidden, or any other HTTP code
 ; This will be served instead of default 401 on bad authentication
 auth_ret_code =
 
 ; use special detection method when serving auth_ret_code, instead of serving
 ; ret_code directly, use 401 initially (Which triggers credentials prompt)
 ; and then serve auth_ret_code to clients
 auth_ret_code_detection = false
 
 ; locking return code. When repository is locked return this HTTP code. 2XX
 ; codes don't break the transactions while 4XX codes do
 lock_ret_code = 423
 
-; allows to change the repository location in settings page
-allow_repo_location_change = true
+; Filesystem location were repositories should be stored
+repo_store.path = /var/opt/rhodecode_repo_store
 
 ; allows to setup custom hooks in settings page
 allow_custom_hooks_settings = true
 
 ; Generated license token required for EE edition license.
 ; New generated token value can be found in Admin > settings > license page.
 license_token =
 
 ; This flag hides sensitive information on the license page such as token, and license data
 license.hide_license_info = false
 
 ; supervisor connection uri, for managing supervisor and logs.
 supervisor.uri =
 
 ; supervisord group name/id we only want this RC instance to handle
 supervisor.group_id = dev
 
 ; Display extended labs settings
 labs_settings_active = true
 
 ; Custom exception store path, defaults to TMPDIR
 ; This is used to store exception from RhodeCode in shared directory
 #exception_tracker.store_path =
 
 ; Send email with exception details when it happens
 #exception_tracker.send_email = false
 
 ; Comma separated list of recipients for exception emails,
 ; e.g [e-mail],[e-mail]
 ; Can be left empty, then emails will be sent to ALL super-admins
 #exception_tracker.send_email_recipients =
 
 ; optional prefix to Add to email Subject
 #exception_tracker.email_prefix = [RHODECODE ERROR]
 
 ; File store configuration. This is used to store and serve uploaded files
 file_store.enabled = true
 
 ; Storage backend, available options are: local
 file_store.backend = local
 
-; path to store the uploaded binaries
-file_store.storage_path = %(here)s/data/file_store
+; path to store the uploaded binaries and artifacts
+file_store.storage_path = /var/opt/rhodecode_data/file_store
+
+
+; Redis url to acquire/check generation of archives locks
+archive_cache.locking.url = redis://redis:6379/1
+
+; Storage backend, only 'filesystem' and 'objectstore' are available now
+archive_cache.backend.type = filesystem
+
+; url for s3 compatible storage that allows to upload artifacts
+; e.g http://minio:9000
+archive_cache.objectstore.url = http://s3-minio:9000
+
+; key for s3 auth
+archive_cache.objectstore.key = key
+
+; secret for s3 auth
+archive_cache.objectstore.secret = secret
 
-; Uncomment and set this path to control settings for archive download cache.
+;region for s3 storage
+archive_cache.objectstore.region = eu-central-1
+
+; number of sharded buckets to create to distribute archives across
+; default is 8 shards
+archive_cache.objectstore.bucket_shards = 8
+
+; a top-level bucket to put all other shards in
+; objects will be stored in rhodecode-archive-cache/shard-N based on the bucket_shards number
+archive_cache.objectstore.bucket = rhodecode-archive-cache
+
+; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
+archive_cache.objectstore.retry = false
+
+; number of seconds to wait for next try using retry
+archive_cache.objectstore.retry_backoff = 1
+
+; how many tries do do a retry fetch from this backend
+archive_cache.objectstore.retry_attempts = 10
+
+; Default is $cache_dir/archive_cache if not set
 ; Generated repo archives will be cached at this location
 ; and served from the cache during subsequent requests for the same archive of
 ; the repository. This path is important to be shared across filesystems and with
 ; RhodeCode and vcsserver
-
-; Default is $cache_dir/archive_cache if not set
-archive_cache.store_dir = %(here)s/data/archive_cache
+archive_cache.filesystem.store_dir = /var/opt/rhodecode_data/archive_cache
 
 ; The limit in GB sets how much data we cache before recycling last used, defaults to 10 gb
-archive_cache.cache_size_gb = 10
+archive_cache.filesystem.cache_size_gb = 1
+
+; Eviction policy used to clear out after cache_size_gb limit is reached
+archive_cache.filesystem.eviction_policy = least-recently-stored
 
 ; By default cache uses sharding technique, this specifies how many shards are there
-archive_cache.cache_shards = 10
+; default is 8 shards
+archive_cache.filesystem.cache_shards = 8
+
+; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
+archive_cache.filesystem.retry = false
+
+; number of seconds to wait for next try using retry
+archive_cache.filesystem.retry_backoff = 1
+
+; how many tries do do a retry fetch from this backend
+archive_cache.filesystem.retry_attempts = 10
+
 
 ; #############
 ; CELERY CONFIG
 ; #############
 
 ; manually run celery: /path/to/celery worker --task-events --beat --app rhodecode.lib.celerylib.loader --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler --loglevel DEBUG --ini /path/to/rhodecode.ini
 
-use_celery = false
+use_celery = true
 
 ; path to store schedule database
 #celerybeat-schedule.path =
 
 ; connection url to the message broker (default redis)
 celery.broker_url = redis://redis:6379/8
 
 ; results backend to get results for (default redis)
 celery.result_backend = redis://redis:6379/8
 
 ; rabbitmq example
 #celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost
 
 ; maximum tasks to execute before worker restart
 celery.max_tasks_per_child = 20
 
 ; tasks will never be sent to the queue, but executed locally instead.
 celery.task_always_eager = false
 
 ; #############
 ; DOGPILE CACHE
 ; #############
 
 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
-cache_dir = %(here)s/data
+cache_dir = /var/opt/rhodecode_data
 
 ; *********************************************
 ; `sql_cache_short` cache for heavy SQL queries
 ; Only supported backend is `memory_lru`
 ; *********************************************
 rc_cache.sql_cache_short.backend = dogpile.cache.rc.memory_lru
 rc_cache.sql_cache_short.expiration_time = 30
 
 
 ; *****************************************************
 ; `cache_repo_longterm` cache for repo object instances
 ; Only supported backend is `memory_lru`
 ; *****************************************************
 rc_cache.cache_repo_longterm.backend = dogpile.cache.rc.memory_lru
 ; by default we use 30 Days, cache is still invalidated on push
 rc_cache.cache_repo_longterm.expiration_time = 2592000
 ; max items in LRU cache, set to smaller number to save memory, and expire last used caches
 rc_cache.cache_repo_longterm.max_size = 10000
 
 
 ; *********************************************
 ; `cache_general` cache for general purpose use
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; *********************************************
 rc_cache.cache_general.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_general.expiration_time = 43200
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_general.arguments.filename = /tmp/cache_general_db
 
 ; alternative `cache_general` redis backend with distributed lock
 #rc_cache.cache_general.backend = dogpile.cache.rc.redis
 #rc_cache.cache_general.expiration_time = 300
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_general.arguments.redis_expiration_time = 7200
 
 #rc_cache.cache_general.arguments.host = localhost
 #rc_cache.cache_general.arguments.port = 6379
 #rc_cache.cache_general.arguments.db = 0
 #rc_cache.cache_general.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_general.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_general.arguments.lock_auto_renewal = true
 
 ; *************************************************
 ; `cache_perms` cache for permission tree, auth TTL
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; *************************************************
 rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_perms.expiration_time = 3600
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_perms.arguments.filename = /tmp/cache_perms_db
 
 ; alternative `cache_perms` redis backend with distributed lock
 #rc_cache.cache_perms.backend = dogpile.cache.rc.redis
 #rc_cache.cache_perms.expiration_time = 300
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_perms.arguments.redis_expiration_time = 7200
 
 #rc_cache.cache_perms.arguments.host = localhost
 #rc_cache.cache_perms.arguments.port = 6379
 #rc_cache.cache_perms.arguments.db = 0
 #rc_cache.cache_perms.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_perms.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_perms.arguments.lock_auto_renewal = true
 
 ; ***************************************************
 ; `cache_repo` cache for file tree, Readme, RSS FEEDS
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; ***************************************************
 rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_repo.expiration_time = 2592000
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_repo.arguments.filename = /tmp/cache_repo_db
 
 ; alternative `cache_repo` redis backend with distributed lock
 #rc_cache.cache_repo.backend = dogpile.cache.rc.redis
 #rc_cache.cache_repo.expiration_time = 2592000
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_repo.arguments.redis_expiration_time = 2678400
 
 #rc_cache.cache_repo.arguments.host = localhost
 #rc_cache.cache_repo.arguments.port = 6379
 #rc_cache.cache_repo.arguments.db = 1
 #rc_cache.cache_repo.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_repo.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_repo.arguments.lock_auto_renewal = true
 
 ; ##############
 ; BEAKER SESSION
 ; ##############
 
 ; beaker.session.type is type of storage options for the logged users sessions. Current allowed
 ; types are file, ext:redis, ext:database, ext:memcached
 ; Fastest ones are ext:redis and ext:database, DO NOT use memory type for session
-beaker.session.type = file
-beaker.session.data_dir = %(here)s/data/sessions
+#beaker.session.type = file
+#beaker.session.data_dir = %(here)s/data/sessions
 
 ; Redis based sessions
-#beaker.session.type = ext:redis
-#beaker.session.url = redis://127.0.0.1:6379/2
+beaker.session.type = ext:redis
+beaker.session.url = redis://redis:6379/2
 
 ; DB based session, fast, and allows easy management over logged in users
 #beaker.session.type = ext:database
 #beaker.session.table_name = db_session
 #beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode
 #beaker.session.sa.url = mysql://root:secret@127.0.0.1/rhodecode
 #beaker.session.sa.pool_recycle = 3600
 #beaker.session.sa.echo = false
 
 beaker.session.key = rhodecode
 beaker.session.secret = develop-rc-uytcxaz
-beaker.session.lock_dir = %(here)s/data/sessions/lock
+beaker.session.lock_dir = /data_ramdisk/lock
 
 ; Secure encrypted cookie. Requires AES and AES python libraries
 ; you must disable beaker.session.secret to use this
 #beaker.session.encrypt_key = key_for_encryption
 #beaker.session.validate_key = validation_key
 
 ; Sets session as invalid (also logging out user) if it haven not been
 ; accessed for given amount of time in seconds
 beaker.session.timeout = 2592000
 beaker.session.httponly = true
 
 ; Path to use for the cookie. Set to prefix if you use prefix middleware
 #beaker.session.cookie_path = /custom_prefix
 
 ; Set https secure cookie
 beaker.session.secure = false
 
 ; default cookie expiration time in seconds, set to `true` to set expire
 ; at browser close
 #beaker.session.cookie_expires = 3600
 
 ; #############################
 ; SEARCH INDEXING CONFIGURATION
 ; #############################
 
 ; Full text search indexer is available in rhodecode-tools under
 ; `rhodecode-tools index` command
 
 ; WHOOSH Backend, doesn't require additional services to run
 ; it works good with few dozen repos
 search.module = rhodecode.lib.index.whoosh
 search.location = %(here)s/data/index
 
 ; ####################
 ; CHANNELSTREAM CONFIG
 ; ####################
 
 ; channelstream enables persistent connections and live notification
 ; in the system. It's also used by the chat system
 
-channelstream.enabled = false
+channelstream.enabled = true
 
 ; server address for channelstream server on the backend
-channelstream.server = 127.0.0.1:9800
+channelstream.server = channelstream:9800
 
 ; location of the channelstream server from outside world
 ; use ws:// for http or wss:// for https. This address needs to be handled
 ; by external HTTP server such as Nginx or Apache
 ; see Nginx/Apache configuration examples in our docs
 channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream
-channelstream.secret = secret
-channelstream.history.location = %(here)s/channelstream_history
+channelstream.secret = ENV_GENERATED
+channelstream.history.location = /var/opt/rhodecode_data/channelstream_history
 
 ; Internal application path that Javascript uses to connect into.
 ; If you use proxy-prefix the prefix should be added before /_channelstream
 channelstream.proxy_path = /_channelstream
 
 
 ; ##############################
 ; MAIN RHODECODE DATABASE CONFIG
 ; ##############################
 
 #sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
 #sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 #sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode?charset=utf8
 ; pymysql is an alternative driver for MySQL, use in case of problems with default one
 #sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode
 
 sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
 
 ; see sqlalchemy docs for other advanced settings
 ; print the sql statements to output
 sqlalchemy.db1.echo = false
 
 ; recycle the connections after this amount of seconds
 sqlalchemy.db1.pool_recycle = 3600
 
 ; the number of connections to keep open inside the connection pool.
 ; 0 indicates no limit
 ; the general calculus with gevent is:
 ; if your system allows 500 concurrent greenlets (max_connections) that all do database access,
 ; then increase pool size + max overflow so that they add up to 500.
 #sqlalchemy.db1.pool_size = 5
 
 ; The number of connections to allow in connection pool "overflow", that is
 ; connections that can be opened above and beyond the pool_size setting,
 ; which defaults to five.
 #sqlalchemy.db1.max_overflow = 10
 
 ; Connection check ping, used to detect broken database connections
 ; could be enabled to better handle cases if MySQL has gone away errors
 #sqlalchemy.db1.ping_connection = true
 
 ; ##########
 ; VCS CONFIG
 ; ##########
 vcs.server.enable = true
-vcs.server = localhost:9900
+vcs.server = vcsserver:10010
 
 ; Web server connectivity protocol, responsible for web based VCS operations
 ; Available protocols are:
 ; `http` - use http-rpc backend (default)
 vcs.server.protocol = http
 
 ; Push/Pull operations protocol, available options are:
 ; `http` - use http-rpc backend (default)
 vcs.scm_app_implementation = http
 
 ; Push/Pull operations hooks protocol, available options are:
 ; `http` - use http-rpc backend (default)
+; `celery` - use celery based hooks
 vcs.hooks.protocol = http
 
 ; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be
 ; accessible via network.
 ; Use vcs.hooks.host = "*" to bind to current hostname (for Docker)
 vcs.hooks.host = *
 
 ; Start VCSServer with this instance as a subprocess, useful for development
 vcs.start_server = false
 
 ; List of enabled VCS backends, available options are:
 ; `hg`  - mercurial
 ; `git` - git
 ; `svn` - subversion
 vcs.backends = hg, git, svn
 
 ; Wait this number of seconds before killing connection to the vcsserver
 vcs.connection_timeout = 3600
 
-; Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
-; Set a numeric version for your current SVN e.g 1.8, or 1.12
-; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
-#vcs.svn.compatible_version = 1.8
-
 ; Cache flag to cache vcsserver remote calls locally
 ; It uses cache_region `cache_repo`
 vcs.methods.cache = true
 
 ; ####################################################
 ; Subversion proxy support (mod_dav_svn)
 ; Maps RhodeCode repo groups into SVN paths for Apache
 ; ####################################################
 
+; Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
+; Set a numeric version for your current SVN e.g 1.8, or 1.12
+; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
+#vcs.svn.compatible_version = 1.8
+
+; Redis connection settings for svn integrations logic
+; This connection string needs to be the same on ce and vcsserver
+vcs.svn.redis_conn = redis://redis:6379/0
+
+; Enable SVN proxy of requests over HTTP
+vcs.svn.proxy.enabled = true
+
+; host to connect to running SVN subsystem
+vcs.svn.proxy.host = http://svn:8090
+
 ; Enable or disable the config file generation.
-svn.proxy.generate_config = false
+svn.proxy.generate_config = true
 
 ; Generate config file with `SVNListParentPath` set to `On`.
 svn.proxy.list_parent_path = true
 
 ; Set location and file name of generated config file.
-svn.proxy.config_file_path = %(here)s/mod_dav_svn.conf
+svn.proxy.config_file_path = /etc/rhodecode/conf/svn/mod_dav_svn.conf
 
 ; alternative mod_dav config template. This needs to be a valid mako template
 ; Example template can be found in the source code:
 ; rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako
 #svn.proxy.config_template = ~/.rccontrol/enterprise-1/custom_svn_conf.mako
 
 ; Used as a prefix to the `Location` block in the generated config file.
 ; In most cases it should be set to `/`.
 svn.proxy.location_root = /
 
 ; Command to reload the mod dav svn configuration on change.
 ; Example: `/etc/init.d/apache2 reload` or /home/USER/apache_reload.sh
 ; Make sure user who runs RhodeCode process is allowed to reload Apache
 #svn.proxy.reload_cmd = /etc/init.d/apache2 reload
 
 ; If the timeout expires before the reload command finishes, the command will
 ; be killed. Setting it to zero means no timeout. Defaults to 10 seconds.
 #svn.proxy.reload_timeout = 10
 
 ; ####################
 ; SSH Support Settings
 ; ####################
 
 ; Defines if a custom authorized_keys file should be created and written on
 ; any change user ssh keys. Setting this to false also disables possibility
 ; of adding SSH keys by users from web interface. Super admins can still
 ; manage SSH Keys.
-ssh.generate_authorized_keyfile = false
+ssh.generate_authorized_keyfile = true
 
 ; Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding`
 # ssh.authorized_keys_ssh_opts =
 
 ; Path to the authorized_keys file where the generate entries are placed.
 ; It is possible to have multiple key files specified in `sshd_config` e.g.
 ; AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode
-ssh.authorized_keys_file_path = ~/.ssh/authorized_keys_rhodecode
+ssh.authorized_keys_file_path = /etc/rhodecode/conf/ssh/authorized_keys_rhodecode
 
 ; Command to execute the SSH wrapper. The binary is available in the
 ; RhodeCode installation directory.
-; e.g ~/.rccontrol/community-1/profile/bin/rc-ssh-wrapper
-ssh.wrapper_cmd = ~/.rccontrol/community-1/rc-ssh-wrapper
+; legacy: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+; new rewrite: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
+ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
 
 ; Allow shell when executing the ssh-wrapper command
 ssh.wrapper_cmd_allow_shell = false
 
 ; Enables logging, and detailed output send back to the client during SSH
 ; operations. Useful for debugging, shouldn't be used in production.
 ssh.enable_debug_logging = true
 
 ; Paths to binary executable, by default they are the names, but we can
 ; override them if we want to use a custom one
-ssh.executable.hg = ~/.rccontrol/vcsserver-1/profile/bin/hg
-ssh.executable.git = ~/.rccontrol/vcsserver-1/profile/bin/git
-ssh.executable.svn = ~/.rccontrol/vcsserver-1/profile/bin/svnserve
+ssh.executable.hg = /usr/local/bin/rhodecode_bin/vcs_bin/hg
+ssh.executable.git = /usr/local/bin/rhodecode_bin/vcs_bin/git
+ssh.executable.svn = /usr/local/bin/rhodecode_bin/vcs_bin/svnserve
 
 ; Enables SSH key generator web interface. Disabling this still allows users
 ; to add their own keys.
 ssh.enable_ui_key_generator = true
 
-
-; #################
-; APPENLIGHT CONFIG
-; #################
-
-; Appenlight is tailored to work with RhodeCode, see
-; http://appenlight.rhodecode.com for details how to obtain an account
-
-; Appenlight integration enabled
-#appenlight = false
-
-#appenlight.server_url = https://api.appenlight.com
-#appenlight.api_key = YOUR_API_KEY
-#appenlight.transport_config = https://api.appenlight.com?threaded=1&timeout=5
-
-; used for JS client
-#appenlight.api_public_key = YOUR_API_PUBLIC_KEY
-
-; TWEAK AMOUNT OF INFO SENT HERE
-
-; enables 404 error logging (default False)
-#appenlight.report_404 = false
-
-; time in seconds after request is considered being slow (default 1)
-#appenlight.slow_request_time = 1
-
-; record slow requests in application
-; (needs to be enabled for slow datastore recording and time tracking)
-#appenlight.slow_requests = true
-
-; enable hooking to application loggers
-#appenlight.logging = true
-
-; minimum log level for log capture
-#ppenlight.logging.level = WARNING
-
-; send logs only from erroneous/slow requests
-; (saves API quota for intensive logging)
-#appenlight.logging_on_error = false
-
-; list of additional keywords that should be grabbed from environ object
-; can be string with comma separated list of words in lowercase
-; (by default client will always send following info:
-; 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE' + all keys that
-; start with HTTP* this list be extended with additional keywords here
-#appenlight.environ_keys_whitelist =
-
-; list of keywords that should be blanked from request object
-; can be string with comma separated list of words in lowercase
-; (by default client will always blank keys that contain following words
-; 'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf'
-; this list be extended with additional keywords set here
-#appenlight.request_keys_blacklist =
-
-; list of namespaces that should be ignores when gathering log entries
-; can be string with comma separated list of namespaces
-; (by default the client ignores own entries: appenlight_client.client)
-#appenlight.log_namespace_blacklist =
-
 ; Statsd client config, this is used to send metrics to statsd
 ; We recommend setting statsd_exported and scrape them using Prometheus
 #statsd.enabled = false
 #statsd.statsd_host = 0.0.0.0
 #statsd.statsd_port = 8125
 #statsd.statsd_prefix =
 #statsd.statsd_ipv6 = false
 
 ; configure logging automatically at server startup set to false
 ; to use the below custom logging config.
 ; RC_LOGGING_FORMATTER
 ; RC_LOGGING_LEVEL
 ; env variables can control the settings for logging in case of autoconfigure
 
 #logging.autoconfigure = true
 
 ; specify your own custom logging config file to configure logging
 #logging.logging_conf_file = /path/to/custom_logging.ini
 
 ; Dummy marker to add new entries after.
 ; Add any custom entries below. Please don't remove this marker.
 custom.conf = 1
 
 
 ; #####################
 ; LOGGING CONFIGURATION
 ; #####################
 
 [loggers]
 keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
 
 [handlers]
 keys = console, console_sql
 
 [formatters]
 keys = generic, json, color_formatter, color_formatter_sql
 
 ; #######
 ; LOGGERS
 ; #######
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_sqlalchemy]
 level = INFO
 handlers = console_sql
 qualname = sqlalchemy.engine
 propagate = 0
 
 [logger_beaker]
 level = DEBUG
 handlers =
 qualname = beaker.container
 propagate = 1
 
 [logger_rhodecode]
 level = DEBUG
 handlers =
 qualname = rhodecode
 propagate = 1
 
 [logger_ssh_wrapper]
 level = DEBUG
 handlers =
 qualname = ssh_wrapper
 propagate = 1
 
 [logger_celery]
 level = DEBUG
 handlers =
 qualname = celery
 
 
 ; ########
 ; HANDLERS
 ; ########
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr, )
 level = DEBUG
 ; To enable JSON formatted logs replace 'generic/color_formatter' with 'json'
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 formatter = color_formatter
 
 [handler_console_sql]
 ; "level = DEBUG" logs SQL queries and results.
 ; "level = INFO" logs SQL queries.
 ; "level = WARN" logs neither.  (Recommended for production systems.)
 class = StreamHandler
 args = (sys.stderr, )
 level = WARN
 ; To enable JSON formatted logs replace 'generic/color_formatter_sql' with 'json'
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 formatter = color_formatter_sql
 
 ; ##########
 ; FORMATTERS
 ; ##########
 
 [formatter_generic]
 class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class = rhodecode.lib.logging_formatter.ColorFormatter
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter_sql]
 class = rhodecode.lib.logging_formatter.ColorFormatterSql
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_json]
 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
 class = rhodecode.lib._vendor.jsonlogger.JsonFormatter

configs/production.ini

@@ -1,816 +1,824 b''
 
 ; #########################################
 ; RHODECODE COMMUNITY EDITION CONFIGURATION
 ; #########################################
 
 [DEFAULT]
 ; Debug flag sets all loggers to debug, and enables request tracking
 debug = false
 
 ; ########################################################################
 ; EMAIL CONFIGURATION
 ; These settings will be used by the RhodeCode mailing system
 ; ########################################################################
 
 ; prefix all emails subjects with given prefix, helps filtering out emails
 #email_prefix = [RhodeCode]
 
 ; email FROM address all mails will be sent
 #app_email_from = rhodecode-noreply@localhost
 
 #smtp_server = mail.server.com
 #smtp_username =
 #smtp_password =
 #smtp_port =
 #smtp_use_tls = false
 #smtp_use_ssl = true
 
 [server:main]
 ; COMMON HOST/IP CONFIG, This applies mostly to develop setup,
 ; Host port for gunicorn are controlled by gunicorn_conf.py
 host = 127.0.0.1
 port = 10020
 
 
 ; ###########################
 ; GUNICORN APPLICATION SERVER
 ; ###########################
 
-; run with gunicorn --paste rhodecode.ini --config gunicorn_conf.py
+; run with gunicorn  --config gunicorn_conf.py --paste rhodecode.ini
 
 ; Module to use, this setting shouldn't be changed
 use = egg:gunicorn#main
 
 ; Prefix middleware for RhodeCode.
 ; recommended when using proxy setup.
 ; allows to set RhodeCode under a prefix in server.
 ; eg https://server.com/custom_prefix. Enable `filter-with =` option below as well.
 ; And set your prefix like: `prefix = /custom_prefix`
 ; be sure to also set beaker.session.cookie_path = /custom_prefix if you need
 ; to make your cookies only work on prefix url
 [filter:proxy-prefix]
 use = egg:PasteDeploy#prefix
 prefix = /
 
 [app:main]
 ; The %(here)s variable will be replaced with the absolute path of parent directory
 ; of this file
 ; Each option in the app:main can be override by an environmental variable
 ;
 ;To override an option:
 ;
 ;RC_<KeyName>
 ;Everything should be uppercase, . and - should be replaced by _.
 ;For example, if you have these configuration settings:
 ;rc_cache.repo_object.backend = foo
 ;can be overridden by
 ;export RC_CACHE_REPO_OBJECT_BACKEND=foo
 
 use = egg:rhodecode-enterprise-ce
 
 ; enable proxy prefix middleware, defined above
 #filter-with = proxy-prefix
 
 ; encryption key used to encrypt social plugin tokens,
 ; remote_urls with credentials etc, if not set it defaults to
 ; `beaker.session.secret`
 #rhodecode.encrypted_values.secret =
 
 ; decryption strict mode (enabled by default). It controls if decryption raises
 ; `SignatureVerificationError` in case of wrong key, or damaged encryption data.
 #rhodecode.encrypted_values.strict = false
 
 ; Pick algorithm for encryption. Either fernet (more secure) or aes (default)
 ; fernet is safer, and we strongly recommend switching to it.
 ; Due to backward compatibility aes is used as default.
 #rhodecode.encrypted_values.algorithm = fernet
 
 ; Return gzipped responses from RhodeCode (static files/application)
 gzip_responses = false
 
 ; Auto-generate javascript routes file on startup
 generate_js_files = false
 
 ; System global default language.
 ; All available languages: en (default), be, de, es, fr, it, ja, pl, pt, ru, zh
 lang = en
 
 ; Perform a full repository scan and import on each server start.
 ; Settings this to true could lead to very long startup time.
 startup.import_repos = false
 
 ; URL at which the application is running. This is used for Bootstrapping
 ; requests in context when no web request is available. Used in ishell, or
 ; SSH calls. Set this for events to receive proper url for SSH calls.
 app.base_url = http://rhodecode.local
 
+; Host at which the Service API is running.
+app.service_api.host = http://rhodecode.local:10020
+
+; Secret for Service API authentication.
+app.service_api.token =
+
 ; Unique application ID. Should be a random unique string for security.
 app_instance_uuid = rc-production
 
 ; Cut off limit for large diffs (size in bytes). If overall diff size on
 ; commit, or pull request exceeds this limit this diff will be displayed
 ; partially. E.g 512000 == 512Kb
 cut_off_limit_diff = 512000
 
 ; Cut off limit for large files inside diffs (size in bytes). Each individual
 ; file inside diff which exceeds this limit will be displayed partially.
 ;  E.g 128000 == 128Kb
 cut_off_limit_file = 128000
 
 ; Use cached version of vcs repositories everywhere. Recommended to be `true`
 vcs_full_cache = true
 
 ; Force https in RhodeCode, fixes https redirects, assumes it's always https.
 ; Normally this is controlled by proper flags sent from http server such as Nginx or Apache
 force_https = false
 
 ; use Strict-Transport-Security headers
 use_htsts = false
 
 ; Set to true if your repos are exposed using the dumb protocol
 git_update_server_info = false
 
 ; RSS/ATOM feed options
 rss_cut_off_limit = 256000
 rss_items_per_page = 10
 rss_include_diff = false
 
 ; gist URL alias, used to create nicer urls for gist. This should be an
 ; url that does rewrites to _admin/gists/{gistid}.
 ; example: http://gist.rhodecode.org/{gistid}. Empty means use the internal
 ; RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/{gistid}
 gist_alias_url =
 
 ; List of views (using glob pattern syntax) that AUTH TOKENS could be
 ; used for access.
 ; Adding ?auth_token=TOKEN_HASH to the url authenticates this request as if it
 ; came from the the logged in user who own this authentication token.
 ; Additionally @TOKEN syntax can be used to bound the view to specific
 ; authentication token. Such view would be only accessible when used together
 ; with this authentication token
 ; list of all views can be found under `/_admin/permissions/auth_token_access`
 ; The list should be "," separated and on a single line.
 ; Most common views to enable:
 
 #    RepoCommitsView:repo_commit_download
 #    RepoCommitsView:repo_commit_patch
 #    RepoCommitsView:repo_commit_raw
 #    RepoCommitsView:repo_commit_raw@TOKEN
 #    RepoFilesView:repo_files_diff
 #    RepoFilesView:repo_archivefile
 #    RepoFilesView:repo_file_raw
 #    GistView:*
 api_access_controllers_whitelist =
 
 ; Default encoding used to convert from and to unicode
 ; can be also a comma separated list of encoding in case of mixed encodings
 default_encoding = UTF-8
 
 ; instance-id prefix
 ; a prefix key for this instance used for cache invalidation when running
 ; multiple instances of RhodeCode, make sure it's globally unique for
 ; all running RhodeCode instances. Leave empty if you don't use it
 instance_id =
 
 ; Fallback authentication plugin. Set this to a plugin ID to force the usage
 ; of an authentication plugin also if it is disabled by it's settings.
 ; This could be useful if you are unable to log in to the system due to broken
 ; authentication settings. Then you can enable e.g. the internal RhodeCode auth
 ; module to log in again and fix the settings.
 ; Available builtin plugin IDs (hash is part of the ID):
 ; egg:rhodecode-enterprise-ce#rhodecode
 ; egg:rhodecode-enterprise-ce#pam
 ; egg:rhodecode-enterprise-ce#ldap
 ; egg:rhodecode-enterprise-ce#jasig_cas
 ; egg:rhodecode-enterprise-ce#headers
 ; egg:rhodecode-enterprise-ce#crowd
 
 #rhodecode.auth_plugin_fallback = egg:rhodecode-enterprise-ce#rhodecode
 
 ; Flag to control loading of legacy plugins in py:/path format
 auth_plugin.import_legacy_plugins = true
 
 ; alternative return HTTP header for failed authentication. Default HTTP
 ; response is 401 HTTPUnauthorized. Currently HG clients have troubles with
 ; handling that causing a series of failed authentication calls.
 ; Set this variable to 403 to return HTTPForbidden, or any other HTTP code
 ; This will be served instead of default 401 on bad authentication
 auth_ret_code =
 
 ; use special detection method when serving auth_ret_code, instead of serving
 ; ret_code directly, use 401 initially (Which triggers credentials prompt)
 ; and then serve auth_ret_code to clients
 auth_ret_code_detection = false
 
 ; locking return code. When repository is locked return this HTTP code. 2XX
 ; codes don't break the transactions while 4XX codes do
 lock_ret_code = 423
 
-; allows to change the repository location in settings page
-allow_repo_location_change = true
+; Filesystem location were repositories should be stored
+repo_store.path = /var/opt/rhodecode_repo_store
 
 ; allows to setup custom hooks in settings page
 allow_custom_hooks_settings = true
 
 ; Generated license token required for EE edition license.
 ; New generated token value can be found in Admin > settings > license page.
 license_token =
 
 ; This flag hides sensitive information on the license page such as token, and license data
 license.hide_license_info = false
 
 ; supervisor connection uri, for managing supervisor and logs.
 supervisor.uri =
 
 ; supervisord group name/id we only want this RC instance to handle
 supervisor.group_id = prod
 
 ; Display extended labs settings
 labs_settings_active = true
 
 ; Custom exception store path, defaults to TMPDIR
 ; This is used to store exception from RhodeCode in shared directory
 #exception_tracker.store_path =
 
 ; Send email with exception details when it happens
 #exception_tracker.send_email = false
 
 ; Comma separated list of recipients for exception emails,
 ; e.g [e-mail],[e-mail]
 ; Can be left empty, then emails will be sent to ALL super-admins
 #exception_tracker.send_email_recipients =
 
 ; optional prefix to Add to email Subject
 #exception_tracker.email_prefix = [RHODECODE ERROR]
 
 ; File store configuration. This is used to store and serve uploaded files
 file_store.enabled = true
 
 ; Storage backend, available options are: local
 file_store.backend = local
 
-; path to store the uploaded binaries
-file_store.storage_path = %(here)s/data/file_store
+; path to store the uploaded binaries and artifacts
+file_store.storage_path = /var/opt/rhodecode_data/file_store
+
+
+; Redis url to acquire/check generation of archives locks
+archive_cache.locking.url = redis://redis:6379/1
+
+; Storage backend, only 'filesystem' and 'objectstore' are available now
+archive_cache.backend.type = filesystem
+
+; url for s3 compatible storage that allows to upload artifacts
+; e.g http://minio:9000
+archive_cache.objectstore.url = http://s3-minio:9000
+
+; key for s3 auth
+archive_cache.objectstore.key = key
+
+; secret for s3 auth
+archive_cache.objectstore.secret = secret
 
-; Uncomment and set this path to control settings for archive download cache.
+;region for s3 storage
+archive_cache.objectstore.region = eu-central-1
+
+; number of sharded buckets to create to distribute archives across
+; default is 8 shards
+archive_cache.objectstore.bucket_shards = 8
+
+; a top-level bucket to put all other shards in
+; objects will be stored in rhodecode-archive-cache/shard-N based on the bucket_shards number
+archive_cache.objectstore.bucket = rhodecode-archive-cache
+
+; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
+archive_cache.objectstore.retry = false
+
+; number of seconds to wait for next try using retry
+archive_cache.objectstore.retry_backoff = 1
+
+; how many tries do do a retry fetch from this backend
+archive_cache.objectstore.retry_attempts = 10
+
+; Default is $cache_dir/archive_cache if not set
 ; Generated repo archives will be cached at this location
 ; and served from the cache during subsequent requests for the same archive of
 ; the repository. This path is important to be shared across filesystems and with
 ; RhodeCode and vcsserver
-
-; Default is $cache_dir/archive_cache if not set
-archive_cache.store_dir = %(here)s/data/archive_cache
+archive_cache.filesystem.store_dir = /var/opt/rhodecode_data/archive_cache
 
 ; The limit in GB sets how much data we cache before recycling last used, defaults to 10 gb
-archive_cache.cache_size_gb = 40
+archive_cache.filesystem.cache_size_gb = 40
+
+; Eviction policy used to clear out after cache_size_gb limit is reached
+archive_cache.filesystem.eviction_policy = least-recently-stored
 
 ; By default cache uses sharding technique, this specifies how many shards are there
-archive_cache.cache_shards = 4
+; default is 8 shards
+archive_cache.filesystem.cache_shards = 8
+
+; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
+archive_cache.filesystem.retry = false
+
+; number of seconds to wait for next try using retry
+archive_cache.filesystem.retry_backoff = 1
+
+; how many tries do do a retry fetch from this backend
+archive_cache.filesystem.retry_attempts = 10
+
 
 ; #############
 ; CELERY CONFIG
 ; #############
 
 ; manually run celery: /path/to/celery worker --task-events --beat --app rhodecode.lib.celerylib.loader --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler --loglevel DEBUG --ini /path/to/rhodecode.ini
 
-use_celery = false
+use_celery = true
 
 ; path to store schedule database
 #celerybeat-schedule.path =
 
 ; connection url to the message broker (default redis)
 celery.broker_url = redis://redis:6379/8
 
 ; results backend to get results for (default redis)
 celery.result_backend = redis://redis:6379/8
 
 ; rabbitmq example
 #celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost
 
 ; maximum tasks to execute before worker restart
 celery.max_tasks_per_child = 20
 
 ; tasks will never be sent to the queue, but executed locally instead.
 celery.task_always_eager = false
 
 ; #############
 ; DOGPILE CACHE
 ; #############
 
 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
-cache_dir = %(here)s/data
+cache_dir = /var/opt/rhodecode_data
 
 ; *********************************************
 ; `sql_cache_short` cache for heavy SQL queries
 ; Only supported backend is `memory_lru`
 ; *********************************************
 rc_cache.sql_cache_short.backend = dogpile.cache.rc.memory_lru
 rc_cache.sql_cache_short.expiration_time = 30
 
 
 ; *****************************************************
 ; `cache_repo_longterm` cache for repo object instances
 ; Only supported backend is `memory_lru`
 ; *****************************************************
 rc_cache.cache_repo_longterm.backend = dogpile.cache.rc.memory_lru
 ; by default we use 30 Days, cache is still invalidated on push
 rc_cache.cache_repo_longterm.expiration_time = 2592000
 ; max items in LRU cache, set to smaller number to save memory, and expire last used caches
 rc_cache.cache_repo_longterm.max_size = 10000
 
 
 ; *********************************************
 ; `cache_general` cache for general purpose use
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; *********************************************
 rc_cache.cache_general.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_general.expiration_time = 43200
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_general.arguments.filename = /tmp/cache_general_db
 
 ; alternative `cache_general` redis backend with distributed lock
 #rc_cache.cache_general.backend = dogpile.cache.rc.redis
 #rc_cache.cache_general.expiration_time = 300
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_general.arguments.redis_expiration_time = 7200
 
 #rc_cache.cache_general.arguments.host = localhost
 #rc_cache.cache_general.arguments.port = 6379
 #rc_cache.cache_general.arguments.db = 0
 #rc_cache.cache_general.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_general.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_general.arguments.lock_auto_renewal = true
 
 ; *************************************************
 ; `cache_perms` cache for permission tree, auth TTL
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; *************************************************
 rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_perms.expiration_time = 3600
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_perms.arguments.filename = /tmp/cache_perms_db
 
 ; alternative `cache_perms` redis backend with distributed lock
 #rc_cache.cache_perms.backend = dogpile.cache.rc.redis
 #rc_cache.cache_perms.expiration_time = 300
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_perms.arguments.redis_expiration_time = 7200
 
 #rc_cache.cache_perms.arguments.host = localhost
 #rc_cache.cache_perms.arguments.port = 6379
 #rc_cache.cache_perms.arguments.db = 0
 #rc_cache.cache_perms.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_perms.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_perms.arguments.lock_auto_renewal = true
 
 ; ***************************************************
 ; `cache_repo` cache for file tree, Readme, RSS FEEDS
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; ***************************************************
 rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_repo.expiration_time = 2592000
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_repo.arguments.filename = /tmp/cache_repo_db
 
 ; alternative `cache_repo` redis backend with distributed lock
 #rc_cache.cache_repo.backend = dogpile.cache.rc.redis
 #rc_cache.cache_repo.expiration_time = 2592000
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_repo.arguments.redis_expiration_time = 2678400
 
 #rc_cache.cache_repo.arguments.host = localhost
 #rc_cache.cache_repo.arguments.port = 6379
 #rc_cache.cache_repo.arguments.db = 1
 #rc_cache.cache_repo.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_repo.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_repo.arguments.lock_auto_renewal = true
 
 ; ##############
 ; BEAKER SESSION
 ; ##############
 
 ; beaker.session.type is type of storage options for the logged users sessions. Current allowed
 ; types are file, ext:redis, ext:database, ext:memcached
 ; Fastest ones are ext:redis and ext:database, DO NOT use memory type for session
-beaker.session.type = file
-beaker.session.data_dir = %(here)s/data/sessions
+#beaker.session.type = file
+#beaker.session.data_dir = %(here)s/data/sessions
 
 ; Redis based sessions
-#beaker.session.type = ext:redis
-#beaker.session.url = redis://127.0.0.1:6379/2
+beaker.session.type = ext:redis
+beaker.session.url = redis://redis:6379/2
 
 ; DB based session, fast, and allows easy management over logged in users
 #beaker.session.type = ext:database
 #beaker.session.table_name = db_session
 #beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode
 #beaker.session.sa.url = mysql://root:secret@127.0.0.1/rhodecode
 #beaker.session.sa.pool_recycle = 3600
 #beaker.session.sa.echo = false
 
 beaker.session.key = rhodecode
 beaker.session.secret = production-rc-uytcxaz
-beaker.session.lock_dir = %(here)s/data/sessions/lock
+beaker.session.lock_dir = /data_ramdisk/lock
 
 ; Secure encrypted cookie. Requires AES and AES python libraries
 ; you must disable beaker.session.secret to use this
 #beaker.session.encrypt_key = key_for_encryption
 #beaker.session.validate_key = validation_key
 
 ; Sets session as invalid (also logging out user) if it haven not been
 ; accessed for given amount of time in seconds
 beaker.session.timeout = 2592000
 beaker.session.httponly = true
 
 ; Path to use for the cookie. Set to prefix if you use prefix middleware
 #beaker.session.cookie_path = /custom_prefix
 
 ; Set https secure cookie
 beaker.session.secure = false
 
 ; default cookie expiration time in seconds, set to `true` to set expire
 ; at browser close
 #beaker.session.cookie_expires = 3600
 
 ; #############################
 ; SEARCH INDEXING CONFIGURATION
 ; #############################
 
 ; Full text search indexer is available in rhodecode-tools under
 ; `rhodecode-tools index` command
 
 ; WHOOSH Backend, doesn't require additional services to run
 ; it works good with few dozen repos
 search.module = rhodecode.lib.index.whoosh
 search.location = %(here)s/data/index
 
 ; ####################
 ; CHANNELSTREAM CONFIG
 ; ####################
 
 ; channelstream enables persistent connections and live notification
 ; in the system. It's also used by the chat system
 
-channelstream.enabled = false
+channelstream.enabled = true
 
 ; server address for channelstream server on the backend
-channelstream.server = 127.0.0.1:9800
+channelstream.server = channelstream:9800
 
 ; location of the channelstream server from outside world
 ; use ws:// for http or wss:// for https. This address needs to be handled
 ; by external HTTP server such as Nginx or Apache
 ; see Nginx/Apache configuration examples in our docs
 channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream
-channelstream.secret = secret
-channelstream.history.location = %(here)s/channelstream_history
+channelstream.secret = ENV_GENERATED
+channelstream.history.location = /var/opt/rhodecode_data/channelstream_history
 
 ; Internal application path that Javascript uses to connect into.
 ; If you use proxy-prefix the prefix should be added before /_channelstream
 channelstream.proxy_path = /_channelstream
 
 
 ; ##############################
 ; MAIN RHODECODE DATABASE CONFIG
 ; ##############################
 
 #sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
 #sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 #sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode?charset=utf8
 ; pymysql is an alternative driver for MySQL, use in case of problems with default one
 #sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode
 
 sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 
 ; see sqlalchemy docs for other advanced settings
 ; print the sql statements to output
 sqlalchemy.db1.echo = false
 
 ; recycle the connections after this amount of seconds
 sqlalchemy.db1.pool_recycle = 3600
 
 ; the number of connections to keep open inside the connection pool.
 ; 0 indicates no limit
 ; the general calculus with gevent is:
 ; if your system allows 500 concurrent greenlets (max_connections) that all do database access,
 ; then increase pool size + max overflow so that they add up to 500.
 #sqlalchemy.db1.pool_size = 5
 
 ; The number of connections to allow in connection pool "overflow", that is
 ; connections that can be opened above and beyond the pool_size setting,
 ; which defaults to five.
 #sqlalchemy.db1.max_overflow = 10
 
 ; Connection check ping, used to detect broken database connections
 ; could be enabled to better handle cases if MySQL has gone away errors
 #sqlalchemy.db1.ping_connection = true
 
 ; ##########
 ; VCS CONFIG
 ; ##########
 vcs.server.enable = true
-vcs.server = localhost:9900
+vcs.server = vcsserver:10010
 
 ; Web server connectivity protocol, responsible for web based VCS operations
 ; Available protocols are:
 ; `http` - use http-rpc backend (default)
 vcs.server.protocol = http
 
 ; Push/Pull operations protocol, available options are:
 ; `http` - use http-rpc backend (default)
 vcs.scm_app_implementation = http
 
 ; Push/Pull operations hooks protocol, available options are:
 ; `http` - use http-rpc backend (default)
+; `celery` - use celery based hooks
 vcs.hooks.protocol = http
 
 ; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be
 ; accessible via network.
 ; Use vcs.hooks.host = "*" to bind to current hostname (for Docker)
 vcs.hooks.host = *
 
 ; Start VCSServer with this instance as a subprocess, useful for development
 vcs.start_server = false
 
 ; List of enabled VCS backends, available options are:
 ; `hg`  - mercurial
 ; `git` - git
 ; `svn` - subversion
 vcs.backends = hg, git, svn
 
 ; Wait this number of seconds before killing connection to the vcsserver
 vcs.connection_timeout = 3600
 
-; Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
-; Set a numeric version for your current SVN e.g 1.8, or 1.12
-; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
-#vcs.svn.compatible_version = 1.8
-
 ; Cache flag to cache vcsserver remote calls locally
 ; It uses cache_region `cache_repo`
 vcs.methods.cache = true
 
 ; ####################################################
 ; Subversion proxy support (mod_dav_svn)
 ; Maps RhodeCode repo groups into SVN paths for Apache
 ; ####################################################
 
+; Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
+; Set a numeric version for your current SVN e.g 1.8, or 1.12
+; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
+#vcs.svn.compatible_version = 1.8
+
+; Redis connection settings for svn integrations logic
+; This connection string needs to be the same on ce and vcsserver
+vcs.svn.redis_conn = redis://redis:6379/0
+
+; Enable SVN proxy of requests over HTTP
+vcs.svn.proxy.enabled = true
+
+; host to connect to running SVN subsystem
+vcs.svn.proxy.host = http://svn:8090
+
 ; Enable or disable the config file generation.
-svn.proxy.generate_config = false
+svn.proxy.generate_config = true
 
 ; Generate config file with `SVNListParentPath` set to `On`.
 svn.proxy.list_parent_path = true
 
 ; Set location and file name of generated config file.
-svn.proxy.config_file_path = %(here)s/mod_dav_svn.conf
+svn.proxy.config_file_path = /etc/rhodecode/conf/svn/mod_dav_svn.conf
 
 ; alternative mod_dav config template. This needs to be a valid mako template
 ; Example template can be found in the source code:
 ; rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako
 #svn.proxy.config_template = ~/.rccontrol/enterprise-1/custom_svn_conf.mako
 
 ; Used as a prefix to the `Location` block in the generated config file.
 ; In most cases it should be set to `/`.
 svn.proxy.location_root = /
 
 ; Command to reload the mod dav svn configuration on change.
 ; Example: `/etc/init.d/apache2 reload` or /home/USER/apache_reload.sh
 ; Make sure user who runs RhodeCode process is allowed to reload Apache
 #svn.proxy.reload_cmd = /etc/init.d/apache2 reload
 
 ; If the timeout expires before the reload command finishes, the command will
 ; be killed. Setting it to zero means no timeout. Defaults to 10 seconds.
 #svn.proxy.reload_timeout = 10
 
 ; ####################
 ; SSH Support Settings
 ; ####################
 
 ; Defines if a custom authorized_keys file should be created and written on
 ; any change user ssh keys. Setting this to false also disables possibility
 ; of adding SSH keys by users from web interface. Super admins can still
 ; manage SSH Keys.
-ssh.generate_authorized_keyfile = false
+ssh.generate_authorized_keyfile = true
 
 ; Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding`
 # ssh.authorized_keys_ssh_opts =
 
 ; Path to the authorized_keys file where the generate entries are placed.
 ; It is possible to have multiple key files specified in `sshd_config` e.g.
 ; AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode
-ssh.authorized_keys_file_path = ~/.ssh/authorized_keys_rhodecode
+ssh.authorized_keys_file_path = /etc/rhodecode/conf/ssh/authorized_keys_rhodecode
 
 ; Command to execute the SSH wrapper. The binary is available in the
 ; RhodeCode installation directory.
-; e.g ~/.rccontrol/community-1/profile/bin/rc-ssh-wrapper
-ssh.wrapper_cmd = ~/.rccontrol/community-1/rc-ssh-wrapper
+; legacy: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+; new rewrite: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
+ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
 
 ; Allow shell when executing the ssh-wrapper command
 ssh.wrapper_cmd_allow_shell = false
 
 ; Enables logging, and detailed output send back to the client during SSH
 ; operations. Useful for debugging, shouldn't be used in production.
 ssh.enable_debug_logging = false
 
 ; Paths to binary executable, by default they are the names, but we can
 ; override them if we want to use a custom one
-ssh.executable.hg = ~/.rccontrol/vcsserver-1/profile/bin/hg
-ssh.executable.git = ~/.rccontrol/vcsserver-1/profile/bin/git
-ssh.executable.svn = ~/.rccontrol/vcsserver-1/profile/bin/svnserve
+ssh.executable.hg = /usr/local/bin/rhodecode_bin/vcs_bin/hg
+ssh.executable.git = /usr/local/bin/rhodecode_bin/vcs_bin/git
+ssh.executable.svn = /usr/local/bin/rhodecode_bin/vcs_bin/svnserve
 
 ; Enables SSH key generator web interface. Disabling this still allows users
 ; to add their own keys.
 ssh.enable_ui_key_generator = true
 
-
-; #################
-; APPENLIGHT CONFIG
-; #################
-
-; Appenlight is tailored to work with RhodeCode, see
-; http://appenlight.rhodecode.com for details how to obtain an account
-
-; Appenlight integration enabled
-#appenlight = false
-
-#appenlight.server_url = https://api.appenlight.com
-#appenlight.api_key = YOUR_API_KEY
-#appenlight.transport_config = https://api.appenlight.com?threaded=1&timeout=5
-
-; used for JS client
-#appenlight.api_public_key = YOUR_API_PUBLIC_KEY
-
-; TWEAK AMOUNT OF INFO SENT HERE
-
-; enables 404 error logging (default False)
-#appenlight.report_404 = false
-
-; time in seconds after request is considered being slow (default 1)
-#appenlight.slow_request_time = 1
-
-; record slow requests in application
-; (needs to be enabled for slow datastore recording and time tracking)
-#appenlight.slow_requests = true
-
-; enable hooking to application loggers
-#appenlight.logging = true
-
-; minimum log level for log capture
-#ppenlight.logging.level = WARNING
-
-; send logs only from erroneous/slow requests
-; (saves API quota for intensive logging)
-#appenlight.logging_on_error = false
-
-; list of additional keywords that should be grabbed from environ object
-; can be string with comma separated list of words in lowercase
-; (by default client will always send following info:
-; 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE' + all keys that
-; start with HTTP* this list be extended with additional keywords here
-#appenlight.environ_keys_whitelist =
-
-; list of keywords that should be blanked from request object
-; can be string with comma separated list of words in lowercase
-; (by default client will always blank keys that contain following words
-; 'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf'
-; this list be extended with additional keywords set here
-#appenlight.request_keys_blacklist =
-
-; list of namespaces that should be ignores when gathering log entries
-; can be string with comma separated list of namespaces
-; (by default the client ignores own entries: appenlight_client.client)
-#appenlight.log_namespace_blacklist =
-
 ; Statsd client config, this is used to send metrics to statsd
 ; We recommend setting statsd_exported and scrape them using Prometheus
 #statsd.enabled = false
 #statsd.statsd_host = 0.0.0.0
 #statsd.statsd_port = 8125
 #statsd.statsd_prefix =
 #statsd.statsd_ipv6 = false
 
 ; configure logging automatically at server startup set to false
 ; to use the below custom logging config.
 ; RC_LOGGING_FORMATTER
 ; RC_LOGGING_LEVEL
 ; env variables can control the settings for logging in case of autoconfigure
 
 #logging.autoconfigure = true
 
 ; specify your own custom logging config file to configure logging
 #logging.logging_conf_file = /path/to/custom_logging.ini
 
 ; Dummy marker to add new entries after.
 ; Add any custom entries below. Please don't remove this marker.
 custom.conf = 1
 
 
 ; #####################
 ; LOGGING CONFIGURATION
 ; #####################
 
 [loggers]
 keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
 
 [handlers]
 keys = console, console_sql
 
 [formatters]
 keys = generic, json, color_formatter, color_formatter_sql
 
 ; #######
 ; LOGGERS
 ; #######
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_sqlalchemy]
 level = INFO
 handlers = console_sql
 qualname = sqlalchemy.engine
 propagate = 0
 
 [logger_beaker]
 level = DEBUG
 handlers =
 qualname = beaker.container
 propagate = 1
 
 [logger_rhodecode]
 level = DEBUG
 handlers =
 qualname = rhodecode
 propagate = 1
 
 [logger_ssh_wrapper]
 level = DEBUG
 handlers =
 qualname = ssh_wrapper
 propagate = 1
 
 [logger_celery]
 level = DEBUG
 handlers =
 qualname = celery
 
 
 ; ########
 ; HANDLERS
 ; ########
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr, )
 level = INFO
 ; To enable JSON formatted logs replace 'generic/color_formatter' with 'json'
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 formatter = generic
 
 [handler_console_sql]
 ; "level = DEBUG" logs SQL queries and results.
 ; "level = INFO" logs SQL queries.
 ; "level = WARN" logs neither.  (Recommended for production systems.)
 class = StreamHandler
 args = (sys.stderr, )
 level = WARN
 ; To enable JSON formatted logs replace 'generic/color_formatter_sql' with 'json'
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 formatter = generic
 
 ; ##########
 ; FORMATTERS
 ; ##########
 
 [formatter_generic]
 class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class = rhodecode.lib.logging_formatter.ColorFormatter
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter_sql]
 class = rhodecode.lib.logging_formatter.ColorFormatterSql
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_json]
 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
 class = rhodecode.lib._vendor.jsonlogger.JsonFormatter

docs/admin/lab-settings.rst

@@ -1,31 +1,31 b''
 .. _lab-settings:
 
 Lab Settings
 ============
 
 |RCE| Lab Settings is for delivering features which may require an additional
 level of support to optimize for production scenarios. To enable lab settings,
 use the following instructions:
 
 1. Open the |RCE| configuration file,
-   :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+   :file:`config/_shared/rhodecode.ini`
 
 2. Add the following configuration option in the ``[app:main]`` section.
 
 .. code-block:: bash
 
    [app:main]
 
    ## Display extended labs settings
    labs_settings_active = true
 
 3. Restart your |RCE| instance
 
 .. code-block:: bash
 
    $ rccontrol restart enterprise-1
 
 4. You will see the labs setting on the
    :menuselection:`Admin --> Settings --> labs` page.
 
 .. image:: ../images/lab-setting.png

docs/admin/sec-x-frame.rst

@@ -1,57 +1,57 b''
 .. _x-frame:
 
 Securing HTTPS Connections
 --------------------------
 
 * To secure your |RCE| instance against `Cross Frame Scripting`_ exploits, you
   should configure your webserver ``x-frame-options`` setting.
 
 * To configure your instance for `HTTP Strict Transport Security`_, you need to
   configure the ``Strict-Transport-Security`` setting.
 
 Nginx
 ^^^^^
 
 In your nginx configuration, add the following lines in the correct files. For
 more detailed information see the :ref:`nginx-ws-ref` section.
 
 .. code-block:: nginx
 
     # Add this line to the nginx.conf file
     add_header X-Frame-Options SAMEORIGIN;
 
     # This line needs to be added inside your virtual hosts block/file
     add_header Strict-Transport-Security "max-age=31536000; includeSubdomains;";
 
 Apache
 ^^^^^^
 
 In your :file:`apache2.conf` file, add the following line. For more detailed
 information see the :ref:`apache-ws-ref` section.
 
 .. code-block:: apache
 
     # Add this to your virtual hosts file
     Header always append X-Frame-Options SAMEORIGIN
 
     # Add this line in your virtual hosts file
     Header always set Strict-Transport-Security "max-age=63072000; includeSubdomains; preload"
 
 |RCE| Configuration
 ^^^^^^^^^^^^^^^^^^^
 
 |RCE| can also be configured to force strict *https* connections and Strict
 Transport Security. To set this, configure the following options to ``true``
-in the :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file.
+in the :file:`config/_shared/rhodecode.ini` file.
 
 .. code-block:: ini
 
     ## force https in RhodeCode, fixes https redirects, assumes it's always https
     force_https = false
 
     ## use Strict-Transport-Security headers
     use_htsts = false
 
 
 .. _Cross Frame Scripting: https://www.owasp.org/index.php/Cross_Frame_Scripting
 .. _HTTP Strict Transport Security: https://www.owasp.org/index.php/HTTP_Strict_Transport_Security
  No newline at end of file

docs/admin/sec-your-server.rst

@@ -1,179 +1,179 b''
 .. _sec-your-server:
 
 Securing Your Server
 --------------------
 
 |RCE| runs on your hardware, and while it is developed with security in mind
 it is also important that you ensure your servers are well secured. In this
 section we will cover some basic security practices that are best to
 configure when setting up your |RCE| instances.
 
 SSH Keys
 ^^^^^^^^
 
 Using SSH keys to access your server provides more security than using the
 standard username and password combination. To set up your SSH Keys, use the
 following steps:
 
 1. On your local machine create the public/private key combination. The
    private key you will keep, and the matching public key is copied to the
    server. Setting a passphrase here is optional, if you set one you will
    always be prompted for it when logging in.
 
 .. code-block:: bash
 
     # Generate SSH Keys
     user@ubuntu:~$ ssh-keygen -t rsa
 
 .. code-block:: bash
 
     Generating public/private rsa key pair.
     Enter file in which to save the key (/home/user/.ssh/id_rsa):
     Created directory '/home/user/.ssh'.
     Enter passphrase (empty for no passphrase):
     Enter same passphrase again:
     Your identification has been saved in /home/user/.ssh/id_rsa.
     Your public key has been saved in /home/user/.ssh/id_rsa.pub.
     The key fingerprint is:
     02:82:38:95:e5:30:d2:ad:17:60:15:7f:94:17:9f:30 user@ubuntu
     The key\'s randomart image is:
     +--[ RSA 2048]----+
 
 2. SFTP to your server, and copy the public key to the ``~/.ssh`` folder.
 
 .. code-block:: bash
 
     # SFTP to your server
     $ sftp user@hostname
 
     # copy your public key
     sftp> mput /home/user/.ssh/id_rsa.pub /home/user/.ssh
     Uploading /home/user/.ssh/id_rsa.pub to /home/user/.ssh/id_rsa.pub
     /home/user/.ssh/id_rsa.pub      100%  394     0.4KB/s   00:00
 
 3. On your server, add the public key to the :file:`~/.ssh/authorized_keys`
    file.
 
 .. code-block:: bash
 
     $ cat /home/user/.ssh/id_rsa.pub > /home/user/.ssh/authorized_keys
 
 You should now be able to log into your server using your SSH
 Keys. If you've added a passphrase you'll be asked for it. For more
 information about using SSH keys with |RCE| |repos|, see the
 :ref:`ssh-connection` section.
 
 VPN Whitelist
 ^^^^^^^^^^^^^
 
 Most company networks will have a VPN. If you need to set one up, there are
 many tutorials online for how to do that. Getting it right requires good
 knowledge and attention to detail. Once set up, you can configure your
 |RCE| instances to only allow user access from the VPN, to do this see the
 :ref:`settip-ip-white` section.
 
 Public Key Infrastructure and SSL/TLS Encryption
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Public key infrastructure (PKI) is a system that creates, manages, and
 validates certificates for identifying nodes on a network and encrypting
 communication between them. SSL or TLS certificates can be used to
 authenticate different entities with one another. To read more about PKIs,
 see the `OpenSSL PKI tutorial`_ site, or this `Cloudflare PKI post`_.
 
 If the network you are running is SSL/TLS encrypted, you can configure |RCE|
 to always use secure connections using the ``force_https`` and ``use_htsts``
-options in the :file:`/home/user/.rccontrol/instance-id/rhodecode.ini` file.
+options in the :file:`config/_shared/rhodecode.ini` file.
 For more details, see the :ref:`x-frame` section.
 
 FireWalls and Ports
 ^^^^^^^^^^^^^^^^^^^
 
 Setting up a network firewall for your internal traffic is a good way
 of keeping it secure by blocking off any ports that should not be used.
 Additionally, you can set non-default ports for certain functions which adds
 an extra layer of security to your setup.
 
 A well configured firewall will restrict access to everything except the
 services you need to remain open. By exposing fewer services you reduce the
 number of potential vulnerabilities.
 
 There are a number of different firewall solutions, but for most Linux systems
 using the built in `IpTables`_ firewall should suffice. On BSD systems you
 can use `IPFILTER`_ or `IPFW`_. Use the following examples, and the IpTables
 documentation to configure your IP Tables on Ubuntu.
 
 Changing the default SSH port.
 
 .. code-block:: bash
 
     # Open SSH config file and change to port 10022
     vi /etc/ssh/sshd_config
 
     # What ports, IPs and protocols we listen for
     Port 10022
 
 Setting IP Table rules for SSH traffic. It is important to note that the
 default policy of your IpTables can differ and it is worth checking how each
 is configured. The options are *ACCEPT*, *REJECT*, *DROP*, or *LOG*. The
 usual practice is to block access on all ports and then enable access only on
 the ports you with to expose.
 
 .. code-block:: bash
 
     # Check iptables policy
     $ sudo iptables -L
 
     Chain INPUT (policy ACCEPT)
     target     prot opt source               destination
 
     Chain FORWARD (policy ACCEPT)
     target     prot opt source               destination
 
     Chain OUTPUT (policy ACCEPT)
     target     prot opt source               destination
 
     # Close all ports by default
     $ sudo iptables -P INPUT DROP
 
     $ sudo iptables -L
     Chain INPUT (policy DROP)
     target     prot opt source               destination
     DROP       all  --  anywhere             anywhere
 
     Chain FORWARD (policy ACCEPT)
     target     prot opt source               destination
 
     Chain OUTPUT (policy ACCEPT)
     target     prot opt source               destination
 
 .. code-block:: bash
 
     # Deny outbound SSH traffic
     sudo iptables -A OUTPUT -p tcp --dport 10022 -j DROP
 
     # Allow incoming SSH traffic on port 10022
     sudo iptables -A INPUT -p tcp --dport 10022 -j ACCEPT
 
     # Allow incoming HTML traffic on port 80 and 443
     iptables -A INPUT -p tcp -m tcp --dport 80 -j ACCEPT
     iptables -A INPUT -p tcp -m tcp --dport 443 -j ACCEPT
 
 Saving your IP Table rules, and restoring them from file.
 
 .. code-block:: bash
 
     # Save you IP Table Rules
     iptables-save
 
     # Save your IP Table Rules to a file
     sudo sh -c "iptables-save > /etc/iptables.rules"
 
     # Restore your IP Table rules from file
     iptables-restore < /etc/iptables.rules
 
 .. _OpenSSL PKI tutorial: https://pki-tutorial.readthedocs.org/en/latest/#
 .. _Cloudflare PKI post: https://blog.cloudflare.com/how-to-build-your-own-public-key-infrastructure/
 .. _IpTables: https://help.ubuntu.com/community/IptablesHowTo
 .. _IPFW: https://www.freebsd.org/doc/handbook/firewalls-ipfw.html
 .. _IPFILTER: https://www.freebsd.org/doc/handbook/firewalls-ipf.html

docs/admin/system-overview.rst

@@ -1,172 +1,172 b''
 .. _system-overview-ref:
 
 System Overview
 ===============
 
 Latest Version
 --------------
 
 * |release| on Unix and Windows systems.
 
 System Architecture
 -------------------
 
 The following diagram shows a typical production architecture.
 
 .. image:: ../images/architecture-diagram.png
   :align: center
 
 Supported Operating Systems
 ---------------------------
 
 Linux
 ^^^^^
 
 * Ubuntu 14.04+
 * CentOS 6.2, 7 and 8
 * RHEL 6.2, 7 and 8
 * Debian 7.8
 * RedHat Fedora
 * Arch Linux
 * SUSE Linux
 
 Windows
 ^^^^^^^
 
 * Windows Vista Ultimate 64bit
 * Windows 7 Ultimate 64bit
 * Windows 8 Professional 64bit
 * Windows 8.1 Enterprise 64bit
 * Windows Server 2008 64bit
 * Windows Server 2008-R2 64bit
 * Windows Server 2012 64bit
 
 Supported Databases
 -------------------
 
 * SQLite
 * MySQL
 * MariaDB
 * PostgreSQL
 
 Supported Browsers
 ------------------
 
 * Chrome
 * Safari
 * Firefox
 * Internet Explorer 10 & 11
 
 System Requirements
 -------------------
 
 |RCE| performs best on machines with ultra-fast hard disks. Generally disk
 performance is more important than CPU performance. In a corporate production
 environment handling 1000s of users and |repos| you should deploy on a 12+
 core 64GB RAM server. In short, the more RAM the better.
 
 
 For example:
 
  - for team of 1 - 5 active users you can run on 1GB RAM machine with 1CPU
  - above 250 active users, |RCE| needs at least 8GB of memory.
    Number of CPUs is less important, but recommended to have at least 2-3 CPUs
 
 
 .. _config-rce-files:
 
 Configuration Files
 -------------------
 
-* :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+* :file:`config/_shared/rhodecode.ini`
 * :file:`/home/{user}/.rccontrol/{instance-id}/search_mapping.ini`
 * :file:`/home/{user}/.rccontrol/{vcsserver-id}/vcsserver.ini`
 * :file:`/home/{user}/.rccontrol/supervisor/supervisord.ini`
 * :file:`/home/{user}/.rccontrol.ini`
 * :file:`/home/{user}/.rhoderc`
 * :file:`/home/{user}/.rccontrol/cache/MANIFEST`
 
 For more information, see the :ref:`config-files` section.
 
 Log Files
 ---------
 
 * :file:`/home/{user}/.rccontrol/{instance-id}/enterprise.log`
 * :file:`/home/{user}/.rccontrol/{vcsserver-id}/vcsserver.log`
 * :file:`/home/{user}/.rccontrol/supervisor/supervisord.log`
 * :file:`/tmp/rccontrol.log`
 * :file:`/tmp/rhodecode_tools.log`
 
 Storage Files
 -------------
 
 * :file:`/home/{user}/.rccontrol/{instance-id}/data/index/{index-file.toc}`
 * :file:`/home/{user}/repos/.rc_gist_store`
 * :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.db`
 * :file:`/opt/rhodecode/store/{unique-hash}`
 
 Default Repositories Location
 -----------------------------
 
 * :file:`/home/{user}/repos`
 
 Connection Methods
 ------------------
 
 * HTTPS
 * SSH
 * |RCE| API
 
 Internationalization Support
 ----------------------------
 
 Currently available in the following languages, see `Transifex`_ for the
 latest details. If you want a new language added, please contact us. To
 configure your language settings, see the :ref:`set-lang` section.
 
 .. hlist::
 
   * Belorussian
   * Chinese
   * French
   * German
   * Italian
   * Japanese
   * Portuguese
   * Polish
   * Russian
   * Spanish
 
 Licencing Information
 ---------------------
 
 * See licencing information `here`_
 
 Peer-to-peer Failover Support
 -----------------------------
 
 * Yes
 
 Additional Binaries
 -------------------
 
 * Yes, see :ref:`rhodecode-nix-ref` for full details.
 
 Remote Connectivity
 -------------------
 
 * Available
 
 Executable Files
 ----------------
 
 Windows: :file:`RhodeCode-installer-{version}.exe`
 
 Deprecated Support
 ------------------
 
 - Internet Explorer 8 support deprecated since version 3.7.0.
 - Internet Explorer 9 support deprecated since version 3.8.0.
 
 .. _here: https://rhodecode.com/licenses/
 .. _Transifex: https://explore.transifex.com/rhodecode/RhodeCode/

docs/admin/system_admin/admin-tricks.rst

@@ -1,300 +1,300 b''
 .. _admin-tricks:
 
 One-time Admin Tasks
 --------------------
 
 * :ref:`web-analytics`
 * :ref:`admin-tricks-license`
 * :ref:`announcements`
 * :ref:`md-rst`
 * :ref:`repo-stats`
 * :ref:`server-side-merge`
 * :ref:`remap-rescan`
 * :ref:`custom-hooks`
 * :ref:`clear-repo-cache`
 * :ref:`set-repo-pub`
 * :ref:`ping`
 
 .. _web-analytics:
 
 Adding Web Analytics
 ^^^^^^^^^^^^^^^^^^^^
 
 If you wish to add a Google Analytics, or any other kind of tracker to your
 |RCE| instance you can add the necessary codes to the header or footer
 section of each instance using the following steps:
 
 1. From the |RCE| interface, select
    :menuselection:`Admin --> Settings --> Global`
 2. To add a tracking code to you instance, enter it in the header or footer
    section and select **Save**
 
 Use the example templates in the drop-down menu to set up your configuration.
 
 .. _admin-tricks-license:
 
 Licence Key Management
 ^^^^^^^^^^^^^^^^^^^^^^
 
 To manage your license key, go to
 :menuselection:`Admin --> Settings --> License`.
 On this page you can see the license key details. If you need a new license,
 or have questions about your current one, contact [e-mail]
 
 .. _announcements:
 
 Server-wide Announcements
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 
 If you need to make a server-wide announcement to all users,
 you can add a message to be displayed using the following steps:
 
 1. From the |RCE| interface, select
    :menuselection:`Admin --> Settings --> Global`
 2. To add a message that will be displayed to all users,
    select :guilabel:`Server Announcement` from the drop-down menu and
    change the ``var message = "TYPE YOUR MESSAGE HERE";`` example line.
 3. Select :guilabel:`Save`, and you will see the message once your page
    refreshes.
 
 .. image:: ../../images/server-wide-announcement.png
    :alt: Server Wide Announcement
 
 .. _md-rst:
 
 
 Suppress license warnings or errors
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 In case you're running on maximum allowed users, RhodeCode will display a
 warning message on pages that you're close to the license limits.
 It's often not desired to show that all the time. Here's how you can suppress
 the license messages.
 
 1. From the |RCE| interface, select
    :menuselection:`Admin --> Settings --> Global`
 2. Select :guilabel:`Flash message filtering` from the drop-down menu.
 3. Select :guilabel:`Save`, and you will no longer see the license message
    once your page refreshes.
 
 .. _admin-tricks-suppress-license-messages:
 
 
 Markdown or RST Rendering
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 
 |RCE| can use `Markdown`_ or `reStructured Text`_ in commit message,
 code review messages, and inline comments. To set the default to either,
 select your preference from the drop-down menu on the
 :menuselection:`Admin --> Settings --> Visual` page and select
 :guilabel:`Save settings`.
 
 .. _repo-stats:
 
 Enabling Repository Statistics
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 To enable |repo| statistics, use the following steps:
 
 1. From the |RCE| interface, open
    :menuselection:`Admin --> Repositories` and select
    :guilabel:`Edit` beside the |repo| for which you wish to enable statistics.
 2. Check the :guilabel:`Enable statistics` box, and select :guilabel:`Save`
 
 .. _server-side-merge:
 
 Enabling Server-side Merging
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 To enable server-side merging, use the following steps:
 
 1. From the |RCE| interface, open :menuselection:`Admin --> Settings --> VCS`
 2. Check the :guilabel:`Server-side merge` box, and select
    :guilabel:`Save Settings`
 
 If you encounter slow performance with server-side merging enabled, check the
 speed at which your server is performing actions. When server-side merging is
 enabled, the following actions occurs on the server.
 
 * A |pr| is created in the database.
 * A shadow |repo| is created as a working environment for the |pr|.
 * On display, |RCE| checks if the |pr| can be merged.
 
 To check how fast the shadow |repo| creation is occurring on your server, use
 the following steps:
 
 1. Log into your server and create a directory in your |repos| folder.
 2. Clone a |repo| that is showing slow performance and time the action.
 
 .. code-block:: bash
 
    # One option is to use the time command
    $ time hg clone SOURCE_REPO TARGET
 
 .. _remap-rescan:
 
 Remap and Rescan Repositories
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 You may want to Remap and rescan the |repos| that |RCE| is managing to ensure
 the system is always up-to-date. This is useful after importing, deleting,
 or carrying out general cleaning up operations. To do this use the
 following steps:
 
 1. From the |RCE|, open
    :menuselection:`Admin --> Settings --> Remap and rescan`
 2. Click :guilabel:`Rescan Repositories`
 
 Check the additional options if needed:
 
 * :guilabel:`Destroy old data`: Useful for purging deleted repository
   information from the database.
 * :guilabel:`Invalidate cache for all repositories`: Use this to completely
   remap all |repos|. Useful when importing or migrating |repos| to ensure all
   new information is picked up.
 
 .. _custom-hooks:
 
 Adding Custom Hooks
 ^^^^^^^^^^^^^^^^^^^
 
 To add custom hooks to your instance, use the following steps:
 
 1. Open :menuselection:`Admin --> Settings --> Hooks`
 2. Add your custom hook details, you can use a file path to specify custom
    hook scripts, for example:
    ``pretxnchangegroup.example`` with value ``python:/path/to/custom_hook.py:my_func_name``
 3. Select :guilabel:`Save`
 
 Also, see the RhodeCode Extensions section of the :ref:`rc-tools` guide. RhodeCode
 Extensions can be used to add additional hooks to your instance and comes
 with a number of pre-built plugins if you chose to install them.
 
 .. _clear-repo-cache:
 
 Clearing |repo| cache
 ^^^^^^^^^^^^^^^^^^^^^
 
 If you need to clear the cache for a particular |repo|, use the following steps:
 
 1. Open :menuselection:`Admin --> Repositories` and select :guilabel:`Edit`
    beside the |repo| whose cache you wish to clear.
 2. On the |repo| settings page, go to the :guilabel:`Caches` tab and select
    :guilabel:`Invalidate repository cache`.
 
 .. _set-lang:
 
 Changing Default Language
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 
 To change the default language of a |RCE| instance, change the language code
-in the :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file. To
+in the :file:`config/_shared/rhodecode.ini` file. To
 do this, use the following steps.
 
 1. Open the :file:`rhodecode.ini` file and set the required language code.
 
 .. code-block:: ini
 
    ## Optional Languages
    ## en(default), de, fr, it, ja, pl, pt, ru, zh
    lang = de
 
 2. Restart the |RCE| instance and check that the language has been updated.
 
 .. code-block:: bash
 
    $ rccontrol restart enterprise-2
    Instance "enterprise-2" successfully stopped.
    Instance "enterprise-2" successfully started.
 
 .. image:: ../../images/language.png
 
 .. _set-repo-pub:
 
 Setting Repositories to Publish
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 To automatically promote your local |repos| to public after pushing to |RCE|,
 enable the :guilabel:`Set repositories as publishing` option on the
 :menuselection:`Admin --> Settings --> VCS` page.
 
 .. note::
 
    This option is enabled by default on most |RCE| versions, but if upgrading
    from a 1.7.x version it could be disabled on upgrade due to inheriting
    older default settings.
 
 .. _ping:
 
 Pinging the |RCE| Server
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
 You can check the IP Address of your |RCE| instance using the
 following URL: ``{instance-URL}/_admin/ping``.
 
 .. code-block:: bash
 
    $ curl https://your.rhodecode.url/_admin/ping
    pong[rce-7880] => 203.0.113.23
 
 .. _Markdown: http://daringfireball.net/projects/markdown/
 .. _reStructured Text: http://docutils.sourceforge.io/docs/index.html
 
 
 Unarchiving a repository
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Archive operation for the repository is similar as delete. Archive keeps the data for future references
 but makes the repository read-only. After archiving the repository it shouldn't be modified in any way.
 This is why repository settings are disabled for an archived repository.
 
 If there's a need for unarchiving a repository for some reasons, the interactive
 ishell interface should be used.
 
 .. code-block:: bash
 
   # Open iShell from the terminal
   $ rccontrol ishell enterprise-1/community-1
 
 .. code-block:: python
 
   # Set repository as un-archived
   In [1]: repo = Repository.get_by_repo_name('SOME_REPO_NAME')
   In [2]: repo.archived = False
   In [3]: Session().add(repo);Session().commit()
 
 
 
 
 Bulk change repository owner
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Here's how one can change an owner of repository for an user who has been de activated.
 Settings a new owner can be done via ishell for all repositories that past owner had.
 
 do run this script the interactive ishell interface should be used.
 
 .. code-block:: bash
 
   # Open iShell from the terminal
   $ rccontrol ishell enterprise-1/community-1
 
 
 .. code-block:: python
 
     from rhodecode.model.db import User, Repository, Session
     from rhodecode.model.permission import PermissionModel
 
     # replace old-owner and new-owner with your exact users
     old_owner = User.get_by_username('old-owner')
     new_owner = User.get_by_username('new-owner')
 
     # list of users we need to "flush" permissions
     affected_user_ids = [new_owner.user_id, old_owner.user_id]
 
     for repo in Repository.get_all_repos(user_id=old_owner.user_id):
         repo.user = new_owner
         Session().add(repo)
         Session().commit()
 
     PermissionModel().trigger_permission_flush(affected_user_ids)

docs/admin/system_admin/config-files-overview.rst

@@ -1,74 +1,74 b''
 .. _config-files:
 
 Configuration Files Overview
 ============================
 
 |RCE| and |RCC| have a number of different configuration files. The following
 is a brief explanation of each, and links to their associated configuration
 sections.
 
 .. rst-class:: dl-horizontal
 
     \- **rhodecode.ini**
         Default location:
-        :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+        :file:`config/_shared/rhodecode.ini`
 
         This is the main |RCE| configuration file and controls much of its
         default behaviour. It is also used to configure certain customer
         settings. Here are some of the most common reasons to make changes to
         this file.
 
         * :ref:`config-database`
         * :ref:`set-up-mail`
         * :ref:`increase-gunicorn`
         * :ref:`x-frame`
 
     \- **search_mapping.ini**
         Default location:
         :file:`/home/{user}/.rccontrol/{instance-id}/search_mapping.ini`
 
         This file is used to control the |RCE| indexer. It comes configured
         to index your instance. To change the default configuration, see
         :ref:`advanced-indexing`.
 
     \- **vcsserver.ini**
         Default location:
         :file:`/home/{user}/.rccontrol/{vcsserver-id}/vcsserver.ini`
 
         The VCS Server handles the connection between your |repos| and |RCE|.
         See the :ref:`vcs-server` section for configuration options and more
         detailed information.
 
     \- **supervisord.ini**
         Default location:
         :file:`/home/{user}/.rccontrol/supervisor/supervisord.ini`
 
         |RCC| uses Supervisor to monitor and manage installed instances of
         |RCE| and the VCS Server. |RCC| will manage this file completely,
         unless you install |RCE| in self-managed mode. For more information,
         see the :ref:`Supervisor Setup<control:supervisor-setup>` section.
 
     \- **.rccontrol.ini**
         Default location: :file:`/home/{user}/.rccontrol.ini`
 
         This file contains the instances that |RCC| starts at boot, which is all
         by default, but for more information, see
         the :ref:`Manually Start At Boot <control:set-start-boot>` section.
 
     \- **.rhoderc**
         Default location: :file:`/home/{user}/.rhoderc`
 
         This file is used by the |RCE| API when accessing an instance from a
         remote machine. The API checks this file for connection and
         authentication details. For more details, see the :ref:`config-rhoderc`
         section.
 
     \- **MANIFEST**
         Default location: :file:`/home/{user}/.rccontrol/cache/MANIFEST`
 
         |RCC| uses this file to source the latest available builds from the
         secure RhodeCode download channels. The only reason to mess with this file
         is if you need to do an offline installation,
         see the :ref:`Offline Installation<control:offline-installer-ref>`
         instructions, otherwise |RCC| will completely manage this file.
 

docs/admin/system_admin/enable-debug.rst

@@ -1,148 +1,148 b''
 .. _debug-mode:
 
 Enabling Debug Mode
 -------------------
 
 Debug Mode will enable debug logging, and request tracking middleware. Debug Mode
 enabled DEBUG log-level which allows tracking various information about authentication
 failures, LDAP connection, email etc.
 
 The request tracking will add a special
 unique ID: `| req_id:00000000-0000-0000-0000-000000000000` at the end of each log line.
 The req_id is the same for each individual requests, it means that if you want to
 track particular user logs only, and exclude other concurrent ones
 simply grep by `req_id` uuid which you'll have to find for the individual request.
 
 To enable debug mode on a |RCE| instance you need to set the debug property
-in the :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file. To
+in the :file:`config/_shared/rhodecode.ini` file. To
 do this, use the following steps
 
 1. Open the file and set the ``debug`` line to ``true``
 2. Restart you instance using the ``rccontrol restart`` command,
    see the following example:
 
 .. code-block:: ini
 
     [DEFAULT]
     debug = true
 
 .. code-block:: bash
 
     # Restart your instance
     $ rccontrol restart enterprise-1
     Instance "enterprise-1" successfully stopped.
     Instance "enterprise-1" successfully started.
 
 
 Debug and Logging Configuration
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Further debugging and logging settings can also be set in the
-:file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file.
+:file:`config/_shared/rhodecode.ini` file.
 
 In the logging section, the various packages that run with |RCE| can have
 different debug levels set. If you want to increase the logging level change
 ``level = DEBUG`` line to one of the valid options.
 
 You also need to change the log level for handlers. See the example
 ``##handler`` section below. The ``handler`` level takes the same options as
 the ``debug`` level.
 
 .. code-block:: ini
 
     ################################
     ### LOGGING CONFIGURATION   ####
     ################################
     [loggers]
     keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
 
     [handlers]
     keys = console, console_sql, file, file_rotating
 
     [formatters]
     keys = generic, color_formatter, color_formatter_sql
 
     #############
     ## LOGGERS ##
     #############
     [logger_root]
     level = NOTSET
     handlers = console
 
     [logger_sqlalchemy]
     level = INFO
     handlers = console_sql
     qualname = sqlalchemy.engine
     propagate = 0
 
     [logger_beaker]
     level = DEBUG
     handlers =
     qualname = beaker.container
     propagate = 1
 
     [logger_rhodecode]
     level = DEBUG
     handlers =
     qualname = rhodecode
     propagate = 1
 
     [logger_ssh_wrapper]
     level = DEBUG
     handlers =
     qualname = ssh_wrapper
     propagate = 1
 
     [logger_celery]
     level = DEBUG
     handlers =
     qualname = celery
 
     ##############
     ## HANDLERS ##
     ##############
 
     [handler_console]
     class = StreamHandler
     args = (sys.stderr, )
     level = DEBUG
     formatter = generic
 
     [handler_console_sql]
     class = StreamHandler
     args = (sys.stderr, )
     level = INFO
     formatter = generic
 
     [handler_file]
     class = FileHandler
     args = ('rhodecode_debug.log', 'a',)
     level = INFO
     formatter = generic
 
     [handler_file_rotating]
     class = logging.handlers.TimedRotatingFileHandler
     # 'D', 5 - rotate every 5days
     # you can set 'h', 'midnight'
     args = ('rhodecode_debug_rotated.log', 'D', 5, 10,)
     level = INFO
     formatter = generic
 
     ################
     ## FORMATTERS ##
     ################
 
     [formatter_generic]
     class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
     format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
     datefmt = %Y-%m-%d %H:%M:%S
 
     [formatter_color_formatter]
     class = rhodecode.lib.logging_formatter.ColorFormatter
     format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
     datefmt = %Y-%m-%d %H:%M:%S
 
     [formatter_color_formatter_sql]
     class = rhodecode.lib.logging_formatter.ColorFormatterSql
     format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
     datefmt = %Y-%m-%d %H:%M:%S
  No newline at end of file

docs/admin/system_admin/svn-http.rst

@@ -1,208 +1,208 b''
 .. _svn-http:
 
 |svn| With Write Over HTTP
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 To use |svn| with read/write support over the |svn| HTTP protocol, you have to
 configure the HTTP |svn| backend.
 
 Prerequisites
 =============
 
 - Enable HTTP support inside the admin VCS settings on your |RCE| instance
 - You need to install the following tools on the machine that is running an
   instance of |RCE|:
   ``Apache HTTP Server`` and ``mod_dav_svn``.
 
 
 .. tip::
 
    We recommend using Wandisco repositories which provide latest SVN versions
    for most platforms. If you skip this version you'll have to ensure the Client version
    is compatible with installed SVN version which might differ depending on the operating system.
    Here is an example how to add the Wandisco repositories for Ubuntu.
 
     .. code-block:: bash
 
         $ sudo sh -c 'echo "deb http://opensource.wandisco.com/ubuntu `lsb_release -cs` svn110" >> /etc/apt/sources.list.d/subversion110.list'
         $ sudo wget -q http://opensource.wandisco.com/wandisco-debian-new.gpg -O- | sudo apt-key add -
         $ sudo apt-get update
 
     Here is an example how to add the Wandisco repositories for Centos/Redhat. Using
     a yum config
 
     .. code-block:: bash
 
         [wandisco-Git]
         name=CentOS-6 - Wandisco Git
         baseurl=http://opensource.wandisco.com/centos/6/git/$basearch/
         enabled=1
         gpgcheck=1
         gpgkey=http://opensource.wandisco.com/RPM-GPG-KEY-WANdisco
 
 
 
 Example installation of required components for Ubuntu platform:
 
 .. code-block:: bash
 
     $ sudo apt-get install apache2
     $ sudo apt-get install libapache2-svn
 
 Once installed you need to enable ``dav_svn`` on Ubuntu:
 
 .. code-block:: bash
 
     $ sudo a2enmod dav_svn
     $ sudo a2enmod headers
     $ sudo a2enmod authn_anon
 
 
 Example installation of required components for RedHat/CentOS platform:
 
 .. code-block:: bash
 
     $ sudo yum install httpd
     $ sudo yum install subversion mod_dav_svn
 
 
 Once installed you need to enable ``dav_svn`` on RedHat/CentOS:
 
 .. code-block:: bash
 
     sudo vi /etc/httpd/conf.modules.d/10-subversion.conf
     ## The file should read:
 
     LoadModule dav_svn_module     modules/mod_dav_svn.so
     LoadModule headers_module     modules/mod_headers.so
     LoadModule authn_anon_module  modules/mod_authn_anon.so
 
 .. tip::
 
    To check the installed mod_dav_svn module version, you can use such command.
 
    `strings /usr/lib/apache2/modules/mod_dav_svn.so | grep 'Powered by'`
 
 
 Configuring Apache Setup
 ========================
 
 .. tip::
 
    It is recommended to run Apache on a port other than 80, due to possible
    conflicts with other HTTP servers like nginx. To do this, set the
    ``Listen`` parameter in the ``/etc/apache2/ports.conf`` file, for example
    ``Listen 8090``.
 
 
 .. warning::
 
    Make sure your Apache instance which runs the mod_dav_svn module is
    only accessible by |RCE|. Otherwise everyone is able to browse
    the repositories or run subversion operations (checkout/commit/etc.).
 
 It is also recommended to run apache as the same user as |RCE|, otherwise
 permission issues could occur. To do this edit the ``/etc/apache2/envvars``
 
    .. code-block:: apache
 
       export APACHE_RUN_USER=rhodecode
       export APACHE_RUN_GROUP=rhodecode
 
 1. To configure Apache, create and edit a virtual hosts file, for example
    :file:`/etc/apache2/sites-enabled/default.conf`. Below is an example
    how to use one with auto-generated config ```mod_dav_svn.conf```
    from configured |RCE| instance.
 
 .. code-block:: apache
 
     <VirtualHost *:8090>
         ServerAdmin rhodecode-admin@localhost
         DocumentRoot /var/www/html
         ErrorLog ${'${APACHE_LOG_DIR}'}/error.log
         CustomLog ${'${APACHE_LOG_DIR}'}/access.log combined
         LogLevel info
         # allows custom host names, prevents 400 errors on checkout
         HttpProtocolOptions Unsafe
         # Most likely this will be: /home/user/.rccontrol/enterprise-1/mod_dav_svn.conf
         Include /home/user/.rccontrol/enterprise-1/mod_dav_svn.conf
     </VirtualHost>
 
 
 2. Go to the :menuselection:`Admin --> Settings --> VCS` page, and
    enable :guilabel:`Proxy Subversion HTTP requests`, and specify the
    :guilabel:`Subversion HTTP Server URL`.
 
 3. Open the |RCE| configuration file,
-   :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+   :file:`config/_shared/rhodecode.ini`
 
 4. Add the following configuration option in the ``[app:main]``
    section if you don't have it yet.
 
    This enables mapping of the created |RCE| repo groups into special
    |svn| paths. Each time a new repository group is created, the system will
    update the template file and create new mapping. Apache web server needs to
    be reloaded to pick up the changes on this file.
    To do this, simply configure `svn.proxy.reload_cmd` inside the .ini file.
    Example configuration:
 
 
 .. code-block:: ini
 
     ############################################################
     ### Subversion proxy support (mod_dav_svn)               ###
     ### Maps RhodeCode repo groups into SVN paths for Apache ###
     ############################################################
     ## Enable or disable the config file generation.
     svn.proxy.generate_config = true
     ## Generate config file with `SVNListParentPath` set to `On`.
     svn.proxy.list_parent_path = true
     ## Set location and file name of generated config file.
     svn.proxy.config_file_path = %(here)s/mod_dav_svn.conf
     ## Used as a prefix to the <Location> block in the generated config file.
     ## In most cases it should be set to `/`.
     svn.proxy.location_root = /
     ## Command to reload the mod dav svn configuration on change.
     ## Example: `/etc/init.d/apache2 reload`
     svn.proxy.reload_cmd = /etc/init.d/apache2 reload
     ## If the timeout expires before the reload command finishes, the command will
     ## be killed. Setting it to zero means no timeout. Defaults to 10 seconds.
     #svn.proxy.reload_timeout = 10
 
 
 This would create a special template file called ```mod_dav_svn.conf```. We
 used that file path in the apache config above inside the Include statement.
 It's also possible to manually generate the config from the
 :menuselection:`Admin --> Settings --> VCS` page by clicking a
 `Generate Apache Config` button.
 
 5. Now only things left is to enable svn support, and generate the initial
    configuration.
 
    - Select `Proxy subversion HTTP requests` checkbox
    - Enter http://localhost:8090 into `Subversion HTTP Server URL`
    - Click the `Generate Apache Config` button.
 
 This config will be automatically re-generated once an user-groups is added
 to properly map the additional paths generated.
 
 
 
 Using |svn|
 ===========
 
 Once |svn| has been enabled on your instance, you can use it with the
 following examples. For more |svn| information, see the `Subversion Red Book`_
 
 .. code-block:: bash
 
     # To clone a repository
     svn checkout http://my-svn-server.example.com/my-svn-repo
 
     # svn commit
     svn commit
 
 
 .. _Subversion Red Book: http://svnbook.red-bean.com/en/1.7/svn-book.html#svn.ref.svn
 
 .. _Ask Ubuntu: http://askubuntu.com/questions/162391/how-do-i-fix-my-locale-issue
  No newline at end of file

docs/admin/system_admin/tuning/tuning-change-encoding.rst

@@ -1,22 +1,22 b''
 .. _change-encoding:
 
 Change Default Encoding
 -----------------------
 
 |RCE| uses ``utf8`` encoding by default. You can change the default encoding
-in the :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file. To
+in the :file:`config/_shared/rhodecode.ini` file. To
 change the default encoding used by |RCE|, set a new value for the
 ``default_encoding``.
 
 .. code-block:: ini
 
         # default encoding used to convert from and to unicode
         # can be also a comma separated list of encoding in case of mixed
         # encodings
         default_encoding = utf8
 
 .. note::
 
        Changing the default encoding will affect many parts of your |RCE|
        installation, including committers names,
        file names, and the encoding of commit messages.

docs/admin/system_admin/tuning/tuning-hg-auth-loop.rst

@@ -1,17 +1,17 b''
 .. _hg-auth-loop:
 
 |hg| Authentication Tuning
 --------------------------
 
 When using external authentication tools such as LDAP with |hg|, a
 password retry loop in |hg| can result in users being locked out due to too
 many failed password attempts. To prevent this from happening, add the
 following setting to your
-:file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file, in the
+:file:`config/_shared/rhodecode.ini` file, in the
 ``[app:main]`` section.
 
 
 .. code-block:: ini
 
    [app:main]
    auth_ret_code_detection = true

docs/admin/system_admin/tuning/tuning-scale-horizontally-cluster.rst

@@ -1,396 +1,396 b''
 .. _scale-horizontal-cluster:
 
 
 Scale Horizontally / RhodeCode Cluster
 --------------------------------------
 
 |RCE| is built in a way it support horizontal scaling across multiple machines.
 There are three main pre-requisites for that:
 
 - Shared storage that each machine can access. Using NFS or other shared storage system.
 - Shared DB connection across machines. Using `MySQL`/`PostgreSQL` that each node can access.
 - |RCE| user sessions and caches need to use a shared storage (e.g `Redis`_/`Memcached`)
 
 
 Horizontal scaling means adding more machines or workers into your pool of
 resources. Horizontally scaling |RCE| gives a huge performance increase,
 especially under large traffic scenarios with a high number of requests.
 This is very beneficial when |RCE| is serving many users simultaneously,
 or if continuous integration servers are automatically pulling and pushing code.
 It also adds High-Availability to your running system.
 
 
 Cluster Overview
 ^^^^^^^^^^^^^^^^
 
 Below we'll present a configuration example that will use two separate nodes to serve
 |RCE| in a load-balanced environment. The 3rd node will act as a shared storage/cache
 and handle load-balancing. In addition 3rd node will be used as shared database instance.
 
 This setup can be used both in Docker based configuration or with individual
 physical/virtual machines. Using the 3rd node for Storage/Redis/PostgreSQL/Nginx is
 optional. All those components can be installed on one of the two nodes used for |RCE|.
 We'll use following naming for our nodes:
 
  - `rc-node-1` (NFS, DB, Cache node)
  - `rc-node-2` (Worker node1)
  - `rc-node-3` (Worker node2)
 
 Our shares NFS storage in the example is located on `/home/rcdev/storage` and
 it's RW accessible on **each** node.
 
 In this example we used certain recommended components, however many
 of those can be replaced by other, in case your organization already uses them, for example:
 
 - `MySQL`/`PostgreSQL`: Aren't replaceable and are the two only supported databases.
 - `Nginx`_ on `rc-node-1` can be replaced by: `Hardware Load Balancer (F5)`, `Apache`_, `HA-Proxy` etc.
 - `Nginx`_ on rc-node-2/3 acts as a reverse proxy and can be replaced by other HTTP server
   acting as reverse proxy such as `Apache`_.
 - `Redis`_ on `rc-node-1` can be replaced by: `Memcached`
 
 
 Here's an overview what components should be installed/setup on each server in our example:
 
 - **rc-node-1**:
 
  - main storage acting as NFS host.
  - `nginx` acting as a load-balancer.
  - `postgresql-server` used for database and sessions.
  - `redis-server` used for storing shared caches.
  - optionally `rabbitmq-server` or `redis` for `Celery` if used.
  - optionally if `Celery` is used Enterprise/Community instance + VCSServer.
  - optionally mailserver that can be shared by other instances.
  - optionally channelstream server to handle live communication for all instances.
 
 
 - **rc-node-2/3**:
 
  - `nginx` acting as a reverse proxy to handle requests to |RCE|.
  - 1x RhodeCode Enterprise/Community instance.
  - 1x VCSServer instance.
  - optionally for testing connection: postgresql-client, redis-client (redis-tools).
 
 
 Before we start here are few assumptions that should be fulfilled:
 
 - make sure each node can access each other.
 - make sure `Redis`_/`MySQL`/`PostgreSQL`/`RabbitMQ`_ are running on `rc-node-1`
 - make sure both `rc-node-2`/`3` can access NFS storage with RW access
 - make sure rc-node-2/3 can access `Redis`_/`PostgreSQL`, `MySQL` database on `rc-node-1`.
 - make sure `Redis`_/Database/`RabbitMQ`_ are password protected and accessible only from rc-node-2/3.
 
 
 
 Setup rc-node-2/3
 ^^^^^^^^^^^^^^^^^
 
 Initially before `rc-node-1` we'll configure both nodes 2 and 3 to operate as standalone
 nodes with their own hostnames. Use a default installation settings, and use
 the default local addresses (127.0.0.1) to configure VCSServer and Community/Enterprise instances.
 All external connectivity will be handled by the reverse proxy (`Nginx`_ in our example).
 
 This way we can ensure each individual host works,
 accepts connections, or do some operations explicitly on chosen node.
 
 In addition this would allow use to explicitly direct certain traffic to a node, e.g
 CI server will only call directly `rc-node-3`. This should be done similar to normal
 installation so check out `Nginx`_/`Apache`_ configuration example to configure each host.
 Each one should already connect to shared database during installation.
 
 
 1) Assuming our final url will be http://rc-node-1, Configure `instances_id`, `app.base_url`
 
-a) On **rc-node-2** find the following settings and edit :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+a) On **rc-node-2** find the following settings and edit :file:`config/_shared/rhodecode.ini`
 
 .. code-block:: ini
 
     ## required format is: *NAME-
     instance_id = *rc-node-2-
     app.base_url = http://rc-node-1
 
 
-b) On **rc-node-3** find the following settings and edit :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+b) On **rc-node-3** find the following settings and edit :file:`config/_shared/rhodecode.ini`
 
 .. code-block:: ini
 
     ## required format is: *NAME-
     instance_id = *rc-node-3-
     app.base_url = http://rc-node-1
 
 
 
 2) Configure `User Session` to use a shared database. Example config that should be
    changed on both **rc-node-2** and **rc-node-3** .
-   Edit :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+   Edit :file:`config/_shared/rhodecode.ini`
 
 .. code-block:: ini
 
     ####################################
     ###       BEAKER SESSION        ####
     ####################################
 
     ## Disable the default `file` sessions
     #beaker.session.type = file
     #beaker.session.data_dir = %(here)s/data/sessions
 
     ## use shared db based session, fast, and allows easy management over logged in users
     beaker.session.type = ext:database
     beaker.session.table_name = db_session
     # use our rc-node-1 here
     beaker.session.sa.url = postgresql://postgres:qweqwe@rc-node-1/rhodecode
     beaker.session.sa.pool_recycle = 3600
     beaker.session.sa.echo = false
 
 In addition make sure both instances use the same `session.secret` so users have
 persistent sessions across nodes. Please generate other one then in this example.
 
 .. code-block:: ini
 
     # use a unique generated long string
     beaker.session.secret = 70e116cae2274656ba7265fd860aebbd
 
 3) Configure stored cached/archive cache to our shared NFS `rc-node-1`
 
 .. code-block:: ini
 
     # note the `_` prefix that allows using a directory without
     # remap and rescan checking for vcs inside it.
     cache_dir = /home/rcdev/storage/_cache_dir/data
     # note archive cache dir is disabled by default, however if you enable
     # it also needs to be shared
     #archive_cache_dir = /home/rcdev/storage/_tarball_cache_dir
 
 
 4) Use shared exception store. Example config that should be
    changed on both **rc-node-2** and **rc-node-3**, and also for VCSServer.
-   Edit :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` and
+   Edit :file:`config/_shared/rhodecode.ini` and
    :file:`/home/{user}/.rccontrol/{vcsserver-instance-id}/vcsserver.ini`
    and add/change following setting.
 
 .. code-block:: ini
 
     exception_tracker.store_path = /home/rcdev/storage/_exception_store_data
 
 
 5) Change cache backends to use `Redis`_ based caches. Below full example config
    that replaces default file-based cache to shared `Redis`_ with Distributed Lock.
 
 
 .. code-block:: ini
 
     #####################################
     ###         DOGPILE CACHE        ####
     #####################################
 
     ## `cache_perms` cache settings for permission tree, auth TTL.
     #rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
     #rc_cache.cache_perms.expiration_time = 300
 
     ## alternative `cache_perms` redis backend with distributed lock
     rc_cache.cache_perms.backend = dogpile.cache.rc.redis
     rc_cache.cache_perms.expiration_time = 300
     ## redis_expiration_time needs to be greater then expiration_time
     rc_cache.cache_perms.arguments.redis_expiration_time = 7200
     rc_cache.cache_perms.arguments.socket_timeout = 30
     rc_cache.cache_perms.arguments.host = rc-node-1
     rc_cache.cache_perms.arguments.password = qweqwe
     rc_cache.cache_perms.arguments.port = 6379
     rc_cache.cache_perms.arguments.db = 0
     rc_cache.cache_perms.arguments.distributed_lock = true
 
     ## `cache_repo` cache settings for FileTree, Readme, RSS FEEDS
     #rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace
     #rc_cache.cache_repo.expiration_time = 2592000
 
     ## alternative `cache_repo` redis backend with distributed lock
     rc_cache.cache_repo.backend = dogpile.cache.rc.redis
     rc_cache.cache_repo.expiration_time = 2592000
     ## redis_expiration_time needs to be greater then expiration_time
     rc_cache.cache_repo.arguments.redis_expiration_time = 2678400
     rc_cache.cache_repo.arguments.socket_timeout = 30
     rc_cache.cache_repo.arguments.host = rc-node-1
     rc_cache.cache_repo.arguments.password = qweqwe
     rc_cache.cache_repo.arguments.port = 6379
     rc_cache.cache_repo.arguments.db = 1
     rc_cache.cache_repo.arguments.distributed_lock = true
 
     ## cache settings for SQL queries, this needs to use memory type backend
     rc_cache.sql_cache_short.backend = dogpile.cache.rc.memory_lru
     rc_cache.sql_cache_short.expiration_time = 30
 
     ## `cache_repo_longterm` cache for repo object instances, this needs to use memory
     ## type backend as the objects kept are not pickle serializable
     rc_cache.cache_repo_longterm.backend = dogpile.cache.rc.memory_lru
     ## by default we use 96H, this is using invalidation on push anyway
     rc_cache.cache_repo_longterm.expiration_time = 345600
     ## max items in LRU cache, reduce this number to save memory, and expire last used
     ## cached objects
     rc_cache.cache_repo_longterm.max_size = 10000
 
 
 6) Configure `Nginx`_ as reverse proxy on `rc-node-2/3`:
    Minimal `Nginx`_ config used:
 
 
 .. code-block:: nginx
 
     ## rate limiter for certain pages to prevent brute force attacks
     limit_req_zone  $binary_remote_addr  zone=req_limit:10m   rate=1r/s;
 
     ## custom log format
     log_format log_custom '$remote_addr - $remote_user [$time_local] '
                           '"$request" $status $body_bytes_sent '
                           '"$http_referer" "$http_user_agent" '
                           '$request_time $upstream_response_time $pipe';
 
     server {
         listen          80;
         server_name     rc-node-2;
         #server_name     rc-node-3;
 
         access_log   /var/log/nginx/rhodecode.access.log log_custom;
         error_log    /var/log/nginx/rhodecode.error.log;
 
         # example of proxy.conf can be found in our docs.
         include     /etc/nginx/proxy.conf;
 
         ## serve static files by Nginx, recommended for performance
         location /_static/rhodecode {
             gzip on;
             gzip_min_length  500;
             gzip_proxied     any;
             gzip_comp_level 4;
             gzip_types  text/css text/javascript text/xml text/plain text/x-component application/javascript application/json application/xml application/rss+xml font/truetype font/opentype application/vnd.ms-fontobject image/svg+xml;
             gzip_vary on;
             gzip_disable     "msie6";
             expires 60d;
             #alias /home/rcdev/.rccontrol/community-1/static;
             alias /home/rcdev/.rccontrol/enterprise-1/static;
         }
 
 
         location /_admin/login {
             limit_req  zone=req_limit  burst=10  nodelay;
             try_files $uri @rhode;
         }
 
         location / {
             try_files $uri @rhode;
         }
 
         location @rhode {
             # Url to running RhodeCode instance.
             # This is shown as `- URL: <host>` in output from rccontrol status.
             proxy_pass      http://127.0.0.1:10020;
         }
 
         ## custom 502 error page. Will be displayed while RhodeCode server
         ## is turned off
         error_page 502 /502.html;
         location = /502.html {
            #root  /home/rcdev/.rccontrol/community-1/static;
            root  /home/rcdev/.rccontrol/enterprise-1/static;
         }
     }
 
 
 7) Optional: Full text search, in case you use `Whoosh` full text search we also need a
    shared storage for the index. In our example our NFS is mounted at `/home/rcdev/storage`
    which represents out storage so we can use the following:
 
 .. code-block:: ini
 
     # note the `_` prefix that allows using a directory without
     # remap and rescan checking for vcs inside it.
     search.location = /home/rcdev/storage/_index_data/index
 
 
 .. note::
 
     If you use ElasticSearch it's by default shared, and simply running ES node is
     by default cluster compatible.
 
 
 8) Optional: If you intend to use mailing all instances need to use either a shared
    mailing node, or each will use individual local mail agent. Simply put node-1/2/3
    needs to use same mailing configuration.
 
 
 
 Setup rc-node-1
 ^^^^^^^^^^^^^^^
 
 
 Configure `Nginx`_ as Load Balancer to rc-node-2/3.
 Minimal `Nginx`_ example below:
 
 .. code-block:: nginx
 
     ## define rc-cluster which contains a pool of our instances to connect to
     upstream rc-cluster {
         # rc-node-2/3 are stored in /etc/hosts with correct IP addresses
         server rc-node-2:80;
         server rc-node-3:80;
     }
 
     server {
         listen          80;
         server_name     rc-node-1;
 
         location / {
             proxy_pass http://rc-cluster;
         }
     }
 
 
 .. note::
 
    You should configure your load balancing accordingly. We recommend writing
    load balancing rules that will separate regular user traffic from
    automated process traffic like continuous servers or build bots. Sticky sessions
    are not required.
 
 
 Show which instance handles a request
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 You can easily check if load-balancing is working as expected. Visit our main node
 `rc-node-1` URL which at that point should already handle incoming requests and balance
 it across node-2/3.
 
 Add a special GET param `?showrcid=1` to show current instance handling your request.
 
 For example: visiting url `http://rc-node-1/?showrcid=1` will show, in the bottom
 of the screen` cluster instance info.
 e.g: `RhodeCode instance id: rc-node-3-rc-node-3-3246`
 which is generated from::
 
     <NODE_HOSTNAME>-<INSTANCE_ID>-<WORKER_PID>
 
 
 Using Celery with cluster
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 
 
 If `Celery` is used we recommend setting also an instance of Enterprise/Community+VCSserver
 on the node that is running `RabbitMQ`_ or `Redis`_. Those instances will be used to
 executed async tasks on the `rc-node-1`. This is the most efficient setup.
 `Celery` usually handles tasks such as sending emails, forking repositories, importing
 repositories from external location etc. Using workers on instance that has
 the direct access to disks used by NFS as well as email server gives noticeable
 performance boost. Running local workers to the NFS storage results in faster
 execution of forking large repositories or sending lots of emails.
 
 Those instances need to be configured in the same way as for other nodes.
 The instance in rc-node-1 can be added to the cluster, but we don't recommend doing it.
 For best results let it be isolated to only executing `Celery` tasks in the cluster setup.
 
 
 .. _Gunicorn: http://gunicorn.org/
 .. _Whoosh: https://pypi.python.org/pypi/Whoosh/
 .. _Elasticsearch: https://www.elastic.co/..
 .. _RabbitMQ: http://www.rabbitmq.com/
 .. _Nginx: http://nginx.io
 .. _Apache: http://nginx.io
 .. _Redis: http://redis.io
 

docs/admin/system_admin/tuning/tuning-user-sessions-performance.rst

@@ -1,67 +1,67 b''
 .. _user-session-ref:
 
 User Session Performance
 ------------------------
 
 The default file-based sessions are only suitable for smaller setups, or
 instances that doesn't have a lot of users or traffic.
 They are set as default option because it's setup-free solution.
 
 The most common issue of file based sessions are file limit errors which occur
 if there are lots of session files.
 
 Therefore, in a large scale deployment, to give better performance,
 scalability, and maintainability we recommend switching from file-based
 sessions to database-based user sessions or Redis based sessions.
 
 To switch to database-based user sessions uncomment the following section in
-your :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file.
+your :file:`config/_shared/rhodecode.ini` file.
 
 
 .. code-block:: ini
 
       ## db based session, fast, and allows easy management over logged in users
       beaker.session.type = ext:database
       beaker.session.table_name = db_session
 
       # use just one of the following according to the type of database
       beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode
       # or
       beaker.session.sa.url = mysql://root:secret@127.0.0.1/rhodecode
 
       beaker.session.sa.pool_recycle = 3600
       beaker.session.sa.echo = false
 
 
 and make sure you comment out the file based sessions.
 
 .. code-block:: ini
 
       ## types are file, ext:memcached, ext:database, and memory (default).
       #beaker.session.type = file
       #beaker.session.data_dir = %(here)s/data/sessions/data
 
 
 The `table_name` will be automatically created on specified database if it isn't yet existing.
 Database specified in the `beaker.session.sa.url` can be the same that RhodeCode
 uses, or if required it can be a different one. We recommend to use the same database.
 
 
 
 To switch to redis-based user sessions uncomment the following section in
-your :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file.
+your :file:`config/_shared/rhodecode.ini` file.
 
 .. code-block:: ini
 
       ## redis sessions
       beaker.session.type = ext:redis
       beaker.session.url = localhost:6379
 
 
 and make sure you comment out the file based sessions.
 
 .. code-block:: ini
 
       ## types are file, ext:memcached, ext:database, and memory (default).
       #beaker.session.type = file
       #beaker.session.data_dir = %(here)s/data/sessions/data
  No newline at end of file

docs/admin/system_admin/vcs-server.rst

@@ -1,379 +1,379 b''
 .. _vcs-server:
 
 VCS Server Management
 ---------------------
 
 The VCS Server handles |RCE| backend functionality. You need to configure
 a VCS Server to run with a |RCE| instance. If you do not, you will be missing
 the connection between |RCE| and its |repos|. This will cause error messages
 on the web interface. You can run your setup in the following configurations,
 currently the best performance is one of following:
 
 * One VCS Server per |RCE| instance.
 * One VCS Server handling multiple instances.
 
 .. important::
 
    If your server locale settings are not correctly configured,
    |RCE| and the VCS Server can run into issues. See this `Ask Ubuntu`_ post
    which explains the problem and gives a solution.
 
 For more information, see the following sections:
 
 * :ref:`install-vcs`
 * :ref:`config-vcs`
 * :ref:`vcs-server-options`
 * :ref:`vcs-server-versions`
 * :ref:`vcs-server-maintain`
 * :ref:`vcs-server-config-file`
 * :ref:`svn-http`
 
 .. _install-vcs:
 
 VCS Server Installation
 ^^^^^^^^^^^^^^^^^^^^^^^
 
 To install a VCS Server, see
 :ref:`Installing a VCS server <control:install-vcsserver>`.
 
 .. _config-vcs:
 
 Hooking |RCE| to its VCS Server
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 To configure a |RCE| instance to use a VCS server, see
 :ref:`Configuring the VCS Server connection <control:manually-vcsserver-ini>`.
 
 .. _vcs-server-options:
 
 |RCE| VCS Server Options
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
 The following list shows the available options on the |RCE| side of the
 connection to the VCS Server. The settings are configured per
 instance in the
-:file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file.
+:file:`config/_shared/rhodecode.ini` file.
 
 .. rst-class:: dl-horizontal
 
     \vcs.backends <available-vcs-systems>
         Set a comma-separated list of the |repo| options available from the
         web interface. The default is ``hg, git, svn``,
         which is all |repo| types available. The order of backends is also the
         order backend will try to detect requests type.
 
     \vcs.connection_timeout <seconds>
         Set the length of time in seconds that the VCS Server waits for
         requests to process. After the timeout expires,
         the request is closed. The default is ``3600``. Set to a higher
         number if you experience network latency, or timeout issues with very
         large push/pull requests.
 
     \vcs.server.enable <boolean>
         Enable or disable the VCS Server. The available options are ``true`` or
         ``false``. The default is ``true``.
 
     \vcs.server <host:port>
         Set the host, either hostname or IP Address, and port of the VCS server
         you wish to run with your |RCE| instance.
 
 .. code-block:: ini
 
     ##################
     ### VCS CONFIG ###
     ##################
     # set this line to match your VCS Server
     vcs.server = 127.0.0.1:10004
     # Set to False to disable the VCS Server
     vcs.server.enable = True
     vcs.backends = hg, git, svn
     vcs.connection_timeout = 3600
 
 
 .. _vcs-server-versions:
 
 VCS Server Versions
 ^^^^^^^^^^^^^^^^^^^
 
 An updated version of the VCS Server is released with each |RCE| version. Use
 the VCS Server number that matches with the |RCE| version to pair the
 appropriate ones together. For |RCE| versions pre 3.3.0,
 VCS Server 1.X.Y works with |RCE| 3.X.Y, for example:
 
 * VCS Server 1.0.0 works with |RCE| 3.0.0
 * VCS Server 1.2.2 works with |RCE| 3.2.2
 
 For |RCE| versions post 3.3.0, the VCS Server and |RCE| version numbers
 match, for example:
 
 * VCS Server |release| works with |RCE| |release|
 
 .. _vcs-server-maintain:
 
 VCS Server Cache Optimization
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 To optimize the VCS server to manage the cache and memory usage efficiently, it's recommended to
 configure the Redis backend for VCSServer caches.
 Once configured, restart the VCS Server.
 
 Make sure Redis is installed and running.
 Open :file:`/home/{user}/.rccontrol/{vcsserver-id}/vcsserver.ini`
 file and ensure the below settings for `repo_object` type cache are set:
 
 .. code-block:: ini
 
     ; ensure the default file based cache is *commented out*
     ##rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
     ##rc_cache.repo_object.expiration_time = 2592000
 
     ; `repo_object` cache settings for vcs methods for repositories
     rc_cache.repo_object.backend = dogpile.cache.rc.redis_msgpack
 
     ; cache auto-expires after N seconds
     ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
     rc_cache.repo_object.expiration_time = 2592000
 
     ; redis_expiration_time needs to be greater then expiration_time
     rc_cache.repo_object.arguments.redis_expiration_time = 3592000
 
     rc_cache.repo_object.arguments.host = localhost
     rc_cache.repo_object.arguments.port = 6379
     rc_cache.repo_object.arguments.db = 5
     rc_cache.repo_object.arguments.socket_timeout = 30
     ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
     rc_cache.repo_object.arguments.distributed_lock = true
 
 
 To clear the cache completely, you can restart the VCS Server.
 
 .. important::
 
    While the VCS Server handles a restart gracefully on the web interface,
    it will drop connections during push/pull requests. So it is recommended
    you only perform this when there is very little traffic on the instance.
 
 Use the following example to restart your VCS Server,
 for full details see the :ref:`RhodeCode Control CLI <control:rcc-cli>`.
 
 .. code-block:: bash
 
     $ rccontrol status
 
 .. code-block:: vim
 
     - NAME: vcsserver-1
     - STATUS: RUNNING
       logs:/home/ubuntu/.rccontrol/vcsserver-1/vcsserver.log
     - VERSION: 4.7.2 VCSServer
     - URL: http://127.0.0.1:10008
     - CONFIG: /home/ubuntu/.rccontrol/vcsserver-1/vcsserver.ini
 
     $ rccontrol restart vcsserver-1
     Instance "vcsserver-1" successfully stopped.
     Instance "vcsserver-1" successfully started.
 
 .. _vcs-server-config-file:
 
 VCS Server Configuration
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
 You can configure settings for multiple VCS Servers on your
 system using their individual configuration files. Use the following
 properties inside the configuration file to set up your system. The default
 location is :file:`home/{user}/.rccontrol/{vcsserver-id}/vcsserver.ini`.
 For a more detailed explanation of the logger levers, see :ref:`debug-mode`.
 
 .. rst-class:: dl-horizontal
 
     \host <ip-address>
         Set the host on which the VCS Server will run. VCSServer is not
         protected by any authentication, so we *highly* recommend running it
         under localhost ip that is `127.0.0.1`
 
     \port <int>
         Set the port number on which the VCS Server will be available.
 
 
 .. note::
 
    After making changes, you need to restart your VCS Server to pick them up.
 
 .. code-block:: ini
 
     ; #################################
     ; RHODECODE VCSSERVER CONFIGURATION
     ; #################################
 
     [server:main]
     ; COMMON HOST/IP CONFIG
     host = 127.0.0.1
     port = 10002
 
     ; ###########################
     ; GUNICORN APPLICATION SERVER
     ; ###########################
 
     ; run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
 
     ; Module to use, this setting shouldn't be changed
     use = egg:gunicorn#main
 
     ; Sets the number of process workers. More workers means more concurrent connections
     ; RhodeCode can handle at the same time. Each additional worker also it increases
     ; memory usage as each has it's own set of caches.
     ; Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more
     ; than 8-10 unless for really big deployments .e.g 700-1000 users.
     ; `instance_id = *` must be set in the [app:main] section below (which is the default)
     ; when using more than 1 worker.
     workers = 6
 
     ; Gunicorn access log level
     loglevel = info
 
     ; Process name visible in process list
     proc_name = rhodecode_vcsserver
 
     ; Type of worker class, one of sync, gevent
     ; currently `sync` is the only option allowed.
     worker_class = sync
 
     ; The maximum number of simultaneous clients. Valid only for gevent
     worker_connections = 10
 
     ; Max number of requests that worker will handle before being gracefully restarted.
     ; Prevents memory leaks, jitter adds variability so not all workers are restarted at once.
     max_requests = 1000
     max_requests_jitter = 30
 
     ; Amount of time a worker can spend with handling a request before it
     ; gets killed and restarted. By default set to 21600 (6hrs)
     ; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
     timeout = 21600
 
     ; The maximum size of HTTP request line in bytes.
     ; 0 for unlimited
     limit_request_line = 0
 
     ; Limit the number of HTTP headers fields in a request.
     ; By default this value is 100 and can't be larger than 32768.
     limit_request_fields = 32768
 
     ; Limit the allowed size of an HTTP request header field.
     ; Value is a positive number or 0.
     ; Setting it to 0 will allow unlimited header field sizes.
     limit_request_field_size = 0
 
     ; Timeout for graceful workers restart.
     ; After receiving a restart signal, workers have this much time to finish
     ; serving requests. Workers still alive after the timeout (starting from the
     ; receipt of the restart signal) are force killed.
     ; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
     graceful_timeout = 3600
 
     # The number of seconds to wait for requests on a Keep-Alive connection.
     # Generally set in the 1-5 seconds range.
     keepalive = 2
 
     ; Maximum memory usage that each worker can use before it will receive a
     ; graceful restart signal 0 = memory monitoring is disabled
     ; Examples: 268435456 (256MB), 536870912 (512MB)
     ; 1073741824 (1GB), 2147483648 (2GB), 4294967296 (4GB)
     memory_max_usage = 1073741824
 
     ; How often in seconds to check for memory usage for each gunicorn worker
     memory_usage_check_interval = 60
 
     ; Threshold value for which we don't recycle worker if GarbageCollection
     ; frees up enough resources. Before each restart we try to run GC on worker
     ; in case we get enough free memory after that, restart will not happen.
     memory_usage_recovery_threshold = 0.8
 
 
     [app:main]
     use = egg:rhodecode-vcsserver
 
     pyramid.default_locale_name = en
     pyramid.includes =
 
     ; default locale used by VCS systems
     locale = en_US.UTF-8
 
     ; #############
     ; DOGPILE CACHE
     ; #############
 
     ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
     ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
     cache_dir = %(here)s/data
 
     ; **********************************************************
     ; `repo_object` cache with redis backend
     ; recommended for larger instance, or for better performance
     ; **********************************************************
 
     ; `repo_object` cache settings for vcs methods for repositories
     rc_cache.repo_object.backend = dogpile.cache.rc.redis_msgpack
 
     ; cache auto-expires after N seconds
     ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
     rc_cache.repo_object.expiration_time = 2592000
 
     ; redis_expiration_time needs to be greater then expiration_time
     rc_cache.repo_object.arguments.redis_expiration_time = 3592000
 
     rc_cache.repo_object.arguments.host = localhost
     rc_cache.repo_object.arguments.port = 6379
     rc_cache.repo_object.arguments.db = 5
     rc_cache.repo_object.arguments.socket_timeout = 30
     ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
     rc_cache.repo_object.arguments.distributed_lock = true
 
     ; #####################
     ; LOGGING CONFIGURATION
     ; #####################
     [loggers]
     keys = root, vcsserver
 
     [handlers]
     keys = console
 
     [formatters]
     keys = generic
 
     ; #######
     ; LOGGERS
     ; #######
     [logger_root]
     level = NOTSET
     handlers = console
 
     [logger_vcsserver]
     level = DEBUG
     handlers =
     qualname = vcsserver
     propagate = 1
 
 
     ; ########
     ; HANDLERS
     ; ########
 
     [handler_console]
     class = StreamHandler
     args = (sys.stderr, )
     level = INFO
     formatter = generic
 
     ; ##########
     ; FORMATTERS
     ; ##########
 
     [formatter_generic]
     format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
     datefmt = %Y-%m-%d %H:%M:%S
 
 
 .. _Subversion Red Book: http://svnbook.red-bean.com/en/1.7/svn-book.html#svn.ref.svn
 
 .. _Ask Ubuntu: http://askubuntu.com/questions/162391/how-do-i-fix-my-locale-issue

docs/api/api.rst

@@ -1,209 +1,209 b''
 .. _api:
 
 API Documentation
 =================
 
 The |RCE| API uses a single scheme for calling all API methods. The API is
 implemented with JSON protocol in both directions. To send API requests to
 your instance of |RCE|, use the following URL format
 ``<your_server>/_admin``
 
 .. note::
 
    To use the API, you should configure the :file:`~/.rhoderc` file with
    access details per instance. For more information, see
    :ref:`config-rhoderc`.
 
 
 API ACCESS FOR WEB VIEWS
 ------------------------
 
 API access can also be turned on for each web view in |RCE| that is
 decorated with a `@LoginRequired` decorator. To enable API access, change
 the standard login decorator to `@LoginRequired(api_access=True)`.
 
 From |RCE| version 1.7.0 you can configure a white list
 of views that have API access enabled by default. To enable these,
 edit the |RCE| configuration ``.ini`` file. The default location is:
 
 * |RCE| Pre-2.2.7 :file:`root/rhodecode/data/production.ini`
-* |RCE| 3.0 :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+* |RCE| 3.0 :file:`config/_shared/rhodecode.ini`
 
 To configure the white list, edit this section of the file. In this
 configuration example, API access is granted to the patch/diff raw file and
 archive.
 
 .. code-block:: ini
 
     ## List of controllers (using glob syntax) that AUTH TOKENS could be used for access.
     ## Adding ?auth_token = <token> to the url authenticates this request as if it
     ## came from the the logged in user who own this authentication token.
     ##
     ## Syntax is <ControllerClass>:<function_pattern>.
     ## The list should be "," separated and on a single line.
     ##
     api_access_controllers_whitelist = RepoCommitsView:repo_commit_raw,RepoCommitsView:repo_commit_patch,RepoCommitsView:repo_commit_download
 
 After this change, a |RCE| view can be accessed without login by adding a
 GET parameter ``?auth_token=<auth_token>`` to a url. For example to
 access the raw diff.
 
 .. code-block:: html
 
    http://<server>/<repo>/changeset-diff/<sha>?auth_token=<auth_token>
 
 By default this is only enabled on RSS/ATOM feed views. Exposing raw diffs is a
 good way to integrate with 3rd party services like code review, or build farms
 that could download archives.
 
 API ACCESS
 ----------
 
 All clients are required to send JSON-RPC spec JSON data.
 
 .. code-block:: bash
 
     {
         "id:"<id>",
         "auth_token":"<auth_token>",
         "method":"<method_name>",
         "args":{"<arg_key>":"<arg_val>"}
     }
 
 Example call for auto pulling from remote repositories using curl:
 
 .. code-block:: bash
 
     curl https://server.com/_admin/api -X POST -H 'content-type:text/plain' --data-binary '{"id":1,
     "auth_token":"xe7cdb2v278e4evbdf5vs04v832v0efvcbcve4a3","method":"pull", "args":{"repoid":"CPython"}}'
 
 Provide those parameters:
  - **id** A value of any type, which is used to match the response with the
    request that it is replying to.
  - **auth_token** for access and permission validation.
  - **method** is name of method to call
  - **args** is an ``key:value`` list of arguments to pass to method
 
 .. note::
 
     To get your |authtoken|, from the |RCE| interface,
     go to:
     :menuselection:`username --> My account --> Auth tokens`
 
     For security reasons you should always create a dedicated |authtoken| for
     API use only.
 
 
 The |RCE| API will always return a JSON-RPC response:
 
 .. code-block:: bash
 
     {
         "id": <id>, # matching id sent by request
         "result": "<result>"|null, # JSON formatted result, null if any errors
         "error": "null"|<error_message> # JSON formatted error (if any)
     }
 
 All responses from API will be with `HTTP/1.0 200 OK` status code.
 If there is an error when calling the API, the *error* key will contain a
 failure description and the *result* will be `null`.
 
 API CLIENT
 ----------
 
 To install the |RCE| API, see :ref:`install-tools`. To configure the API per
 instance, see the :ref:`rc-tools` section as you need to configure a
 :file:`~/.rhoderc` file with your |authtokens|.
 
 Once you have set up your instance API access, use the following examples to
 get started.
 
 .. code-block:: bash
 
     # Getting the 'rhodecode' repository
     # from a RhodeCode Enterprise instance
     rhodecode-api --instance-name=enterprise-1 get_repo repoid:rhodecode
 
     Calling method get_repo => http://127.0.0.1:5000
     Server response
     {
         <json data>
     }
 
     # Creating a new mercurial repository called 'brand-new'
     # with a description 'Repo-description'
     rhodecode-api --instance-name=enterprise-1 create_repo repo_name:brand-new repo_type:hg description:Repo-description
     {
       "error": null,
       "id": 1110,
       "result": {
         "msg": "Created new repository `brand-new`",
         "success": true,
         "task": null
       }
     }
 
 A broken example, what not to do.
 
 .. code-block:: bash
 
     # A call missing the required arguments
     # and not specifying the instance
     rhodecode-api get_repo
 
     Calling method get_repo => http://127.0.0.1:5000
     Server response
     "Missing non optional `repoid` arg in JSON DATA"
 
 You can specify pure JSON using the ``--format`` parameter.
 
 .. code-block:: bash
 
     rhodecode-api --format=json get_repo repoid:rhodecode
 
 In such case only output that this function shows is pure JSON, we can use that
 and pipe output to some json formatter.
 
 If output is in pure JSON format, you can pipe output to a JSON formatter.
 
 .. code-block:: bash
 
     rhodecode-api --instance-name=enterprise-1 --format=json get_repo repoid:rhodecode | python -m json.tool
 
 API METHODS
 -----------
 
 Each method by default required following arguments.
 
 .. code-block:: bash
 
     id :      "<id_for_response>"
     auth_token : "<auth_token>"
     method :  "<method name>"
     args :    {}
 
 Use each **param** from docs and put it in args, Optional parameters
 are not required in args.
 
 .. code-block:: bash
 
     args: {"repoid": "rhodecode"}
 
 .. Note: From this point on things are generated by the script in
    `scripts/fabfile.py`. To change things below, update the docstrings in the
    ApiController.
 
 .. --- API DEFS MARKER ---
 .. toctree::
 
    methods/repo-methods
    methods/store-methods
    methods/license-methods
    methods/deprecated-methods
    methods/gist-methods
    methods/pull-request-methods
    methods/repo-group-methods
    methods/search-methods
    methods/server-methods
    methods/user-methods
    methods/user-group-methods

docs/auth/ssh-connection.rst

@@ -1,144 +1,144 b''
 .. _ssh-connection:
 
 SSH Connection
 --------------
 
 If you wish to connect to your |repos| using SSH protocol, use the
 following instructions.
 
 1. Include |RCE| generated `authorized_keys` file into your sshd_config.
 
    By default a file `authorized_keys_rhodecode` is created containing
    configuration and all allowed user connection keys are stored inside.
    On each change of stored keys inside |RCE| this file is updated with
    proper data.
 
    .. code-block:: bash
 
        # Edit sshd_config file most likely at /etc/ssh/sshd_config
        # add or edit the AuthorizedKeysFile, and set to use custom files
 
        AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode
 
    This way we use a separate file for SSH access and separate one for
    SSH access to |RCE| repositories.
 
 
 2. Enable the SSH module on instance.
 
    On the server where |RCE| is running executing:
 
    .. code-block:: bash
 
        rccontrol enable-module ssh {instance-id}
 
    This will add the following configuration into :file:`rhodecode.ini`.
    This also can be done manually:
 
    .. code-block:: ini
 
         ############################################################
         ### SSH Support Settings                                 ###
         ############################################################
 
         ## Defines if a custom authorized_keys file should be created and written on
         ## any change user ssh keys. Setting this to false also disables posibility
         ## of adding SSH keys by users from web interface. Super admins can still
         ## manage SSH Keys.
         ssh.generate_authorized_keyfile = true
 
         ## Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding`
         # ssh.authorized_keys_ssh_opts =
 
         ## Path to the authrozied_keys file where the generate entries are placed.
         ## It is possible to have multiple key files specified in `sshd_config` e.g.
         ## AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode
         ssh.authorized_keys_file_path = ~/.ssh/authorized_keys_rhodecode
 
         ## Command to execute the SSH wrapper. The binary is available in the
         ## rhodecode installation directory.
         ## e.g ~/.rccontrol/community-1/profile/bin/rc-ssh-wrapper
         ssh.wrapper_cmd = ~/.rccontrol/community-1/rc-ssh-wrapper
 
         ## Allow shell when executing the ssh-wrapper command
         ssh.wrapper_cmd_allow_shell = false
 
         ## Enables logging, and detailed output send back to the client during SSH
         ## operations. Useful for debugging, shouldn't be used in production.
         ssh.enable_debug_logging = false
 
         ## Paths to binary executable, by default they are the names, but we can
         ## override them if we want to use a custom one
         ssh.executable.hg = ~/.rccontrol/vcsserver-1/profile/bin/hg
         ssh.executable.git = ~/.rccontrol/vcsserver-1/profile/bin/git
         ssh.executable.svn = ~/.rccontrol/vcsserver-1/profile/bin/svnserve
 
         ## Enables SSH key generator web interface. Disabling this still allows users
         ## to add their own keys.
         ssh.enable_ui_key_generator = true
 
 
 3. Set base_url for instance to enable proper event handling (Optional):
 
    If you wish to have integrations working correctly via SSH please configure
    The Application base_url.
 
    Use the ``rccontrol status`` command to view instance details.
    Hostname is required for the integration to properly set the instance URL.
 
    When your hostname is known (e.g https://code.rhodecode.com) please set it
-   inside :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+   inside :file:`config/_shared/rhodecode.ini`
 
    add into `[app:main]` section the following configuration:
 
    .. code-block:: ini
 
        app.base_url = https://code.rhodecode.com
 
 
 4. Add the public key to your user account for testing.
    First generate a new key, or use your existing one and have your public key
    at hand.
 
    Go to
    :menuselection:`My Account --> SSH Keys` and add the public key with proper description.
 
    This will generate a new entry inside our configured `authorized_keys_rhodecode` file.
 
    Test the connection from your local machine using the following example:
 
    .. note::
 
        In case of connection problems please set
        `ssh.enable_debug_logging = true` inside the SSH configuration of
-       :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+       :file:`config/_shared/rhodecode.ini`
        Then add, remove your SSH key and try connecting again.
        Debug logging will be printed to help find the problems on the server side.
 
    Test connection using the ssh command from the local machine. Make sure
    to use the use who is running the |RCE| server, and not your username from
    the web interface.
 
 
    For SVN:
 
    .. code-block:: bash
 
        SVN_SSH="ssh -i ~/.ssh/id_rsa_test_ssh_private.key" svn checkout svn+ssh://rhodecode@rc-server/repo_name
 
    For GIT:
 
    .. code-block:: bash
 
        GIT_SSH_COMMAND='ssh -i ~/.ssh/id_rsa_test_ssh_private.key' git clone ssh://rhodecode@rc-server/repo_name
 
    For Mercurial:
 
    .. code-block:: bash
 
        Add to hgrc:
 
        [ui]
        ssh = ssh -C -i ~/.ssh/id_rsa_test_ssh_private.key
 
        hg clone ssh://rhodecode@rc-server/repo_name

docs/install/setup-email.rst

@@ -1,45 +1,45 b''
 .. _set-up-mail:
 
 Set up Email
 ------------
 
 To setup email with your |RCE| instance, open the default
-:file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+:file:`config/_shared/rhodecode.ini`
 file and uncomment and configure the email section. If it is not there,
 use the below example to insert it.
 
 Once configured you can check the settings for your |RCE| instance on the
 :menuselection:`Admin --> Settings --> Email` page.
 
 Please be aware that both section should be changed the `[DEFAULT]` for main applications
 email config, and `[server:main]` for exception tracking email
 
 .. code-block:: ini
 
     [DEFAULT]
     ; ########################################################################
     ; EMAIL CONFIGURATION
     ; These settings will be used by the RhodeCode mailing system
     ; ########################################################################
 
     ; prefix all emails subjects with given prefix, helps filtering out emails
     #email_prefix = [RhodeCode]
 
     ; email FROM address all mails will be sent
     #app_email_from = rhodecode-noreply@localhost
 
     #smtp_server = mail.server.com
     #smtp_username =
     #smtp_password =
     #smtp_port =
     #smtp_use_tls = false
     #smtp_use_ssl = true
 
     [server:main]
     ; Send email with exception details when it happens
     #exception_tracker.send_email = true
 
     ; Comma separated list of recipients for exception emails,
     ; e.g [e-mail],[e-mail]
     ; Can be left empty, then emails will be sent to ALL super-admins
     #exception_tracker.send_email_recipients =

docs/release-notes/release-notes-5.0.0.rst

@@ -1,51 +1,52 b''
 |RCE| 5.0.0 |RNS|
 -----------------
 
 Release Date
 ^^^^^^^^^^^^
 
-- TBA
+
+- 2024-05-14
 
 
 New Features
 ^^^^^^^^^^^^
 
 - Full support of Python3 and Python3.11
 - Git repositories with LFS object are now pushing and pulling the LFS objects when remote sync is enabled.
-- Archive generation: implemented a new system for caching generated archive files that allows setting cache size limit
-  see: `archive_cache.cache_size_gb=` option.
+- Archive generation: implemented a new system for caching generated archive files that allows setting cache size limit see: archive_cache.cache_size_gb= option.
 - Introduced statsd metrics in various places for new monitoring stack to provide useful details on traffic and usage.
 
 
 General
 ^^^^^^^
 
-- Upgraded all dependency libraries to their latest available versions
-- Dropped support for deprecated hgsubversion no longer available in python3
+- Upgraded all dependency libraries to their latest available versions for python3 compatability
 
 
 Security
 ^^^^^^^^
 
+- fixed few edge cases of permission invalidation on change of permissions
 
 
 Performance
 ^^^^^^^^^^^
 
 - Moved lot of dulwich based code into libgit2 implementation for better performance
 
 
 Fixes
 ^^^^^
 
+- Various small fixes and improvements found during python3 migration
 
 
 Upgrade notes
 ^^^^^^^^^^^^^
 
 - RhodeCode 5.0.0 is a mayor upgrade that support latest Python versions 3.11,
   and also comes with lots of internal changes and performance improvements.
 - RhodeCode 5.0.0 is only supported via our new docker rcstack installer or helm/k8s stack
 - Please see migration guide :ref:`rcstack:migration-to-rcstack`
 - RhodeCode 5.0.0 also requires a new license for EE instances, please reach out to us via [e-mail] to
   convert your license key.

docs/release-notes/release-notes.rst

@@ -1,168 +1,173 b''
 .. _rhodecode-release-notes-ref:
 
 Release Notes
 =============
 
 |RCE| 5.x Versions
 ------------------
 
 .. toctree::
    :maxdepth: 1
 
+
+   release-notes-5.1.0.rst
+   release-notes-5.0.3.rst
+   release-notes-5.0.2.rst
+   release-notes-5.0.1.rst
    release-notes-5.0.0.rst
 
 
 |RCE| 4.x Versions
 ------------------
 
 .. toctree::
    :maxdepth: 1
 
    release-notes-4.27.1.rst
    release-notes-4.27.0.rst
    release-notes-4.26.0.rst
    release-notes-4.25.2.rst
    release-notes-4.25.1.rst
    release-notes-4.25.0.rst
    release-notes-4.24.1.rst
    release-notes-4.24.0.rst
    release-notes-4.23.2.rst
    release-notes-4.23.1.rst
    release-notes-4.23.0.rst
    release-notes-4.22.0.rst
    release-notes-4.21.0.rst
    release-notes-4.20.1.rst
    release-notes-4.20.0.rst
    release-notes-4.19.3.rst
    release-notes-4.19.2.rst
    release-notes-4.19.1.rst
    release-notes-4.19.0.rst
    release-notes-4.18.3.rst
    release-notes-4.18.2.rst
    release-notes-4.18.1.rst
    release-notes-4.18.0.rst
    release-notes-4.17.4.rst
    release-notes-4.17.3.rst
    release-notes-4.17.2.rst
    release-notes-4.17.1.rst
    release-notes-4.17.0.rst
    release-notes-4.16.2.rst
    release-notes-4.16.1.rst
    release-notes-4.16.0.rst
    release-notes-4.15.2.rst
    release-notes-4.15.1.rst
    release-notes-4.15.0.rst
    release-notes-4.14.1.rst
    release-notes-4.14.0.rst
    release-notes-4.13.3.rst
    release-notes-4.13.2.rst
    release-notes-4.13.1.rst
    release-notes-4.13.0.rst
    release-notes-4.12.4.rst
    release-notes-4.12.3.rst
    release-notes-4.12.2.rst
    release-notes-4.12.1.rst
    release-notes-4.12.0.rst
    release-notes-4.11.6.rst
    release-notes-4.11.5.rst
    release-notes-4.11.4.rst
    release-notes-4.11.3.rst
    release-notes-4.11.2.rst
    release-notes-4.11.1.rst
    release-notes-4.11.0.rst
    release-notes-4.10.6.rst
    release-notes-4.10.5.rst
    release-notes-4.10.4.rst
    release-notes-4.10.3.rst
    release-notes-4.10.2.rst
    release-notes-4.10.1.rst
    release-notes-4.10.0.rst
    release-notes-4.9.1.rst
    release-notes-4.9.0.rst
    release-notes-4.8.0.rst
    release-notes-4.7.2.rst
    release-notes-4.7.1.rst
    release-notes-4.7.0.rst
    release-notes-4.6.1.rst
    release-notes-4.6.0.rst
    release-notes-4.5.2.rst
    release-notes-4.5.1.rst
    release-notes-4.5.0.rst
    release-notes-4.4.2.rst
    release-notes-4.4.1.rst
    release-notes-4.4.0.rst
    release-notes-4.3.1.rst
    release-notes-4.3.0.rst
    release-notes-4.2.1.rst
    release-notes-4.2.0.rst
    release-notes-4.1.2.rst
    release-notes-4.1.1.rst
    release-notes-4.1.0.rst
    release-notes-4.0.1.rst
    release-notes-4.0.0.rst
 
 |RCE| 3.x Versions
 ------------------
 
 .. toctree::
    :maxdepth: 1
 
    release-notes-3.8.4.rst
    release-notes-3.8.3.rst
    release-notes-3.8.2.rst
    release-notes-3.8.1.rst
    release-notes-3.8.0.rst
    release-notes-3.7.1.rst
    release-notes-3.7.0.rst
    release-notes-3.6.1.rst
    release-notes-3.6.0.rst
    release-notes-3.5.2.rst
    release-notes-3.5.1.rst
    release-notes-3.5.0.rst
    release-notes-3.4.1.rst
    release-notes-3.4.0.rst
    release-notes-3.3.4.rst
    release-notes-3.3.3.rst
    release-notes-3.3.2.rst
    release-notes-3.3.1.rst
    release-notes-3.3.0.rst
    release-notes-3.2.3.rst
    release-notes-3.2.2.rst
    release-notes-3.2.1.rst
    release-notes-3.2.0.rst
    release-notes-3.1.1.rst
    release-notes-3.1.0.rst
    release-notes-3.0.2.rst
    release-notes-3.0.1.rst
    release-notes-3.0.0.rst
 
 |RCE| 2.x Versions
 ------------------
 
 .. toctree::
    :maxdepth: 1
 
    release-notes-2.2.8.rst
    release-notes-2.2.7.rst
    release-notes-2.2.6.rst
    release-notes-2.2.5.rst
    release-notes-2.2.4.rst
    release-notes-2.2.3.rst
    release-notes-2.2.2.rst
    release-notes-2.2.1.rst
    release-notes-2.2.0.rst
    release-notes-2.1.0.rst
    release-notes-2.0.2.rst
    release-notes-2.0.1.rst
    release-notes-2.0.0.rst
 
 |RCE| 1.x Versions
 ------------------
 
 .. toctree::
    :maxdepth: 1
 
    release-notes-1.7.2.rst
    release-notes-1.7.1.rst
    release-notes-1.7.0.rst
    release-notes-1.6.0.rst

docs/tutorials/multi-instance-setup.rst

@@ -1,91 +1,91 b''
 .. _multi-instance-setup:
 
 Scaling |RCE| Using Multiple Instances
 ======================================
 
 Running multiple instances of |RCE| from a single database can be used to
 scale the application for the following deployment setups:
 
 * Using dedicated Continuous Integrations instances.
 * Locating instances closer to geographically dispersed development teams.
 * Running production and testing instances, or failover instances on a
   different server.
 * Running proxy read-only instances for pull operations.
 
 If you wish to run multiple instances of |RCE| using a single database for
 settings, use the following instructions to set this up. Before you get onto
 multiple instances though, you should install |RCE|, and set
 up your first instance as you see fit. You can see the full instructions here
 :ref:`Installing RhodeCode Enterprise <control:rcc>`
 
 Once you have configured your first instance, you can run additional instances
 from the same database using the following steps:
 
 1. Install a new instance of |RCE|, choosing SQLite as the database. It is
    important to choose SQLite, because this will not overwrite any other
    database settings you may have.
 
    Once the new instance is installed you need to update the licence token and
    database connection string in the
-   :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file.
+   :file:`config/_shared/rhodecode.ini` file.
 
 .. code-block:: bash
 
    $ rccontrol install Enterprise
 
     Agree to the licence agreement? [y/N]: y
     Username [admin]: username
     Password (min 6 chars):
     Repeat for confirmation:
     Email: [e-mail]
     Respositories location [/home/brian/repos]:
     IP to start the Enterprise server on [127.0.0.1]:
     Port for the Enterprise server to use [10000]:
     Database type - [s]qlite, [m]ysql, [p]ostresql: s
 
 2. The licence token used on each new instance needs to be the token from your
    initial instance. This allows multiple instances to run the same licence key.
 
    To get the licence token, go to the |RCE| interface of your primary
    instance and select :menuselection:`admin --> setting --> license`. Then
    update the licence token setting in each new instance's
    :file:`rhodecode.ini` file.
 
 .. code-block:: ini
 
     ## generated license token, goto license page in RhodeCode settings to get
     ## new token
     license_token = add-token-here
 
 3. Update the database connection string in the
    :file:`rhodecode.ini` file to point to your database. For
    more information, see :ref:`config-database`.
 
 .. code-block:: ini
 
     #########################################################
     ### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###
     #########################################################
 
     # Default SQLite config
     sqlalchemy.db1.url = sqlite:////home/user/.rccontrol/enterprise-1/rhodecode.db
 
     # Use this example for a PostgreSQL
     sqlalchemy.db1.url = postgresql://username:password@localhost/rhodecode
 
 4. Restart your updated instance. Once restarted the new instance will read
    the licence key in the database and will function identically as the
    original instance.
 
 .. code-block:: bash
 
    $ rccontrol restart enterprise-2
 
 If you wish to add additional performance to your setup, see the
 :ref:`rhodecode-tuning-ref` section.
 
 Scaling Deployment Diagram
 --------------------------
 
 .. image:: ../images/scaling-diagrm.png
    :align: center

package.json

@@ -1,61 +1,60 b''
 {
   "name": "rhodecode-enterprise",
   "version": "5.0.0",
   "private": true,
   "description": "RhodeCode JS packaged",
   "license": "SEE LICENSE IN LICENSE.txt",
   "repository": {
     "type": "hg",
     "url": "https://code.rhodecode.com/rhodecode-enterprise-ce"
   },
   "devDependencies": {
     "@polymer/iron-a11y-keys": "^3.0.0",
     "@polymer/iron-ajax": "^3.0.0",
     "@polymer/iron-autogrow-textarea": "^3.0.0",
     "@polymer/paper-button": "^3.0.0",
     "@polymer/paper-spinner": "^3.0.0",
     "@polymer/paper-toast": "^3.0.0",
     "@polymer/paper-toggle-button": "^3.0.0",
     "@polymer/paper-tooltip": "^3.0.0",
     "@polymer/polymer": "^3.0.0",
     "@webcomponents/webcomponentsjs": "^2.0.0",
     "babel-core": "^6.26.3",
     "babel-loader": "^7.1.2",
     "babel-plugin-transform-object-rest-spread": "^6.26.0",
     "babel-preset-env": "^1.6.0",
     "clipboard": "^2.0.1",
     "copy-webpack-plugin": "^4.4.2",
     "css-loader": "^0.28.11",
     "dropzone": "^5.5.0",
     "exports-loader": "^0.6.4",
     "favico.js": "^0.3.10",
     "grunt": "^0.4.5",
     "grunt-cli": "^1.4.3",
     "grunt-contrib-concat": "^0.5.1",
     "grunt-contrib-copy": "^1.0.0",
     "grunt-contrib-jshint": "^0.12.0",
     "grunt-contrib-less": "^1.1.0",
     "grunt-contrib-uglify": "^4.0.1",
     "grunt-contrib-watch": "^0.6.1",
     "grunt-webpack": "^3.1.3",
     "html-loader": "^0.4.4",
     "html-webpack-plugin": "^3.2.0",
     "imports-loader": "^0.7.1",
     "jquery": "1.11.3",
     "jshint": "^2.9.1-rc3",
     "mark.js": "8.11.1",
     "moment": "^2.18.1",
     "mousetrap": "^1.6.1",
     "polymer-webpack-loader": "^2.0.1",
-    "qrious": "^4.0.2",
     "raw-loader": "1.0.0-beta.0",
     "sticky-sidebar": "3.3.1",
     "style-loader": "^0.21.0",
     "sweetalert2": "^9.10.12",
     "ts-loader": "^1.3.3",
     "waypoints": "4.0.1",
     "webpack": "4.23.1",
     "webpack-cli": "3.1.2",
     "webpack-uglify-js-plugin": "^1.1.9"
   }
 }

pytest.ini

@@ -1,23 +1,27 b''
 [pytest]
 testpaths = rhodecode
 norecursedirs = rhodecode/public rhodecode/templates tests/scripts
 cache_dir = /tmp/.pytest_cache
 
 pyramid_config = rhodecode/tests/rhodecode.ini
 vcsserver_protocol = http
 vcsserver_config_http = rhodecode/tests/vcsserver_http.ini
 
 addopts =
   --pdbcls=IPython.terminal.debugger:TerminalPdb
   --strict-markers
   --capture=no
   --show-capture=all
 
 #  --test-loglevel=INFO, show log-level during execution
 
 markers =
   vcs_operations: Mark tests depending on a running RhodeCode instance.
   xfail_backends: Mark tests as xfail for given backends.
   skip_backends: Mark tests as skipped for given backends.
   backends: Mark backends
   dbs: database markers for running tests for given DB
+
+env =
+    RC_TEST=1
+    RUN_ENV=test

requirements.txt

@@ -1,295 +1,313 b''
 # deps, generated via pipdeptree --exclude setuptools,wheel,pipdeptree,pip -f | tr '[:upper:]' '[:lower:]'
 
-alembic==1.12.1
+alembic==1.13.1
   mako==1.2.4
     markupsafe==2.1.2
-  sqlalchemy==1.4.51
+  sqlalchemy==1.4.52
     greenlet==3.0.3
   typing_extensions==4.9.0
 async-timeout==4.0.3
 babel==2.12.1
 beaker==1.12.1
 celery==5.3.6
   billiard==4.2.0
   click==8.1.3
   click-didyoumean==0.3.0
     click==8.1.3
   click-plugins==1.1.1
     click==8.1.3
   click-repl==0.2.0
     click==8.1.3
     prompt-toolkit==3.0.38
       wcwidth==0.2.6
     six==1.16.0
   kombu==5.3.5
     amqp==5.2.0
       vine==5.1.0
     vine==5.1.0
   python-dateutil==2.8.2
     six==1.16.0
-  tzdata==2023.4
+  tzdata==2024.1
   vine==5.1.0
 channelstream==0.7.1
   gevent==24.2.1
     greenlet==3.0.3
     zope.event==5.0.0
-    zope.interface==6.1.0
+    zope.interface==6.3.0
   itsdangerous==1.1.0
   marshmallow==2.18.0
   pyramid==2.0.2
     hupper==1.12
     plaster==1.1.2
     plaster-pastedeploy==1.0.1
       pastedeploy==3.1.0
       plaster==1.1.2
     translationstring==1.4
     venusian==3.0.0
     webob==1.8.7
     zope.deprecation==5.0.0
-    zope.interface==6.1.0
-  pyramid-apispec==0.3.3
-    apispec==1.3.3
+    zope.interface==6.3.0
   pyramid-jinja2==2.10
     jinja2==3.1.2
       markupsafe==2.1.2
     markupsafe==2.1.2
     pyramid==2.0.2
       hupper==1.12
       plaster==1.1.2
       plaster-pastedeploy==1.0.1
         pastedeploy==3.1.0
         plaster==1.1.2
       translationstring==1.4
       venusian==3.0.0
       webob==1.8.7
       zope.deprecation==5.0.0
-      zope.interface==6.1.0
+      zope.interface==6.3.0
     zope.deprecation==5.0.0
   python-dateutil==2.8.2
     six==1.16.0
   requests==2.28.2
     certifi==2022.12.7
     charset-normalizer==3.1.0
     idna==3.4
     urllib3==1.26.14
   ws4py==0.5.1
 deform==2.0.15
   chameleon==3.10.2
   colander==2.0
     iso8601==1.1.0
     translationstring==1.4
   iso8601==1.1.0
   peppercorn==0.6
   translationstring==1.4
   zope.deprecation==5.0.0
-diskcache==5.6.3
 docutils==0.19
-dogpile.cache==1.3.0
+dogpile.cache==1.3.3
   decorator==5.1.1
   stevedore==5.1.0
     pbr==5.11.1
 formencode==2.1.0
   six==1.16.0
+fsspec==2024.6.0
 gunicorn==21.2.0
-  packaging==23.1
+  packaging==24.0
 gevent==24.2.1
   greenlet==3.0.3
   zope.event==5.0.0
-  zope.interface==6.1.0
+  zope.interface==6.3.0
 ipython==8.14.0
   backcall==0.2.0
   decorator==5.1.1
   jedi==0.19.0
     parso==0.8.3
   matplotlib-inline==0.1.6
     traitlets==5.9.0
   pexpect==4.8.0
     ptyprocess==0.7.0
   pickleshare==0.7.5
   prompt-toolkit==3.0.38
     wcwidth==0.2.6
   pygments==2.15.1
   stack-data==0.6.2
     asttokens==2.2.1
       six==1.16.0
     executing==1.2.0
     pure-eval==0.2.2
   traitlets==5.9.0
 markdown==3.4.3
-msgpack==1.0.7
+msgpack==1.0.8
 mysqlclient==2.1.1
 nbconvert==7.7.3
-  beautifulsoup4==4.11.2
-    soupsieve==2.4
+  beautifulsoup4==4.12.3
+    soupsieve==2.5
   bleach==6.1.0
     six==1.16.0
     webencodings==0.5.1
   defusedxml==0.7.1
   jinja2==3.1.2
     markupsafe==2.1.2
   jupyter_core==5.3.1
     platformdirs==3.10.0
     traitlets==5.9.0
   jupyterlab-pygments==0.2.2
   markupsafe==2.1.2
   mistune==2.0.5
   nbclient==0.8.0
     jupyter_client==8.3.0
       jupyter_core==5.3.1
         platformdirs==3.10.0
         traitlets==5.9.0
       python-dateutil==2.8.2
         six==1.16.0
       pyzmq==25.0.0
       tornado==6.2
       traitlets==5.9.0
     jupyter_core==5.3.1
       platformdirs==3.10.0
       traitlets==5.9.0
     nbformat==5.9.2
       fastjsonschema==2.18.0
       jsonschema==4.18.6
         attrs==22.2.0
         pyrsistent==0.19.3
       jupyter_core==5.3.1
         platformdirs==3.10.0
         traitlets==5.9.0
       traitlets==5.9.0
     traitlets==5.9.0
   nbformat==5.9.2
     fastjsonschema==2.18.0
     jsonschema==4.18.6
       attrs==22.2.0
       pyrsistent==0.19.3
     jupyter_core==5.3.1
       platformdirs==3.10.0
       traitlets==5.9.0
     traitlets==5.9.0
-  packaging==23.1
   pandocfilters==1.5.0
   pygments==2.15.1
   tinycss2==1.2.1
     webencodings==0.5.1
   traitlets==5.9.0
-orjson==3.9.13
-pastescript==3.4.0
-  paste==3.7.1
-    six==1.16.0
-  pastedeploy==3.1.0
-  six==1.16.0
+orjson==3.10.3
+paste==3.10.1
 premailer==3.10.0
-  cachetools==5.3.2
+  cachetools==5.3.3
   cssselect==1.2.0
   cssutils==2.6.0
   lxml==4.9.3
   requests==2.28.2
     certifi==2022.12.7
     charset-normalizer==3.1.0
     idna==3.4
     urllib3==1.26.14
 psutil==5.9.8
 psycopg2==2.9.9
 py-bcrypt==0.4
 pycmarkgfm==1.2.0
   cffi==1.16.0
     pycparser==2.21
 pycryptodome==3.17
-pycurl==7.45.2
+pycurl==7.45.3
 pymysql==1.0.3
 pyotp==2.8.0
 pyparsing==3.1.1
-pyramid-debugtoolbar==4.11
+pyramid-debugtoolbar==4.12.1
   pygments==2.15.1
   pyramid==2.0.2
     hupper==1.12
     plaster==1.1.2
     plaster-pastedeploy==1.0.1
       pastedeploy==3.1.0
       plaster==1.1.2
     translationstring==1.4
     venusian==3.0.0
     webob==1.8.7
     zope.deprecation==5.0.0
-    zope.interface==6.1.0
+    zope.interface==6.3.0
   pyramid-mako==1.1.0
     mako==1.2.4
       markupsafe==2.1.2
     pyramid==2.0.2
       hupper==1.12
       plaster==1.1.2
       plaster-pastedeploy==1.0.1
         pastedeploy==3.1.0
         plaster==1.1.2
       translationstring==1.4
       venusian==3.0.0
       webob==1.8.7
       zope.deprecation==5.0.0
-      zope.interface==6.1.0
+      zope.interface==6.3.0
 pyramid-mailer==0.15.1
   pyramid==2.0.2
     hupper==1.12
     plaster==1.1.2
     plaster-pastedeploy==1.0.1
       pastedeploy==3.1.0
       plaster==1.1.2
     translationstring==1.4
     venusian==3.0.0
     webob==1.8.7
     zope.deprecation==5.0.0
-    zope.interface==6.1.0
+    zope.interface==6.3.0
   repoze.sendmail==4.4.1
     transaction==3.1.0
-      zope.interface==6.1.0
-    zope.interface==6.1.0
+      zope.interface==6.3.0
+    zope.interface==6.3.0
   transaction==3.1.0
-    zope.interface==6.1.0
+    zope.interface==6.3.0
 python-ldap==3.4.3
   pyasn1==0.4.8
   pyasn1-modules==0.2.8
     pyasn1==0.4.8
 python-memcached==1.59
   six==1.16.0
 python-pam==2.0.2
 python3-saml==1.15.0
   isodate==0.6.1
     six==1.16.0
   lxml==4.9.3
   xmlsec==1.3.13
     lxml==4.9.3
 pyyaml==6.0.1
-redis==5.0.1
+redis==5.0.4
+  async-timeout==4.0.3
 regex==2022.10.31
 routes==2.5.1
   repoze.lru==0.7
   six==1.16.0
-simplejson==3.19.1
+s3fs==2024.6.0
+  aiobotocore==2.13.0
+    aiohttp==3.9.5
+      aiosignal==1.3.1
+        frozenlist==1.4.1
+      attrs==22.2.0
+      frozenlist==1.4.1
+      multidict==6.0.5
+      yarl==1.9.4
+        idna==3.4
+        multidict==6.0.5
+    aioitertools==0.11.0
+    botocore==1.34.106
+      jmespath==1.0.1
+      python-dateutil==2.8.2
+        six==1.16.0
+      urllib3==1.26.14
+    wrapt==1.16.0
+  aiohttp==3.9.5
+    aiosignal==1.3.1
+      frozenlist==1.4.1
+    attrs==22.2.0
+    frozenlist==1.4.1
+    multidict==6.0.5
+    yarl==1.9.4
+      idna==3.4
+      multidict==6.0.5
+  fsspec==2024.6.0
+simplejson==3.19.2
 sshpubkeys==3.3.1
   cryptography==40.0.2
     cffi==1.16.0
       pycparser==2.21
   ecdsa==0.18.0
     six==1.16.0
-sqlalchemy==1.4.51
+sqlalchemy==1.4.52
   greenlet==3.0.3
   typing_extensions==4.9.0
 supervisor==4.2.5
 tzlocal==4.3
   pytz-deprecation-shim==0.1.0.post0
-    tzdata==2023.4
+    tzdata==2024.1
+tempita==0.5.2
 unidecode==1.3.6
 urlobject==2.4.3
 waitress==3.0.0
-weberror==0.13.1
-  paste==3.7.1
-    six==1.16.0
-  pygments==2.15.1
-  tempita==0.5.2
-  webob==1.8.7
-webhelpers2==2.0
+webhelpers2==2.1
   markupsafe==2.1.2
   six==1.16.0
 whoosh==2.7.4
 zope.cachedescriptors==5.0.0
+qrcode==7.4.2
 
 ## uncomment to add the debug libraries
 #-r requirements_debug.txt

requirements_test.txt

@@ -1,46 +1,48 b''
 # test related requirements
-
-cov-core==1.15.0
-  coverage==7.2.3
-mock==5.0.2
-py==1.11.0
-pytest-cov==4.0.0
-  coverage==7.2.3
-  pytest==7.3.1
-    attrs==22.2.0
+mock==5.1.0
+pytest-cov==4.1.0
+  coverage==7.4.3
+  pytest==8.1.1
     iniconfig==2.0.0
-    packaging==23.1
-    pluggy==1.0.0
-pytest-rerunfailures==12.0
+    packaging==24.0
+    pluggy==1.4.0
+pytest-env==1.1.3
+  pytest==8.1.1
+    iniconfig==2.0.0
+    packaging==24.0
+    pluggy==1.4.0
 pytest-profiling==1.7.0
   gprof2dot==2022.7.29
-  pytest==7.3.1
-    attrs==22.2.0
+  pytest==8.1.1
     iniconfig==2.0.0
-    packaging==23.1
-    pluggy==1.0.0
+    packaging==24.0
+    pluggy==1.4.0
   six==1.16.0
-pytest-runner==6.0.0
-pytest-sugar==0.9.7
-  packaging==23.1
-  pytest==7.3.1
-    attrs==22.2.0
+pytest-rerunfailures==13.0
+  packaging==24.0
+  pytest==8.1.1
     iniconfig==2.0.0
-    packaging==23.1
-    pluggy==1.0.0
-  termcolor==2.3.0
-pytest-timeout==2.1.0
-  pytest==7.3.1
-    attrs==22.2.0
+    packaging==24.0
+    pluggy==1.4.0
+pytest-runner==6.0.1
+pytest-sugar==1.0.0
+  packaging==24.0
+  pytest==8.1.1
     iniconfig==2.0.0
-    packaging==23.1
-    pluggy==1.0.0
+    packaging==24.0
+    pluggy==1.4.0
+  termcolor==2.4.0
+pytest-timeout==2.3.1
+  pytest==8.1.1
+    iniconfig==2.0.0
+    packaging==24.0
+    pluggy==1.4.0
 webtest==3.0.0
-  beautifulsoup4==4.11.2
-    soupsieve==2.4
+  beautifulsoup4==4.12.3
+    soupsieve==2.5
   waitress==3.0.0
   webob==1.8.7
 
 # RhodeCode test-data
 rc_testdata @ https://code.rhodecode.com/upstream/rc-testdata-dist/raw/77378e9097f700b4c1b9391b56199fe63566b5c9/rc_testdata-0.11.0.tar.gz#egg=rc_testdata
 rc_testdata==0.11.0

rhodecode/VERSION

@@ -1,1 +1,1 b''
-5.0.3
  No newline at end of file
+5.1.0
  No newline at end of file

rhodecode/__init__.py

@@ -1,91 +1,91 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import datetime
 import collections
 import logging
 
 
 now = datetime.datetime.now()
 now = now.strftime("%Y-%m-%d %H:%M:%S") + '.' + f"{int(now.microsecond/1000):03d}"
 
 log = logging.getLogger(__name__)
 log.debug(f'{now} Starting RhodeCode imports...')
 
 
 VERSION = tuple(open(os.path.join(
     os.path.dirname(__file__), 'VERSION')).read().split('.'))
 
 BACKENDS = collections.OrderedDict()
 
 BACKENDS['hg'] = 'Mercurial repository'
 BACKENDS['git'] = 'Git repository'
 BACKENDS['svn'] = 'Subversion repository'
 
 
 CELERY_ENABLED = False
 CELERY_EAGER = False
 
 # link to config for pyramid
 CONFIG = {}
 
 
 class ConfigGet:
     NotGiven = object()
 
     def _get_val_or_missing(self, key, missing):
         if key not in CONFIG:
             if missing == self.NotGiven:
                 return missing
             # we don't get key, we don't get missing value, return nothing similar as config.get(key)
             return None
         else:
             val = CONFIG[key]
         return val
 
     def get_str(self, key, missing=NotGiven):
         from rhodecode.lib.str_utils import safe_str
         val = self._get_val_or_missing(key, missing)
         return safe_str(val)
 
     def get_int(self, key, missing=NotGiven):
         from rhodecode.lib.str_utils import safe_int
         val = self._get_val_or_missing(key, missing)
         return safe_int(val)
 
     def get_bool(self, key, missing=NotGiven):
         from rhodecode.lib.type_utils import str2bool
         val = self._get_val_or_missing(key, missing)
         return str2bool(val)
 
 # Populated with the settings dictionary from application init in
 # rhodecode.conf.environment.load_pyramid_environment
 PYRAMID_SETTINGS = {}
 
 # Linked module for extensions
 EXTENSIONS = {}
 
 __version__ = ('.'.join((str(each) for each in VERSION[:3])))
-__dbversion__ = 114  # defines current db version for migrations
+__dbversion__ = 115  # defines current db version for migrations
 __license__ = 'AGPLv3, and Commercial License'
 __author__ = 'RhodeCode GmbH'
 __url__ = 'https://code.rhodecode.com'
 
-is_test = False
+is_test = os.getenv('RC_TEST', '0') == '1'
 disable_error_handler = False

rhodecode/api/__init__.py

@@ -1,573 +1,581 b''
 # Copyright (C) 2011-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import itertools
 import logging
 import sys
 import fnmatch
 
 import decorator
-import typing
 import venusian
 from collections import OrderedDict
 
 from pyramid.exceptions import ConfigurationError
 from pyramid.renderers import render
 from pyramid.response import Response
 from pyramid.httpexceptions import HTTPNotFound
 
 from rhodecode.api.exc import (
     JSONRPCBaseError, JSONRPCError, JSONRPCForbidden, JSONRPCValidationError)
 from rhodecode.apps._base import TemplateArgs
 from rhodecode.lib.auth import AuthUser
 from rhodecode.lib.base import get_ip_addr, attach_context_attributes
 from rhodecode.lib.exc_tracking import store_exception
 from rhodecode.lib import ext_json
 from rhodecode.lib.utils2 import safe_str
 from rhodecode.lib.plugins.utils import get_plugin_settings
 from rhodecode.model.db import User, UserApiKeys
 
 log = logging.getLogger(__name__)
 
 DEFAULT_RENDERER = 'jsonrpc_renderer'
-DEFAULT_URL = '/_admin/apiv2'
+DEFAULT_URL = '/_admin/api'
+SERVICE_API_IDENTIFIER = 'service_'
 
 
 def find_methods(jsonrpc_methods, pattern):
     matches = OrderedDict()
     if not isinstance(pattern, (list, tuple)):
         pattern = [pattern]
 
     for single_pattern in pattern:
-        for method_name, method in jsonrpc_methods.items():
+        for method_name, method in filter(
+                lambda x: not x[0].startswith(SERVICE_API_IDENTIFIER), jsonrpc_methods.items()
+        ):
             if fnmatch.fnmatch(method_name, single_pattern):
                 matches[method_name] = method
     return matches
 
 
 class ExtJsonRenderer(object):
     """
     Custom renderer that makes use of our ext_json lib
 
     """
 
     def __init__(self):
         self.serializer = ext_json.formatted_json
 
     def __call__(self, info):
         """ Returns a plain JSON-encoded string with content-type
         ``application/json``. The content-type may be overridden by
         setting ``request.response.content_type``."""
 
         def _render(value, system):
             request = system.get('request')
             if request is not None:
                 response = request.response
                 ct = response.content_type
                 if ct == response.default_content_type:
                     response.content_type = 'application/json'
 
             return self.serializer(value)
 
         return _render
 
 
 def jsonrpc_response(request, result):
     rpc_id = getattr(request, 'rpc_id', None)
 
     ret_value = ''
     if rpc_id:
         ret_value = {'id': rpc_id, 'result': result, 'error': None}
 
         # fetch deprecation warnings, and store it inside results
         deprecation = getattr(request, 'rpc_deprecation', None)
         if deprecation:
             ret_value['DEPRECATION_WARNING'] = deprecation
 
     raw_body = render(DEFAULT_RENDERER, ret_value, request=request)
     content_type = 'application/json'
     content_type_header = 'Content-Type'
     headers = {
         content_type_header: content_type
     }
     return Response(
         body=raw_body,
         content_type=content_type,
         headerlist=[(k, v) for k, v in headers.items()]
     )
 
 
 def jsonrpc_error(request, message, retid=None, code: int | None = None, headers: dict | None = None):
     """
     Generate a Response object with a JSON-RPC error body
     """
     headers = headers or {}
     content_type = 'application/json'
     content_type_header = 'Content-Type'
     if content_type_header not in headers:
         headers[content_type_header] = content_type
 
     err_dict = {'id': retid, 'result': None, 'error': message}
     raw_body = render(DEFAULT_RENDERER, err_dict, request=request)
 
     return Response(
         body=raw_body,
         status=code,
         content_type=content_type,
         headerlist=[(k, v) for k, v in headers.items()]
     )
 
 
 def exception_view(exc, request):
     rpc_id = getattr(request, 'rpc_id', None)
 
     if isinstance(exc, JSONRPCError):
         fault_message = safe_str(exc)
         log.debug('json-rpc error rpc_id:%s "%s"', rpc_id, fault_message)
     elif isinstance(exc, JSONRPCValidationError):
         colander_exc = exc.colander_exception
         # TODO(marcink): think maybe of nicer way to serialize errors ?
         fault_message = colander_exc.asdict()
         log.debug('json-rpc colander error rpc_id:%s "%s"', rpc_id, fault_message)
     elif isinstance(exc, JSONRPCForbidden):
         fault_message = 'Access was denied to this resource.'
         log.warning('json-rpc forbidden call rpc_id:%s "%s"', rpc_id, fault_message)
     elif isinstance(exc, HTTPNotFound):
         method = request.rpc_method
         log.debug('json-rpc method `%s` not found in list of '
                   'api calls: %s, rpc_id:%s',
                   method, list(request.registry.jsonrpc_methods.keys()), rpc_id)
 
         similar = 'none'
         try:
             similar_paterns = [f'*{x}*' for x in method.split('_')]
             similar_found = find_methods(
                 request.registry.jsonrpc_methods, similar_paterns)
             similar = ', '.join(similar_found.keys()) or similar
         except Exception:
             # make the whole above block safe
             pass
 
         fault_message = f"No such method: {method}. Similar methods: {similar}"
     else:
         fault_message = 'undefined error'
         exc_info = exc.exc_info()
         store_exception(id(exc_info), exc_info, prefix='rhodecode-api')
 
         statsd = request.registry.statsd
         if statsd:
             exc_type = f"{exc.__class__.__module__}.{exc.__class__.__name__}"
             statsd.incr('rhodecode_exception_total',
                         tags=["exc_source:api", f"type:{exc_type}"])
 
     return jsonrpc_error(request, fault_message, rpc_id)
 
 
 def request_view(request):
     """
     Main request handling method. It handles all logic to call a specific
     exposed method
     """
     # cython compatible inspect
     from rhodecode.config.patches import inspect_getargspec
     inspect = inspect_getargspec()
 
     # check if we can find this session using api_key, get_by_auth_token
     # search not expired tokens only
     try:
-        api_user = User.get_by_auth_token(request.rpc_api_key)
+        if not request.rpc_method.startswith(SERVICE_API_IDENTIFIER):
+            api_user = User.get_by_auth_token(request.rpc_api_key)
 
-        if api_user is None:
-            return jsonrpc_error(
-                request, retid=request.rpc_id, message='Invalid API KEY')
+            if api_user is None:
+                return jsonrpc_error(
+                    request, retid=request.rpc_id, message='Invalid API KEY')
 
-        if not api_user.active:
-            return jsonrpc_error(
-                request, retid=request.rpc_id,
-                message='Request from this user not allowed')
+            if not api_user.active:
+                return jsonrpc_error(
+                    request, retid=request.rpc_id,
+                    message='Request from this user not allowed')
 
-        # check if we are allowed to use this IP
-        auth_u = AuthUser(
-            api_user.user_id, request.rpc_api_key, ip_addr=request.rpc_ip_addr)
-        if not auth_u.ip_allowed:
-            return jsonrpc_error(
-                request, retid=request.rpc_id,
-                message='Request from IP:{} not allowed'.format(
-                    request.rpc_ip_addr))
-        else:
-            log.info('Access for IP:%s allowed', request.rpc_ip_addr)
+            # check if we are allowed to use this IP
+            auth_u = AuthUser(
+                api_user.user_id, request.rpc_api_key, ip_addr=request.rpc_ip_addr)
+            if not auth_u.ip_allowed:
+                return jsonrpc_error(
+                    request, retid=request.rpc_id,
+                    message='Request from IP:{} not allowed'.format(
+                        request.rpc_ip_addr))
+            else:
+                log.info('Access for IP:%s allowed', request.rpc_ip_addr)
+
+            # register our auth-user
+            request.rpc_user = auth_u
+            request.environ['rc_auth_user_id'] = str(auth_u.user_id)
 
-        # register our auth-user
-        request.rpc_user = auth_u
-        request.environ['rc_auth_user_id'] = str(auth_u.user_id)
+            # now check if token is valid for API
+            auth_token = request.rpc_api_key
+            token_match = api_user.authenticate_by_token(
+                auth_token, roles=[UserApiKeys.ROLE_API])
+            invalid_token = not token_match
 
-        # now check if token is valid for API
-        auth_token = request.rpc_api_key
-        token_match = api_user.authenticate_by_token(
-            auth_token, roles=[UserApiKeys.ROLE_API])
-        invalid_token = not token_match
-
-        log.debug('Checking if API KEY is valid with proper role')
-        if invalid_token:
-            return jsonrpc_error(
-                request, retid=request.rpc_id,
-                message='API KEY invalid or, has bad role for an API call')
+            log.debug('Checking if API KEY is valid with proper role')
+            if invalid_token:
+                return jsonrpc_error(
+                    request, retid=request.rpc_id,
+                    message='API KEY invalid or, has bad role for an API call')
+        else:
+            auth_u = 'service'
+            if request.rpc_api_key != request.registry.settings['app.service_api.token']:
+                raise Exception("Provided service secret is not recognized!")
 
     except Exception:
         log.exception('Error on API AUTH')
         return jsonrpc_error(
             request, retid=request.rpc_id, message='Invalid API KEY')
 
     method = request.rpc_method
     func = request.registry.jsonrpc_methods[method]
 
     # now that we have a method, add request._req_params to
     # self.kargs and dispatch control to WGIController
 
     argspec = inspect.getargspec(func)
     arglist = argspec[0]
     defs = argspec[3] or []
     defaults = [type(a) for a in defs]
     default_empty = type(NotImplemented)
 
     # kw arguments required by this method
     func_kwargs = dict(itertools.zip_longest(
         reversed(arglist), reversed(defaults), fillvalue=default_empty))
 
     # This attribute will need to be first param of a method that uses
     # api_key, which is translated to instance of user at that name
     user_var = 'apiuser'
     request_var = 'request'
 
     for arg in [user_var, request_var]:
         if arg not in arglist:
             return jsonrpc_error(
                 request,
                 retid=request.rpc_id,
                 message='This method [%s] does not support '
                         'required parameter `%s`' % (func.__name__, arg))
 
     # get our arglist and check if we provided them as args
     for arg, default in func_kwargs.items():
         if arg in [user_var, request_var]:
             # user_var and request_var are pre-hardcoded parameters and we
             # don't need to do any translation
             continue
 
         # skip the required param check if it's default value is
         # NotImplementedType (default_empty)
         if default == default_empty and arg not in request.rpc_params:
             return jsonrpc_error(
                 request,
                 retid=request.rpc_id,
                 message=('Missing non optional `%s` arg in JSON DATA' % arg)
             )
 
     # sanitize extra passed arguments
     for k in list(request.rpc_params.keys()):
         if k not in func_kwargs:
             del request.rpc_params[k]
 
     call_params = request.rpc_params
     call_params.update({
         'request': request,
         'apiuser': auth_u
     })
 
     # register some common functions for usage
-    attach_context_attributes(TemplateArgs(), request, request.rpc_user.user_id)
+    rpc_user = request.rpc_user.user_id if hasattr(request, 'rpc_user') else None
+    attach_context_attributes(TemplateArgs(), request, rpc_user)
 
     statsd = request.registry.statsd
 
     try:
         ret_value = func(**call_params)
         resp = jsonrpc_response(request, ret_value)
         if statsd:
             statsd.incr('rhodecode_api_call_success_total')
         return resp
     except JSONRPCBaseError:
         raise
     except Exception:
         log.exception('Unhandled exception occurred on api call: %s', func)
         exc_info = sys.exc_info()
         exc_id, exc_type_name = store_exception(
             id(exc_info), exc_info, prefix='rhodecode-api')
         error_headers = {
             'RhodeCode-Exception-Id': str(exc_id),
             'RhodeCode-Exception-Type': str(exc_type_name)
         }
         err_resp = jsonrpc_error(
             request, retid=request.rpc_id, message='Internal server error',
             headers=error_headers)
         if statsd:
             statsd.incr('rhodecode_api_call_fail_total')
         return err_resp
 
 
 def setup_request(request):
     """
     Parse a JSON-RPC request body. It's used inside the predicates method
     to validate and bootstrap requests for usage in rpc calls.
 
     We need to raise JSONRPCError here if we want to return some errors back to
     user.
     """
 
     log.debug('Executing setup request: %r', request)
     request.rpc_ip_addr = get_ip_addr(request.environ)
     # TODO(marcink): deprecate GET at some point
     if request.method not in ['POST', 'GET']:
         log.debug('unsupported request method "%s"', request.method)
         raise JSONRPCError(
             'unsupported request method "%s". Please use POST' % request.method)
 
     if 'CONTENT_LENGTH' not in request.environ:
         log.debug("No Content-Length")
         raise JSONRPCError("Empty body, No Content-Length in request")
 
     else:
         length = request.environ['CONTENT_LENGTH']
         log.debug('Content-Length: %s', length)
 
         if length == 0:
             log.debug("Content-Length is 0")
             raise JSONRPCError("Content-Length is 0")
 
     raw_body = request.body
     log.debug("Loading JSON body now")
     try:
         json_body = ext_json.json.loads(raw_body)
     except ValueError as e:
         # catch JSON errors Here
         raise JSONRPCError(f"JSON parse error ERR:{e} RAW:{raw_body!r}")
 
     request.rpc_id = json_body.get('id')
     request.rpc_method = json_body.get('method')
 
     # check required base parameters
     try:
         api_key = json_body.get('api_key')
         if not api_key:
             api_key = json_body.get('auth_token')
 
         if not api_key:
             raise KeyError('api_key or auth_token')
 
         # TODO(marcink): support passing in token in request header
 
         request.rpc_api_key = api_key
         request.rpc_id = json_body['id']
         request.rpc_method = json_body['method']
         request.rpc_params = json_body['args'] \
             if isinstance(json_body['args'], dict) else {}
 
         log.debug('method: %s, params: %.10240r', request.rpc_method, request.rpc_params)
     except KeyError as e:
         raise JSONRPCError(f'Incorrect JSON data. Missing {e}')
 
     log.debug('setup complete, now handling method:%s rpcid:%s',
               request.rpc_method, request.rpc_id, )
 
 
 class RoutePredicate(object):
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         return f'jsonrpc route = {self.val}'
 
     phash = text
 
     def __call__(self, info, request):
         if self.val:
             # potentially setup and bootstrap our call
             setup_request(request)
 
             # Always return True so that even if it isn't a valid RPC it
             # will fall through to the underlaying handlers like notfound_view
             return True
 
 
 class NotFoundPredicate(object):
     def __init__(self, val, config):
         self.val = val
         self.methods = config.registry.jsonrpc_methods
 
     def text(self):
         return f'jsonrpc method not found = {self.val}'
 
     phash = text
 
     def __call__(self, info, request):
         return hasattr(request, 'rpc_method')
 
 
 class MethodPredicate(object):
     def __init__(self, val, config):
         self.method = val
 
     def text(self):
         return f'jsonrpc method = {self.method}'
 
     phash = text
 
     def __call__(self, context, request):
         # we need to explicitly return False here, so pyramid doesn't try to
         # execute our view directly. We need our main handler to execute things
         return getattr(request, 'rpc_method') == self.method
 
 
 def add_jsonrpc_method(config, view, **kwargs):
     # pop the method name
     method = kwargs.pop('method', None)
 
     if method is None:
         raise ConfigurationError(
             'Cannot register a JSON-RPC method without specifying the "method"')
 
     # we define custom predicate, to enable to detect conflicting methods,
     # those predicates are kind of "translation" from the decorator variables
     # to internal predicates names
 
     kwargs['jsonrpc_method'] = method
 
     # register our view into global view store for validation
     config.registry.jsonrpc_methods[method] = view
 
     # we're using our main request_view handler, here, so each method
     # has a unified handler for itself
     config.add_view(request_view, route_name='apiv2', **kwargs)
 
 
 class jsonrpc_method(object):
     """
     decorator that works similar to @add_view_config decorator,
     but tailored for our JSON RPC
     """
 
     venusian = venusian  # for testing injection
 
     def __init__(self, method=None, **kwargs):
         self.method = method
         self.kwargs = kwargs
 
     def __call__(self, wrapped):
         kwargs = self.kwargs.copy()
         kwargs['method'] = self.method or wrapped.__name__
         depth = kwargs.pop('_depth', 0)
 
         def callback(context, name, ob):
             config = context.config.with_package(info.module)
             config.add_jsonrpc_method(view=ob, **kwargs)
 
         info = venusian.attach(wrapped, callback, category='pyramid',
                                depth=depth + 1)
         if info.scope == 'class':
             # ensure that attr is set if decorating a class method
             kwargs.setdefault('attr', wrapped.__name__)
 
         kwargs['_info'] = info.codeinfo  # fbo action_method
         return wrapped
 
 
 class jsonrpc_deprecated_method(object):
     """
     Marks method as deprecated, adds log.warning, and inject special key to
     the request variable to mark method as deprecated.
     Also injects special docstring that extract_docs will catch to mark
     method as deprecated.
 
     :param use_method: specify which method should be used instead of
         the decorated one
 
     Use like::
 
         @jsonrpc_method()
         @jsonrpc_deprecated_method(use_method='new_func', deprecated_at_version='3.0.0')
         def old_func(request, apiuser, arg1, arg2):
             ...
     """
 
     def __init__(self, use_method, deprecated_at_version):
         self.use_method = use_method
         self.deprecated_at_version = deprecated_at_version
         self.deprecated_msg = ''
 
     def __call__(self, func):
         self.deprecated_msg = 'Please use method `{method}` instead.'.format(
             method=self.use_method)
 
         docstring = """\n
         .. deprecated:: {version}
 
            {deprecation_message}
 
         {original_docstring}
         """
         func.__doc__ = docstring.format(
             version=self.deprecated_at_version,
             deprecation_message=self.deprecated_msg,
             original_docstring=func.__doc__)
         return decorator.decorator(self.__wrapper, func)
 
     def __wrapper(self, func, *fargs, **fkwargs):
         log.warning('DEPRECATED API CALL on function %s, please '
                     'use `%s` instead', func, self.use_method)
         # alter function docstring to mark as deprecated, this is picked up
         # via fabric file that generates API DOC.
         result = func(*fargs, **fkwargs)
 
         request = fargs[0]
         request.rpc_deprecation = 'DEPRECATED METHOD ' + self.deprecated_msg
         return result
 
 
 def add_api_methods(config):
     from rhodecode.api.views import (
         deprecated_api, gist_api, pull_request_api, repo_api, repo_group_api,
         server_api, search_api, testing_api, user_api, user_group_api)
 
     config.scan('rhodecode.api.views')
 
 
 def includeme(config):
     plugin_module = 'rhodecode.api'
     plugin_settings = get_plugin_settings(
         plugin_module, config.registry.settings)
 
     if not hasattr(config.registry, 'jsonrpc_methods'):
         config.registry.jsonrpc_methods = OrderedDict()
 
     # match filter by given method only
     config.add_view_predicate('jsonrpc_method', MethodPredicate)
     config.add_view_predicate('jsonrpc_method_not_found', NotFoundPredicate)
 
     config.add_renderer(DEFAULT_RENDERER, ExtJsonRenderer())
     config.add_directive('add_jsonrpc_method', add_jsonrpc_method)
 
     config.add_route_predicate(
         'jsonrpc_call', RoutePredicate)
 
     config.add_route(
         'apiv2', plugin_settings.get('url', DEFAULT_URL), jsonrpc_call=True)
 
     # register some exception handling view
     config.add_view(exception_view, context=JSONRPCBaseError)
     config.add_notfound_view(exception_view, jsonrpc_method_not_found=True)
 
     add_api_methods(config)

rhodecode/api/tests/test_create_repo.py

@@ -1,348 +1,348 b''
 
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import mock
 import pytest
 
 from rhodecode.lib.vcs import settings
 from rhodecode.model.meta import Session
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.user import UserModel
 from rhodecode.tests import TEST_USER_ADMIN_LOGIN
 from rhodecode.api.tests.utils import (
     build_data, api_call, assert_ok, assert_error, crash)
 from rhodecode.tests.fixture import Fixture
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.str_utils import safe_str
 
 
 fixture = Fixture()
 
 
 @pytest.mark.usefixtures("testuser_api", "app")
 class TestCreateRepo(object):
 
     @pytest.mark.parametrize('given, expected_name, expected_exc', [
         ('api repo-1', 'api-repo-1', False),
         ('api-repo 1-ąć', 'api-repo-1-ąć', False),
-        (u'unicode-ąć', u'unicode-ąć', False),
+        ('unicode-ąć', u'unicode-ąć', False),
         ('some repo v1.2', 'some-repo-v1.2', False),
         ('v2.0', 'v2.0', False),
     ])
     def test_api_create_repo(self, backend, given, expected_name, expected_exc):
 
         id_, params = build_data(
             self.apikey,
             'create_repo',
             repo_name=given,
             owner=TEST_USER_ADMIN_LOGIN,
             repo_type=backend.alias,
         )
         response = api_call(self.app, params)
 
         ret = {
             'msg': 'Created new repository `%s`' % (expected_name,),
             'success': True,
             'task': None,
         }
         expected = ret
         assert_ok(id_, expected, given=response.body)
 
         repo = RepoModel().get_by_repo_name(safe_str(expected_name))
         assert repo is not None
 
         id_, params = build_data(self.apikey, 'get_repo', repoid=expected_name)
         response = api_call(self.app, params)
         body = json.loads(response.body)
 
         assert body['result']['enable_downloads'] is False
         assert body['result']['enable_locking'] is False
         assert body['result']['enable_statistics'] is False
 
         fixture.destroy_repo(safe_str(expected_name))
 
     def test_api_create_restricted_repo_type(self, backend):
         repo_name = 'api-repo-type-{0}'.format(backend.alias)
         id_, params = build_data(
             self.apikey,
             'create_repo',
             repo_name=repo_name,
             owner=TEST_USER_ADMIN_LOGIN,
             repo_type=backend.alias,
         )
         git_backend = settings.BACKENDS['git']
         with mock.patch(
                 'rhodecode.lib.vcs.settings.BACKENDS', {'git': git_backend}):
             response = api_call(self.app, params)
 
         repo = RepoModel().get_by_repo_name(repo_name)
 
         if backend.alias == 'git':
             assert repo is not None
             expected = {
                 'msg': 'Created new repository `{0}`'.format(repo_name,),
                 'success': True,
                 'task': None,
             }
             assert_ok(id_, expected, given=response.body)
         else:
             assert repo is None
 
         fixture.destroy_repo(repo_name)
 
     def test_api_create_repo_with_booleans(self, backend):
         repo_name = 'api-repo-2'
         id_, params = build_data(
             self.apikey,
             'create_repo',
             repo_name=repo_name,
             owner=TEST_USER_ADMIN_LOGIN,
             repo_type=backend.alias,
             enable_statistics=True,
             enable_locking=True,
             enable_downloads=True
         )
         response = api_call(self.app, params)
 
         repo = RepoModel().get_by_repo_name(repo_name)
 
         assert repo is not None
         ret = {
             'msg': 'Created new repository `%s`' % (repo_name,),
             'success': True,
             'task': None,
         }
         expected = ret
         assert_ok(id_, expected, given=response.body)
 
         id_, params = build_data(self.apikey, 'get_repo', repoid=repo_name)
         response = api_call(self.app, params)
         body = json.loads(response.body)
 
         assert body['result']['enable_downloads'] is True
         assert body['result']['enable_locking'] is True
         assert body['result']['enable_statistics'] is True
 
         fixture.destroy_repo(repo_name)
 
     def test_api_create_repo_in_group(self, backend):
         repo_group_name = 'my_gr'
         # create the parent
         fixture.create_repo_group(repo_group_name)
 
         repo_name = '%s/api-repo-gr' % (repo_group_name,)
         id_, params = build_data(
             self.apikey, 'create_repo',
             repo_name=repo_name,
             owner=TEST_USER_ADMIN_LOGIN,
             repo_type=backend.alias,)
         response = api_call(self.app, params)
         repo = RepoModel().get_by_repo_name(repo_name)
         assert repo is not None
         assert repo.group is not None
 
         ret = {
             'msg': 'Created new repository `%s`' % (repo_name,),
             'success': True,
             'task': None,
         }
         expected = ret
         assert_ok(id_, expected, given=response.body)
         fixture.destroy_repo(repo_name)
         fixture.destroy_repo_group(repo_group_name)
 
     def test_create_repo_in_group_that_doesnt_exist(self, backend, user_util):
         repo_group_name = 'fake_group'
 
         repo_name = '%s/api-repo-gr' % (repo_group_name,)
         id_, params = build_data(
             self.apikey, 'create_repo',
             repo_name=repo_name,
             owner=TEST_USER_ADMIN_LOGIN,
             repo_type=backend.alias,)
         response = api_call(self.app, params)
 
         expected = {'repo_group': 'Repository group `{}` does not exist'.format(
             repo_group_name)}
         assert_error(id_, expected, given=response.body)
 
     def test_api_create_repo_unknown_owner(self, backend):
         repo_name = 'api-repo-2'
         owner = 'i-dont-exist'
         id_, params = build_data(
             self.apikey, 'create_repo',
             repo_name=repo_name,
             owner=owner,
             repo_type=backend.alias)
         response = api_call(self.app, params)
         expected = 'user `%s` does not exist' % (owner,)
         assert_error(id_, expected, given=response.body)
 
     def test_api_create_repo_dont_specify_owner(self, backend):
         repo_name = 'api-repo-3'
         id_, params = build_data(
             self.apikey, 'create_repo',
             repo_name=repo_name,
             repo_type=backend.alias)
         response = api_call(self.app, params)
 
         repo = RepoModel().get_by_repo_name(repo_name)
         assert repo is not None
         ret = {
             'msg': 'Created new repository `%s`' % (repo_name,),
             'success': True,
             'task': None,
         }
         expected = ret
         assert_ok(id_, expected, given=response.body)
         fixture.destroy_repo(repo_name)
 
     def test_api_create_repo_by_non_admin(self, backend):
         repo_name = 'api-repo-4'
         id_, params = build_data(
             self.apikey_regular, 'create_repo',
             repo_name=repo_name,
             repo_type=backend.alias)
         response = api_call(self.app, params)
 
         repo = RepoModel().get_by_repo_name(repo_name)
         assert repo is not None
         ret = {
             'msg': 'Created new repository `%s`' % (repo_name,),
             'success': True,
             'task': None,
         }
         expected = ret
         assert_ok(id_, expected, given=response.body)
         fixture.destroy_repo(repo_name)
 
     def test_api_create_repo_by_non_admin_specify_owner(self, backend):
         repo_name = 'api-repo-5'
         owner = 'i-dont-exist'
         id_, params = build_data(
             self.apikey_regular, 'create_repo',
             repo_name=repo_name,
             repo_type=backend.alias,
             owner=owner)
         response = api_call(self.app, params)
 
         expected = 'Only RhodeCode super-admin can specify `owner` param'
         assert_error(id_, expected, given=response.body)
         fixture.destroy_repo(repo_name)
 
     def test_api_create_repo_by_non_admin_no_parent_group_perms(self, backend):
         repo_group_name = 'no-access'
         fixture.create_repo_group(repo_group_name)
         repo_name = 'no-access/api-repo'
 
         id_, params = build_data(
             self.apikey_regular, 'create_repo',
             repo_name=repo_name,
             repo_type=backend.alias)
         response = api_call(self.app, params)
 
         expected = {'repo_group': 'Repository group `{}` does not exist'.format(
             repo_group_name)}
         assert_error(id_, expected, given=response.body)
         fixture.destroy_repo_group(repo_group_name)
         fixture.destroy_repo(repo_name)
 
     def test_api_create_repo_non_admin_no_permission_to_create_to_root_level(
             self, backend, user_util):
 
         regular_user = user_util.create_user()
         regular_user_api_key = regular_user.api_key
 
         usr = UserModel().get_by_username(regular_user.username)
         usr.inherit_default_permissions = False
         Session().add(usr)
 
         repo_name = backend.new_repo_name()
         id_, params = build_data(
             regular_user_api_key, 'create_repo',
             repo_name=repo_name,
             repo_type=backend.alias)
         response = api_call(self.app, params)
         expected = {
             "repo_name": "You do not have the permission to "
                          "store repositories in the root location."}
         assert_error(id_, expected, given=response.body)
 
     def test_api_create_repo_exists(self, backend):
         repo_name = backend.repo_name
         id_, params = build_data(
             self.apikey, 'create_repo',
             repo_name=repo_name,
             owner=TEST_USER_ADMIN_LOGIN,
             repo_type=backend.alias,)
         response = api_call(self.app, params)
         expected = {
             'unique_repo_name': 'Repository with name `{}` already exists'.format(
                 repo_name)}
         assert_error(id_, expected, given=response.body)
 
     @mock.patch.object(RepoModel, 'create', crash)
     def test_api_create_repo_exception_occurred(self, backend):
         repo_name = 'api-repo-6'
         id_, params = build_data(
             self.apikey, 'create_repo',
             repo_name=repo_name,
             owner=TEST_USER_ADMIN_LOGIN,
             repo_type=backend.alias,)
         response = api_call(self.app, params)
         expected = 'failed to create repository `%s`' % (repo_name,)
         assert_error(id_, expected, given=response.body)
 
     @pytest.mark.parametrize('parent_group, dirty_name, expected_name', [
         (None, 'foo bar x', 'foo-bar-x'),
         ('foo', '/foo//bar x', 'foo/bar-x'),
         ('foo-bar', 'foo-bar //bar x', 'foo-bar/bar-x'),
     ])
     def test_create_repo_with_extra_slashes_in_name(
             self, backend, parent_group, dirty_name, expected_name):
 
         if parent_group:
             gr = fixture.create_repo_group(parent_group)
             assert gr.group_name == parent_group
 
         id_, params = build_data(
             self.apikey, 'create_repo',
             repo_name=dirty_name,
             repo_type=backend.alias,
             owner=TEST_USER_ADMIN_LOGIN,)
         response = api_call(self.app, params)
         expected ={
            "msg": "Created new repository `{}`".format(expected_name),
            "task": None,
            "success": True
         }
         assert_ok(id_, expected, response.body)
 
         repo = RepoModel().get_by_repo_name(expected_name)
         assert repo is not None
 
         expected = {
             'msg': 'Created new repository `%s`' % (expected_name,),
             'success': True,
             'task': None,
         }
         assert_ok(id_, expected, given=response.body)
         fixture.destroy_repo(expected_name)
         if parent_group:
             fixture.destroy_repo_group(parent_group)

rhodecode/api/tests/test_create_repo_group.py

@@ -1,288 +1,288 b''
 
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import mock
 import pytest
 
 from rhodecode.model.meta import Session
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.user import UserModel
 from rhodecode.tests import TEST_USER_ADMIN_LOGIN
 from rhodecode.api.tests.utils import (
     build_data, api_call, assert_ok, assert_error, crash)
 from rhodecode.tests.fixture import Fixture
 
 
 fixture = Fixture()
 
 
 @pytest.mark.usefixtures("testuser_api", "app")
 class TestCreateRepoGroup(object):
     def test_api_create_repo_group(self):
         repo_group_name = 'api-repo-group'
 
         repo_group = RepoGroupModel.cls.get_by_group_name(repo_group_name)
         assert repo_group is None
 
         id_, params = build_data(
             self.apikey, 'create_repo_group',
             group_name=repo_group_name,
             owner=TEST_USER_ADMIN_LOGIN,)
         response = api_call(self.app, params)
 
         repo_group = RepoGroupModel.cls.get_by_group_name(repo_group_name)
         assert repo_group is not None
         ret = {
             'msg': 'Created new repo group `%s`' % (repo_group_name,),
             'repo_group': repo_group.get_api_data()
         }
         expected = ret
         try:
             assert_ok(id_, expected, given=response.body)
         finally:
             fixture.destroy_repo_group(repo_group_name)
 
     def test_api_create_repo_group_in_another_group(self):
         repo_group_name = 'api-repo-group'
 
         repo_group = RepoGroupModel.cls.get_by_group_name(repo_group_name)
         assert repo_group is None
         # create the parent
         fixture.create_repo_group(repo_group_name)
 
         full_repo_group_name = repo_group_name+'/'+repo_group_name
         id_, params = build_data(
             self.apikey, 'create_repo_group',
             group_name=full_repo_group_name,
             owner=TEST_USER_ADMIN_LOGIN,
             copy_permissions=True)
         response = api_call(self.app, params)
 
         repo_group = RepoGroupModel.cls.get_by_group_name(full_repo_group_name)
         assert repo_group is not None
         ret = {
             'msg': 'Created new repo group `%s`' % (full_repo_group_name,),
             'repo_group': repo_group.get_api_data()
         }
         expected = ret
         try:
             assert_ok(id_, expected, given=response.body)
         finally:
             fixture.destroy_repo_group(full_repo_group_name)
             fixture.destroy_repo_group(repo_group_name)
 
     def test_api_create_repo_group_in_another_group_not_existing(self):
         repo_group_name = 'api-repo-group-no'
 
         repo_group = RepoGroupModel.cls.get_by_group_name(repo_group_name)
         assert repo_group is None
 
         full_repo_group_name = repo_group_name+'/'+repo_group_name
         id_, params = build_data(
             self.apikey, 'create_repo_group',
             group_name=full_repo_group_name,
             owner=TEST_USER_ADMIN_LOGIN,
             copy_permissions=True)
         response = api_call(self.app, params)
         expected = {
             'repo_group':
                 'Parent repository group `{}` does not exist'.format(
                     repo_group_name)}
         assert_error(id_, expected, given=response.body)
 
     def test_api_create_repo_group_that_exists(self):
         repo_group_name = 'api-repo-group'
 
         repo_group = RepoGroupModel.cls.get_by_group_name(repo_group_name)
         assert repo_group is None
 
         fixture.create_repo_group(repo_group_name)
         id_, params = build_data(
             self.apikey, 'create_repo_group',
             group_name=repo_group_name,
             owner=TEST_USER_ADMIN_LOGIN,)
         response = api_call(self.app, params)
         expected = {
             'unique_repo_group_name':
                 'Repository group with name `{}` already exists'.format(
                     repo_group_name)}
         try:
             assert_error(id_, expected, given=response.body)
         finally:
             fixture.destroy_repo_group(repo_group_name)
 
     def test_api_create_repo_group_regular_user_wit_root_location_perms(
             self, user_util):
         regular_user = user_util.create_user()
         regular_user_api_key = regular_user.api_key
 
         repo_group_name = 'api-repo-group-by-regular-user'
 
         usr = UserModel().get_by_username(regular_user.username)
         usr.inherit_default_permissions = False
         Session().add(usr)
 
         UserModel().grant_perm(
             regular_user.username, 'hg.repogroup.create.true')
         Session().commit()
 
         repo_group = RepoGroupModel.cls.get_by_group_name(repo_group_name)
         assert repo_group is None
 
         id_, params = build_data(
             regular_user_api_key, 'create_repo_group',
             group_name=repo_group_name)
         response = api_call(self.app, params)
 
         repo_group = RepoGroupModel.cls.get_by_group_name(repo_group_name)
         assert repo_group is not None
         expected = {
             'msg': 'Created new repo group `%s`' % (repo_group_name,),
             'repo_group': repo_group.get_api_data()
         }
         try:
             assert_ok(id_, expected, given=response.body)
         finally:
             fixture.destroy_repo_group(repo_group_name)
 
     def test_api_create_repo_group_regular_user_with_admin_perms_to_parent(
             self, user_util):
 
         repo_group_name = 'api-repo-group-parent'
 
         repo_group = RepoGroupModel.cls.get_by_group_name(repo_group_name)
         assert repo_group is None
         # create the parent
         fixture.create_repo_group(repo_group_name)
 
         # user perms
         regular_user = user_util.create_user()
         regular_user_api_key = regular_user.api_key
 
         usr = UserModel().get_by_username(regular_user.username)
         usr.inherit_default_permissions = False
         Session().add(usr)
 
         RepoGroupModel().grant_user_permission(
             repo_group_name, regular_user.username, 'group.admin')
         Session().commit()
 
         full_repo_group_name = repo_group_name + '/' + repo_group_name
         id_, params = build_data(
             regular_user_api_key, 'create_repo_group',
             group_name=full_repo_group_name)
         response = api_call(self.app, params)
 
         repo_group = RepoGroupModel.cls.get_by_group_name(full_repo_group_name)
         assert repo_group is not None
         expected = {
             'msg': 'Created new repo group `{}`'.format(full_repo_group_name),
             'repo_group': repo_group.get_api_data()
         }
         try:
             assert_ok(id_, expected, given=response.body)
         finally:
             fixture.destroy_repo_group(full_repo_group_name)
             fixture.destroy_repo_group(repo_group_name)
 
     def test_api_create_repo_group_regular_user_no_permission_to_create_to_root_level(self):
         repo_group_name = 'api-repo-group'
 
         id_, params = build_data(
             self.apikey_regular, 'create_repo_group',
             group_name=repo_group_name)
         response = api_call(self.app, params)
 
         expected = {
             'repo_group':
-                u'You do not have the permission to store '
-                u'repository groups in the root location.'}
+                'You do not have the permission to store '
+                'repository groups in the root location.'}
         assert_error(id_, expected, given=response.body)
 
     def test_api_create_repo_group_regular_user_no_parent_group_perms(self):
         repo_group_name = 'api-repo-group-regular-user'
 
         repo_group = RepoGroupModel.cls.get_by_group_name(repo_group_name)
         assert repo_group is None
         # create the parent
         fixture.create_repo_group(repo_group_name)
 
         full_repo_group_name = repo_group_name+'/'+repo_group_name
 
         id_, params = build_data(
             self.apikey_regular, 'create_repo_group',
             group_name=full_repo_group_name)
         response = api_call(self.app, params)
 
         expected = {
             'repo_group':
-                u"You do not have the permissions to store "
-                u"repository groups inside repository group `{}`".format(repo_group_name)}
+                "You do not have the permissions to store "
+                "repository groups inside repository group `{}`".format(repo_group_name)}
         try:
             assert_error(id_, expected, given=response.body)
         finally:
             fixture.destroy_repo_group(repo_group_name)
 
     def test_api_create_repo_group_regular_user_no_permission_to_specify_owner(
             self):
         repo_group_name = 'api-repo-group'
 
         id_, params = build_data(
             self.apikey_regular, 'create_repo_group',
             group_name=repo_group_name,
             owner=TEST_USER_ADMIN_LOGIN,)
         response = api_call(self.app, params)
 
         expected = "Only RhodeCode super-admin can specify `owner` param"
         assert_error(id_, expected, given=response.body)
 
     @mock.patch.object(RepoGroupModel, 'create', crash)
     def test_api_create_repo_group_exception_occurred(self):
         repo_group_name = 'api-repo-group'
 
         repo_group = RepoGroupModel.cls.get_by_group_name(repo_group_name)
         assert repo_group is None
 
         id_, params = build_data(
             self.apikey, 'create_repo_group',
             group_name=repo_group_name,
             owner=TEST_USER_ADMIN_LOGIN,)
         response = api_call(self.app, params)
         expected = 'failed to create repo group `%s`' % (repo_group_name,)
         assert_error(id_, expected, given=response.body)
 
     def test_create_group_with_extra_slashes_in_name(self, user_util):
         existing_repo_group = user_util.create_repo_group()
         dirty_group_name = '//{}//group2//'.format(
             existing_repo_group.group_name)
         cleaned_group_name = '{}/group2'.format(
             existing_repo_group.group_name)
 
         id_, params = build_data(
             self.apikey, 'create_repo_group',
             group_name=dirty_group_name,
             owner=TEST_USER_ADMIN_LOGIN,)
         response = api_call(self.app, params)
         repo_group = RepoGroupModel.cls.get_by_group_name(cleaned_group_name)
         expected = {
             'msg': 'Created new repo group `%s`' % (cleaned_group_name,),
             'repo_group': repo_group.get_api_data()
         }
         assert_ok(id_, expected, given=response.body)
         fixture.destroy_repo_group(cleaned_group_name)

rhodecode/api/tests/test_get_gist.py

@@ -1,101 +1,101 b''
 
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 import pytest
 
 from rhodecode.lib.str_utils import safe_bytes
 from rhodecode.model.db import Gist
 from rhodecode.api.tests.utils import (
     build_data, api_call, assert_error, assert_ok)
 
 
 @pytest.mark.usefixtures("testuser_api", "app")
 class TestApiGetGist(object):
     def test_api_get_gist(self, gist_util, http_host_only_stub):
         gist = gist_util.create_gist()
         gist_id = gist.gist_access_id
         gist_created_on = gist.created_on
         gist_modified_at = gist.modified_at
         id_, params = build_data(
             self.apikey, 'get_gist', gistid=gist_id, )
         response = api_call(self.app, params)
 
         expected = {
             'access_id': gist_id,
             'created_on': gist_created_on,
             'modified_at': gist_modified_at,
             'description': 'new-gist',
             'expires': -1.0,
             'gist_id': int(gist_id),
             'type': 'public',
             'url': 'http://%s/_admin/gists/%s' % (http_host_only_stub, gist_id,),
             'acl_level': Gist.ACL_LEVEL_PUBLIC,
             'content': None,
         }
 
         assert_ok(id_, expected, given=response.body)
 
     def test_api_get_gist_with_content(self, gist_util, http_host_only_stub):
         mapping = {
             b'filename1.txt': {'content': b'hello world'},
             safe_bytes('filename1ą.txt'): {'content': safe_bytes('hello worldę')}
         }
         gist = gist_util.create_gist(gist_mapping=mapping)
         gist_id = gist.gist_access_id
         gist_created_on = gist.created_on
         gist_modified_at = gist.modified_at
         id_, params = build_data(
             self.apikey, 'get_gist', gistid=gist_id, content=True)
         response = api_call(self.app, params)
 
         expected = {
             'access_id': gist_id,
             'created_on': gist_created_on,
             'modified_at': gist_modified_at,
             'description': 'new-gist',
             'expires': -1.0,
             'gist_id': int(gist_id),
             'type': 'public',
             'url': 'http://%s/_admin/gists/%s' % (http_host_only_stub, gist_id,),
             'acl_level': Gist.ACL_LEVEL_PUBLIC,
             'content': {
-                u'filename1.txt': u'hello world',
-                u'filename1ą.txt': u'hello worldę'
+                'filename1.txt': 'hello world',
+                'filename1ą.txt': 'hello worldę'
             },
         }
 
         assert_ok(id_, expected, given=response.body)
 
     def test_api_get_gist_not_existing(self):
         id_, params = build_data(
             self.apikey_regular, 'get_gist', gistid='12345', )
         response = api_call(self.app, params)
         expected = 'gist `%s` does not exist' % ('12345',)
         assert_error(id_, expected, given=response.body)
 
     def test_api_get_gist_private_gist_without_permission(self, gist_util):
         gist = gist_util.create_gist()
         gist_id = gist.gist_access_id
         id_, params = build_data(
             self.apikey_regular, 'get_gist', gistid=gist_id, )
         response = api_call(self.app, params)
 
         expected = 'gist `%s` does not exist' % (gist_id,)
         assert_error(id_, expected, given=response.body)

rhodecode/api/views/server_api.py

@@ -1,424 +1,423 b''
 # Copyright (C) 2011-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import itertools
 import base64
 
 from rhodecode.api import (
     jsonrpc_method, JSONRPCError, JSONRPCForbidden, find_methods)
 
 from rhodecode.api.utils import (
     Optional, OAttr, has_superadmin_permission, get_user_or_error)
-from rhodecode.lib.utils import repo2db_mapper
+from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_repo_store_path
 from rhodecode.lib import system_info
 from rhodecode.lib import user_sessions
 from rhodecode.lib import exc_tracking
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.utils2 import safe_int
 from rhodecode.model.db import UserIpMap
 from rhodecode.model.scm import ScmModel
-from rhodecode.model.settings import VcsSettingsModel
 from rhodecode.apps.file_store import utils
 from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \
     FileOverSizeException
 
 log = logging.getLogger(__name__)
 
 
 @jsonrpc_method()
 def get_server_info(request, apiuser):
     """
     Returns the |RCE| server information.
 
     This includes the running version of |RCE| and all installed
     packages. This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         'modules': [<module name>,...]
         'py_version': <python version>,
         'platform': <platform type>,
         'rhodecode_version': <rhodecode version>
       }
       error :  null
     """
 
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     server_info = ScmModel().get_server_info(request.environ)
     # rhodecode-index requires those
 
     server_info['index_storage'] = server_info['search']['value']['location']
     server_info['storage'] = server_info['storage']['value']['path']
 
     return server_info
 
 
 @jsonrpc_method()
 def get_repo_store(request, apiuser):
     """
     Returns the |RCE| repository storage information.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         'modules': [<module name>,...]
         'py_version': <python version>,
         'platform': <platform type>,
         'rhodecode_version': <rhodecode version>
       }
       error :  null
     """
 
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
-    path = VcsSettingsModel().get_repos_location()
+    path = get_rhodecode_repo_store_path()
     return {"path": path}
 
 
 @jsonrpc_method()
 def get_ip(request, apiuser, userid=Optional(OAttr('apiuser'))):
     """
     Displays the IP Address as seen from the |RCE| server.
 
     * This command displays the IP Address, as well as all the defined IP
       addresses for the specified user. If the ``userid`` is not set, the
       data returned is for the user calling the method.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from |authtoken|.
     :type apiuser: AuthUser
     :param userid: Sets the userid for which associated IP Address data
         is returned.
     :type userid: Optional(str or int)
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result : {
                      "server_ip_addr": "<ip_from_clien>",
                      "user_ips": [
                                     {
                                        "ip_addr": "<ip_with_mask>",
                                        "ip_range": ["<start_ip>", "<end_ip>"],
                                     },
                                     ...
                                  ]
         }
 
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     userid = Optional.extract(userid, evaluate_locals=locals())
     userid = getattr(userid, 'user_id', userid)
 
     user = get_user_or_error(userid)
     ips = UserIpMap.query().filter(UserIpMap.user == user).all()
     return {
         'server_ip_addr': request.rpc_ip_addr,
         'user_ips': ips
     }
 
 
 @jsonrpc_method()
 def rescan_repos(request, apiuser, remove_obsolete=Optional(False)):
     """
     Triggers a rescan of the specified repositories.
 
     * If the ``remove_obsolete`` option is set, it also deletes repositories
       that are found in the database but not on the file system, so called
       "clean zombies".
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param remove_obsolete: Deletes repositories from the database that
         are not found on the filesystem.
     :type remove_obsolete: Optional(``True`` | ``False``)
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         'added': [<added repository name>,...]
         'removed': [<removed repository name>,...]
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         'Error occurred during rescan repositories action'
       }
 
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     try:
         rm_obsolete = Optional.extract(remove_obsolete)
         added, removed = repo2db_mapper(ScmModel().repo_scan(),
                                         remove_obsolete=rm_obsolete, force_hooks_rebuild=True)
         return {'added': added, 'removed': removed}
     except Exception:
         log.exception('Failed to run repo rescann')
         raise JSONRPCError(
             'Error occurred during rescan repositories action'
         )
 
 
 @jsonrpc_method()
 def cleanup_sessions(request, apiuser, older_then=Optional(60)):
     """
     Triggers a session cleanup action.
 
     If the ``older_then`` option is set, only sessions that hasn't been
     accessed in the given number of days will be removed.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param older_then: Deletes session that hasn't been accessed
         in given number of days.
     :type older_then: Optional(int)
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result: {
         "backend": "<type of backend>",
         "sessions_removed": <number_of_removed_sessions>
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         'Error occurred during session cleanup'
       }
 
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     older_then = safe_int(Optional.extract(older_then)) or 60
     older_than_seconds = 60 * 60 * 24 * older_then
 
     config = system_info.rhodecode_config().get_value()['value']['config']
     session_model = user_sessions.get_session_handler(
         config.get('beaker.session.type', 'memory'))(config)
 
     backend = session_model.SESSION_TYPE
     try:
         cleaned = session_model.clean_sessions(
             older_than_seconds=older_than_seconds)
         return {'sessions_removed': cleaned, 'backend': backend}
     except user_sessions.CleanupCommand as msg:
         return {'cleanup_command': str(msg), 'backend': backend}
     except Exception as e:
         log.exception('Failed session cleanup')
         raise JSONRPCError(
             'Error occurred during session cleanup'
         )
 
 
 @jsonrpc_method()
 def get_method(request, apiuser, pattern=Optional('*')):
     """
     Returns list of all available API methods. By default match pattern
     os "*" but any other pattern can be specified. eg *comment* will return
     all methods with comment inside them. If just single method is matched
     returned data will also include method specification
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param pattern: pattern to match method names against
     :type pattern: Optional("*")
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       "result": [
         "changeset_comment",
         "comment_pull_request",
         "comment_commit"
       ]
       error :  null
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       "result": [
         "comment_commit",
         {
           "apiuser": "<RequiredType>",
           "comment_type": "<Optional:u'note'>",
           "commit_id": "<RequiredType>",
           "message": "<RequiredType>",
           "repoid": "<RequiredType>",
           "request": "<RequiredType>",
           "resolves_comment_id": "<Optional:None>",
           "status": "<Optional:None>",
           "userid": "<Optional:<OptionalAttr:apiuser>>"
         }
       ]
       error :  null
     """
     from rhodecode.config.patches import inspect_getargspec
     inspect = inspect_getargspec()
 
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     pattern = Optional.extract(pattern)
 
     matches = find_methods(request.registry.jsonrpc_methods, pattern)
 
     args_desc = []
     matches_keys = list(matches.keys())
     if len(matches_keys) == 1:
         func = matches[matches_keys[0]]
 
         argspec = inspect.getargspec(func)
         arglist = argspec[0]
         defaults = list(map(repr, argspec[3] or []))
 
         default_empty = '<RequiredType>'
 
         # kw arguments required by this method
         func_kwargs = dict(itertools.zip_longest(
             reversed(arglist), reversed(defaults), fillvalue=default_empty))
         args_desc.append(func_kwargs)
 
     return matches_keys + args_desc
 
 
 @jsonrpc_method()
 def store_exception(request, apiuser, exc_data_json, prefix=Optional('rhodecode')):
     """
     Stores sent exception inside the built-in exception tracker in |RCE| server.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
 
     :param exc_data_json: JSON data with exception e.g
         {"exc_traceback": "Value `1` is not allowed", "exc_type_name": "ValueError"}
     :type exc_data_json: JSON data
 
     :param prefix: prefix for error type, e.g 'rhodecode', 'vcsserver', 'rhodecode-tools'
     :type prefix: Optional("rhodecode")
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       "result": {
         "exc_id": 139718459226384,
         "exc_url": "http://localhost:8080/_admin/settings/exceptions/139718459226384"
       }
       error :  null
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     prefix = Optional.extract(prefix)
     exc_id = exc_tracking.generate_id()
 
     try:
         exc_data = json.loads(exc_data_json)
     except Exception:
         log.error('Failed to parse JSON: %r', exc_data_json)
         raise JSONRPCError('Failed to parse JSON data from exc_data_json field. '
                            'Please make sure it contains a valid JSON.')
 
     try:
         exc_traceback = exc_data['exc_traceback']
         exc_type_name = exc_data['exc_type_name']
         exc_value = ''
     except KeyError as err:
         raise JSONRPCError(
             f'Missing exc_traceback, or exc_type_name '
             f'in exc_data_json field. Missing: {err}')
 
     class ExcType:
         __name__ = exc_type_name
 
     exc_info = (ExcType(), exc_value, exc_traceback)
 
     exc_tracking._store_exception(
         exc_id=exc_id, exc_info=exc_info, prefix=prefix)
 
     exc_url = request.route_url(
         'admin_settings_exception_tracker_show', exception_id=exc_id)
     return {'exc_id': exc_id, 'exc_url': exc_url}

rhodecode/apps/_base/__init__.py

@@ -1,947 +1,987 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import time
 import logging
 import operator
 
 from pyramid.httpexceptions import HTTPFound, HTTPForbidden, HTTPBadRequest
 
 from rhodecode.lib import helpers as h, diffs, rc_cache
 from rhodecode.lib.str_utils import safe_str
 from rhodecode.lib.utils import repo_name_slug
 from rhodecode.lib.utils2 import (
     StrictAttributeDict,
     str2bool,
     safe_int,
     datetime_to_time,
 )
 from rhodecode.lib.markup_renderer import MarkupRenderer, relative_links
 from rhodecode.lib.vcs.backends.base import EmptyCommit
 from rhodecode.lib.vcs.exceptions import RepositoryRequirementError
 from rhodecode.model import repo
 from rhodecode.model import repo_group
 from rhodecode.model import user_group
 from rhodecode.model import user
 from rhodecode.model.db import User
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.settings import VcsSettingsModel, IssueTrackerSettingsModel
 from rhodecode.model.repo import ReadmeFinder
 
 log = logging.getLogger(__name__)
 
 
 ADMIN_PREFIX: str = "/_admin"
 STATIC_FILE_PREFIX: str = "/_static"
 
 URL_NAME_REQUIREMENTS = {
     # group name can have a slash in them, but they must not end with a slash
     "group_name": r".*?[^/]",
     "repo_group_name": r".*?[^/]",
     # repo names can have a slash in them, but they must not end with a slash
     "repo_name": r".*?[^/]",
     # file path eats up everything at the end
     "f_path": r".*",
     # reference types
     "source_ref_type": r"(branch|book|tag|rev|\%\(source_ref_type\)s)",
     "target_ref_type": r"(branch|book|tag|rev|\%\(target_ref_type\)s)",
 }
 
 
 def add_route_with_slash(config, name, pattern, **kw):
     config.add_route(name, pattern, **kw)
     if not pattern.endswith("/"):
         config.add_route(name + "_slash", pattern + "/", **kw)
 
 
 def add_route_requirements(route_path, requirements=None):
     """
     Adds regex requirements to pyramid routes using a mapping dict
     e.g::
         add_route_requirements('{repo_name}/settings')
     """
     requirements = requirements or URL_NAME_REQUIREMENTS
     for key, regex in list(requirements.items()):
         route_path = route_path.replace("{%s}" % key, "{%s:%s}" % (key, regex))
     return route_path
 
 
 def get_format_ref_id(repo):
     """Returns a `repo` specific reference formatter function"""
     if h.is_svn(repo):
         return _format_ref_id_svn
     else:
         return _format_ref_id
 
 
 def _format_ref_id(name, raw_id):
     """Default formatting of a given reference `name`"""
     return name
 
 
 def _format_ref_id_svn(name, raw_id):
     """Special way of formatting a reference for Subversion including path"""
     return f"{name}@{raw_id}"
 
 
 class TemplateArgs(StrictAttributeDict):
     pass
 
 
 class BaseAppView(object):
+    DONT_CHECKOUT_VIEWS = ["channelstream_connect", "ops_ping"]
+    EXTRA_VIEWS_TO_IGNORE = ['login', 'register', 'logout']
+    SETUP_2FA_VIEW = 'setup_2fa'
+    VERIFY_2FA_VIEW = 'check_2fa'
+
     def __init__(self, context, request):
         self.request = request
         self.context = context
         self.session = request.session
         if not hasattr(request, "user"):
             # NOTE(marcink): edge case, we ended up in matched route
             # but probably of web-app context, e.g API CALL/VCS CALL
             if hasattr(request, "vcs_call") or hasattr(request, "rpc_method"):
                 log.warning("Unable to process request `%s` in this scope", request)
                 raise HTTPBadRequest()
 
         self._rhodecode_user = request.user  # auth user
         self._rhodecode_db_user = self._rhodecode_user.get_instance()
+        self.user_data = self._rhodecode_db_user.user_data if self._rhodecode_db_user else {}
         self._maybe_needs_password_change(
             request.matched_route.name, self._rhodecode_db_user
         )
+        self._maybe_needs_2fa_configuration(
+            request.matched_route.name, self._rhodecode_db_user
+        )
+        self._maybe_needs_2fa_check(
+            request.matched_route.name, self._rhodecode_db_user
+        )
 
     def _maybe_needs_password_change(self, view_name, user_obj):
-        dont_check_views = ["channelstream_connect", "ops_ping"]
-        if view_name in dont_check_views:
+        if view_name in self.DONT_CHECKOUT_VIEWS:
             return
 
         log.debug(
             "Checking if user %s needs password change on view %s", user_obj, view_name
         )
 
         skip_user_views = [
             "logout",
             "login",
+            "check_2fa",
             "my_account_password",
             "my_account_password_update",
         ]
 
         if not user_obj:
             return
 
         if user_obj.username == User.DEFAULT_USER:
             return
 
         now = time.time()
-        should_change = user_obj.user_data.get("force_password_change")
+        should_change = self.user_data.get("force_password_change")
         change_after = safe_int(should_change) or 0
         if should_change and now > change_after:
             log.debug("User %s requires password change", user_obj)
             h.flash(
                 "You are required to change your password",
                 "warning",
                 ignore_duplicate=True,
             )
 
             if view_name not in skip_user_views:
                 raise HTTPFound(self.request.route_path("my_account_password"))
 
+    def _maybe_needs_2fa_configuration(self, view_name, user_obj):
+        if view_name in self.DONT_CHECKOUT_VIEWS + self.EXTRA_VIEWS_TO_IGNORE:
+            return
+
+        if not user_obj:
+            return
+
+        if user_obj.needs_2fa_configure and view_name != self.SETUP_2FA_VIEW:
+            h.flash(
+                "You are required to configure 2FA",
+                "warning",
+                ignore_duplicate=False,
+            )
+            # Special case for users created "on the fly" (ldap case for new user)
+            user_obj.check_2fa_required = False
+            raise HTTPFound(self.request.route_path(self.SETUP_2FA_VIEW))
+
+    def _maybe_needs_2fa_check(self, view_name, user_obj):
+        if view_name in self.DONT_CHECKOUT_VIEWS + self.EXTRA_VIEWS_TO_IGNORE:
+            return
+
+        if not user_obj:
+            return
+
+        if user_obj.check_2fa_required and view_name != self.VERIFY_2FA_VIEW:
+            raise HTTPFound(self.request.route_path(self.VERIFY_2FA_VIEW))
+
     def _log_creation_exception(self, e, repo_name):
         _ = self.request.translate
         reason = None
         if len(e.args) == 2:
             reason = e.args[1]
 
         if reason == "INVALID_CERTIFICATE":
             log.exception("Exception creating a repository: invalid certificate")
             msg = _("Error creating repository %s: invalid certificate") % repo_name
         else:
             log.exception("Exception creating a repository")
             msg = _("Error creating repository %s") % repo_name
         return msg
 
     def _get_local_tmpl_context(self, include_app_defaults=True):
         c = TemplateArgs()
         c.auth_user = self.request.user
         # TODO(marcink): migrate the usage of c.rhodecode_user to c.auth_user
         c.rhodecode_user = self.request.user
 
         if include_app_defaults:
             from rhodecode.lib.base import attach_context_attributes
 
             attach_context_attributes(c, self.request, self.request.user.user_id)
 
         c.is_super_admin = c.auth_user.is_admin
 
         c.can_create_repo = c.is_super_admin
         c.can_create_repo_group = c.is_super_admin
         c.can_create_user_group = c.is_super_admin
 
         c.is_delegated_admin = False
 
         if not c.auth_user.is_default and not c.is_super_admin:
             c.can_create_repo = h.HasPermissionAny("hg.create.repository")(
                 user=self.request.user
             )
             repositories = c.auth_user.repositories_admin or c.can_create_repo
 
             c.can_create_repo_group = h.HasPermissionAny("hg.repogroup.create.true")(
                 user=self.request.user
             )
             repository_groups = (
                 c.auth_user.repository_groups_admin or c.can_create_repo_group
             )
 
             c.can_create_user_group = h.HasPermissionAny("hg.usergroup.create.true")(
                 user=self.request.user
             )
             user_groups = c.auth_user.user_groups_admin or c.can_create_user_group
             # delegated admin can create, or manage some objects
             c.is_delegated_admin = repositories or repository_groups or user_groups
         return c
 
     def _get_template_context(self, tmpl_args, **kwargs):
         local_tmpl_args = {"defaults": {}, "errors": {}, "c": tmpl_args}
         local_tmpl_args.update(kwargs)
         return local_tmpl_args
 
     def load_default_context(self):
         """
         example:
 
         def load_default_context(self):
             c = self._get_local_tmpl_context()
             c.custom_var = 'foobar'
 
             return c
         """
         raise NotImplementedError("Needs implementation in view class")
 
 
 class RepoAppView(BaseAppView):
     def __init__(self, context, request):
         super().__init__(context, request)
         self.db_repo = request.db_repo
         self.db_repo_name = self.db_repo.repo_name
         self.db_repo_pull_requests = ScmModel().get_pull_requests(self.db_repo)
         self.db_repo_artifacts = ScmModel().get_artifacts(self.db_repo)
         self.db_repo_patterns = IssueTrackerSettingsModel(repo=self.db_repo)
 
     def _handle_missing_requirements(self, error):
         log.error(
             "Requirements are missing for repository %s: %s",
             self.db_repo_name,
             safe_str(error),
         )
 
     def _prepare_and_set_clone_url(self, c):
         username = ""
         if self._rhodecode_user.username != User.DEFAULT_USER:
             username = self._rhodecode_user.username
 
         _def_clone_uri = c.clone_uri_tmpl
         _def_clone_uri_id = c.clone_uri_id_tmpl
         _def_clone_uri_ssh = c.clone_uri_ssh_tmpl
 
         c.clone_repo_url = self.db_repo.clone_url(
             user=username, uri_tmpl=_def_clone_uri
         )
         c.clone_repo_url_id = self.db_repo.clone_url(
             user=username, uri_tmpl=_def_clone_uri_id
         )
         c.clone_repo_url_ssh = self.db_repo.clone_url(
             uri_tmpl=_def_clone_uri_ssh, ssh=True
         )
 
     def _get_local_tmpl_context(self, include_app_defaults=True):
         _ = self.request.translate
         c = super()._get_local_tmpl_context(include_app_defaults=include_app_defaults)
 
         # register common vars for this type of view
         c.rhodecode_db_repo = self.db_repo
         c.repo_name = self.db_repo_name
         c.repository_pull_requests = self.db_repo_pull_requests
         c.repository_artifacts = self.db_repo_artifacts
         c.repository_is_user_following = ScmModel().is_following_repo(
             self.db_repo_name, self._rhodecode_user.user_id
         )
         self.path_filter = PathFilter(None)
 
         c.repository_requirements_missing = {}
         try:
             self.rhodecode_vcs_repo = self.db_repo.scm_instance()
             # NOTE(marcink):
             # comparison to None since if it's an object __bool__ is expensive to
             # calculate
             if self.rhodecode_vcs_repo is not None:
                 path_perms = self.rhodecode_vcs_repo.get_path_permissions(
                     c.auth_user.username
                 )
                 self.path_filter = PathFilter(path_perms)
         except RepositoryRequirementError as e:
             c.repository_requirements_missing = {"error": str(e)}
             self._handle_missing_requirements(e)
             self.rhodecode_vcs_repo = None
 
         c.path_filter = self.path_filter  # used by atom_feed_entry.mako
 
         if self.rhodecode_vcs_repo is None:
             # unable to fetch this repo as vcs instance, report back to user
             log.debug(
                 "Repository was not found on filesystem, check if it exists or is not damaged"
             )
             h.flash(
                 _(
                     "The repository `%(repo_name)s` cannot be loaded in filesystem. "
                     "Please check if it exist, or is not damaged."
                 )
                 % {"repo_name": c.repo_name},
                 category="error",
                 ignore_duplicate=True,
             )
             if c.repository_requirements_missing:
                 route = self.request.matched_route.name
                 if route.startswith(("edit_repo", "repo_summary")):
                     # allow summary and edit repo on missing requirements
                     return c
 
                 raise HTTPFound(
                     h.route_path("repo_summary", repo_name=self.db_repo_name)
                 )
 
             else:  # redirect if we don't show missing requirements
                 raise HTTPFound(h.route_path("home"))
 
         c.has_origin_repo_read_perm = False
         if self.db_repo.fork:
             c.has_origin_repo_read_perm = h.HasRepoPermissionAny(
                 "repository.write", "repository.read", "repository.admin"
             )(self.db_repo.fork.repo_name, "summary fork link")
 
         return c
 
     def _get_f_path_unchecked(self, matchdict, default=None):
         """
         Should only be used by redirects, everything else should call _get_f_path
         """
         f_path = matchdict.get("f_path")
         if f_path:
             # fix for multiple initial slashes that causes errors for GIT
             return f_path.lstrip("/")
 
         return default
 
     def _get_f_path(self, matchdict, default=None):
         f_path_match = self._get_f_path_unchecked(matchdict, default)
         return self.path_filter.assert_path_permissions(f_path_match)
 
     def _get_general_setting(self, target_repo, settings_key, default=False):
         settings_model = VcsSettingsModel(repo=target_repo)
         settings = settings_model.get_general_settings()
         return settings.get(settings_key, default)
 
     def _get_repo_setting(self, target_repo, settings_key, default=False):
         settings_model = VcsSettingsModel(repo=target_repo)
         settings = settings_model.get_repo_settings_inherited()
         return settings.get(settings_key, default)
 
     def _get_readme_data(self, db_repo, renderer_type, commit_id=None, path="/"):
         log.debug("Looking for README file at path %s", path)
         if commit_id:
             landing_commit_id = commit_id
         else:
             landing_commit = db_repo.get_landing_commit()
             if isinstance(landing_commit, EmptyCommit):
                 return None, None
             landing_commit_id = landing_commit.raw_id
 
         cache_namespace_uid = f"repo.{db_repo.repo_id}"
         region = rc_cache.get_or_create_region(
             "cache_repo", cache_namespace_uid, use_async_runner=False
         )
         start = time.time()
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid)
         def generate_repo_readme(
             repo_id, _commit_id, _repo_name, _readme_search_path, _renderer_type
         ):
             readme_data = None
             readme_filename = None
 
             commit = db_repo.get_commit(_commit_id)
             log.debug("Searching for a README file at commit %s.", _commit_id)
             readme_node = ReadmeFinder(_renderer_type).search(
                 commit, path=_readme_search_path
             )
 
             if readme_node:
                 log.debug("Found README node: %s", readme_node)
 
                 relative_urls = {
                     "raw": h.route_path(
                         "repo_file_raw",
                         repo_name=_repo_name,
                         commit_id=commit.raw_id,
                         f_path=readme_node.path,
                     ),
                     "standard": h.route_path(
                         "repo_files",
                         repo_name=_repo_name,
                         commit_id=commit.raw_id,
                         f_path=readme_node.path,
                     ),
                 }
 
                 readme_data = self._render_readme_or_none(
                     commit, readme_node, relative_urls
                 )
                 readme_filename = readme_node.str_path
 
             return readme_data, readme_filename
 
         readme_data, readme_filename = generate_repo_readme(
             db_repo.repo_id,
             landing_commit_id,
             db_repo.repo_name,
             path,
             renderer_type,
         )
 
         compute_time = time.time() - start
         log.debug(
             "Repo README for path %s generated and computed in %.4fs",
             path,
             compute_time,
         )
         return readme_data, readme_filename
 
     def _render_readme_or_none(self, commit, readme_node, relative_urls):
         log.debug("Found README file `%s` rendering...", readme_node.path)
         renderer = MarkupRenderer()
         try:
             html_source = renderer.render(
                 readme_node.str_content, filename=readme_node.path
             )
             if relative_urls:
                 return relative_links(html_source, relative_urls)
             return html_source
         except Exception:
             log.exception("Exception while trying to render the README")
 
     def get_recache_flag(self):
         for flag_name in ["force_recache", "force-recache", "no-cache"]:
             flag_val = self.request.GET.get(flag_name)
             if str2bool(flag_val):
                 return True
         return False
 
     def get_commit_preload_attrs(cls):
         pre_load = [
             "author",
             "branch",
             "date",
             "message",
             "parents",
             "obsolete",
             "phase",
             "hidden",
         ]
         return pre_load
 
 
 class PathFilter(object):
     # Expects and instance of BasePathPermissionChecker or None
     def __init__(self, permission_checker):
         self.permission_checker = permission_checker
 
     def assert_path_permissions(self, path):
         if self.path_access_allowed(path):
             return path
         raise HTTPForbidden()
 
     def path_access_allowed(self, path):
         log.debug("Checking ACL permissions for PathFilter for `%s`", path)
         if self.permission_checker:
             has_access = path and self.permission_checker.has_access(path)
             log.debug(
                 "ACL Permissions checker enabled, ACL Check has_access: %s", has_access
             )
             return has_access
 
         log.debug("ACL permissions checker not enabled, skipping...")
         return True
 
     def filter_patchset(self, patchset):
         if not self.permission_checker or not patchset:
             return patchset, False
         had_filtered = False
         filtered_patchset = []
         for patch in patchset:
             filename = patch.get("filename", None)
             if not filename or self.permission_checker.has_access(filename):
                 filtered_patchset.append(patch)
             else:
                 had_filtered = True
         if had_filtered:
             if isinstance(patchset, diffs.LimitedDiffContainer):
                 filtered_patchset = diffs.LimitedDiffContainer(
                     patchset.diff_limit, patchset.cur_diff_size, filtered_patchset
                 )
             return filtered_patchset, True
         else:
             return patchset, False
 
     def render_patchset_filtered(
         self, diffset, patchset, source_ref=None, target_ref=None
     ):
         filtered_patchset, has_hidden_changes = self.filter_patchset(patchset)
         result = diffset.render_patchset(
             filtered_patchset, source_ref=source_ref, target_ref=target_ref
         )
         result.has_hidden_changes = has_hidden_changes
         return result
 
     def get_raw_patch(self, diff_processor):
         if self.permission_checker is None:
             return diff_processor.as_raw()
         elif self.permission_checker.has_full_access:
             return diff_processor.as_raw()
         else:
             return "# Repository has user-specific filters, raw patch generation is disabled."
 
     @property
     def is_enabled(self):
         return self.permission_checker is not None
 
 
 class RepoGroupAppView(BaseAppView):
     def __init__(self, context, request):
         super().__init__(context, request)
         self.db_repo_group = request.db_repo_group
         self.db_repo_group_name = self.db_repo_group.group_name
 
     def _get_local_tmpl_context(self, include_app_defaults=True):
         _ = self.request.translate
         c = super()._get_local_tmpl_context(include_app_defaults=include_app_defaults)
         c.repo_group = self.db_repo_group
         return c
 
     def _revoke_perms_on_yourself(self, form_result):
         _updates = [
             u
             for u in form_result["perm_updates"]
             if self._rhodecode_user.user_id == int(u[0])
         ]
         _additions = [
             u
             for u in form_result["perm_additions"]
             if self._rhodecode_user.user_id == int(u[0])
         ]
         _deletions = [
             u
             for u in form_result["perm_deletions"]
             if self._rhodecode_user.user_id == int(u[0])
         ]
         admin_perm = "group.admin"
         if (
             _updates
             and _updates[0][1] != admin_perm
             or _additions
             and _additions[0][1] != admin_perm
             or _deletions
             and _deletions[0][1] != admin_perm
         ):
             return True
         return False
 
 
 class UserGroupAppView(BaseAppView):
     def __init__(self, context, request):
         super().__init__(context, request)
         self.db_user_group = request.db_user_group
         self.db_user_group_name = self.db_user_group.users_group_name
 
 
 class UserAppView(BaseAppView):
     def __init__(self, context, request):
         super().__init__(context, request)
         self.db_user = request.db_user
         self.db_user_id = self.db_user.user_id
 
         _ = self.request.translate
         if not request.db_user_supports_default:
             if self.db_user.username == User.DEFAULT_USER:
                 h.flash(
                     _("Editing user `{}` is disabled.".format(User.DEFAULT_USER)),
                     category="warning",
                 )
                 raise HTTPFound(h.route_path("users"))
 
 
 class DataGridAppView(object):
     """
     Common class to have re-usable grid rendering components
     """
 
     def _extract_ordering(self, request, column_map=None):
         column_map = column_map or {}
         column_index = safe_int(request.GET.get("order[0][column]"))
         order_dir = request.GET.get("order[0][dir]", "desc")
         order_by = request.GET.get("columns[%s][data][sort]" % column_index, "name_raw")
 
         # translate datatable to DB columns
         order_by = column_map.get(order_by) or order_by
 
         search_q = request.GET.get("search[value]")
         return search_q, order_by, order_dir
 
     def _extract_chunk(self, request):
         start = safe_int(request.GET.get("start"), 0)
         length = safe_int(request.GET.get("length"), 25)
         draw = safe_int(request.GET.get("draw"))
         return draw, start, length
 
     def _get_order_col(self, order_by, model):
         if isinstance(order_by, str):
             try:
                 return operator.attrgetter(order_by)(model)
             except AttributeError:
                 return None
         else:
             return order_by
 
 
 class BaseReferencesView(RepoAppView):
     """
     Base for reference view for branches, tags and bookmarks.
     """
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         return c
 
     def load_refs_context(self, ref_items, partials_template):
         _render = self.request.get_partial_renderer(partials_template)
         pre_load = ["author", "date", "message", "parents"]
 
         is_svn = h.is_svn(self.rhodecode_vcs_repo)
         is_hg = h.is_hg(self.rhodecode_vcs_repo)
 
         format_ref_id = get_format_ref_id(self.rhodecode_vcs_repo)
 
         closed_refs = {}
         if is_hg:
             closed_refs = self.rhodecode_vcs_repo.branches_closed
 
         data = []
         for ref_name, commit_id in ref_items:
             commit = self.rhodecode_vcs_repo.get_commit(
                 commit_id=commit_id, pre_load=pre_load
             )
             closed = ref_name in closed_refs
 
             # TODO: johbo: Unify generation of reference links
             use_commit_id = "/" in ref_name or is_svn
 
             if use_commit_id:
                 files_url = h.route_path(
                     "repo_files",
                     repo_name=self.db_repo_name,
                     f_path=ref_name if is_svn else "",
                     commit_id=commit_id,
                     _query=dict(at=ref_name),
                 )
 
             else:
                 files_url = h.route_path(
                     "repo_files",
                     repo_name=self.db_repo_name,
                     f_path=ref_name if is_svn else "",
                     commit_id=ref_name,
                     _query=dict(at=ref_name),
                 )
 
             data.append(
                 {
                     "name": _render("name", ref_name, files_url, closed),
                     "name_raw": ref_name,
+                    "closed": closed,
                     "date": _render("date", commit.date),
                     "date_raw": datetime_to_time(commit.date),
                     "author": _render("author", commit.author),
                     "commit": _render(
                         "commit", commit.message, commit.raw_id, commit.idx
                     ),
                     "commit_raw": commit.idx,
                     "compare": _render(
                         "compare", format_ref_id(ref_name, commit.raw_id)
                     ),
                 }
             )
 
         return data
 
 
 class RepoRoutePredicate(object):
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         return f"repo_route = {self.val}"
 
     phash = text
 
     def __call__(self, info, request):
         if hasattr(request, "vcs_call"):
             # skip vcs calls
             return
 
         repo_name = info["match"]["repo_name"]
 
         repo_name_parts = repo_name.split("/")
         repo_slugs = [x for x in (repo_name_slug(x) for x in repo_name_parts)]
 
         if repo_name_parts != repo_slugs:
             # short-skip if the repo-name doesn't follow slug rule
             log.warning(
                 "repo_name: %s is different than slug %s", repo_name_parts, repo_slugs
             )
             return False
 
         repo_model = repo.RepoModel()
 
         by_name_match = repo_model.get_by_repo_name(repo_name, cache=False)
 
         def redirect_if_creating(route_info, db_repo):
             skip_views = ["edit_repo_advanced_delete"]
             route = route_info["route"]
             # we should skip delete view so we can actually "remove" repositories
             # if they get stuck in creating state.
             if route.name in skip_views:
                 return
 
             if db_repo.repo_state in [repo.Repository.STATE_PENDING]:
                 repo_creating_url = request.route_path(
                     "repo_creating", repo_name=db_repo.repo_name
                 )
                 raise HTTPFound(repo_creating_url)
 
         if by_name_match:
             # register this as request object we can re-use later
             request.db_repo = by_name_match
             request.db_repo_name = request.db_repo.repo_name
 
             redirect_if_creating(info, by_name_match)
             return True
 
         by_id_match = repo_model.get_repo_by_id(repo_name)
         if by_id_match:
             request.db_repo = by_id_match
             request.db_repo_name = request.db_repo.repo_name
             redirect_if_creating(info, by_id_match)
             return True
 
         return False
 
 
 class RepoForbidArchivedRoutePredicate(object):
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         return f"repo_forbid_archived = {self.val}"
 
     phash = text
 
     def __call__(self, info, request):
         _ = request.translate
         rhodecode_db_repo = request.db_repo
 
         log.debug(
             "%s checking if archived flag for repo for %s",
             self.__class__.__name__,
             rhodecode_db_repo.repo_name,
         )
 
         if rhodecode_db_repo.archived:
             log.warning(
                 "Current view is not supported for archived repo:%s",
                 rhodecode_db_repo.repo_name,
             )
 
             h.flash(
                 h.literal(_("Action not supported for archived repository.")),
                 category="warning",
             )
             summary_url = request.route_path(
                 "repo_summary", repo_name=rhodecode_db_repo.repo_name
             )
             raise HTTPFound(summary_url)
         return True
 
 
 class RepoTypeRoutePredicate(object):
     def __init__(self, val, config):
         self.val = val or ["hg", "git", "svn"]
 
     def text(self):
         return f"repo_accepted_type = {self.val}"
 
     phash = text
 
     def __call__(self, info, request):
         if hasattr(request, "vcs_call"):
             # skip vcs calls
             return
 
         rhodecode_db_repo = request.db_repo
 
         log.debug(
             "%s checking repo type for %s in %s",
             self.__class__.__name__,
             rhodecode_db_repo.repo_type,
             self.val,
         )
 
         if rhodecode_db_repo.repo_type in self.val:
             return True
         else:
             log.warning(
                 "Current view is not supported for repo type:%s",
                 rhodecode_db_repo.repo_type,
             )
             return False
 
 
 class RepoGroupRoutePredicate(object):
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         return f"repo_group_route = {self.val}"
 
     phash = text
 
     def __call__(self, info, request):
         if hasattr(request, "vcs_call"):
             # skip vcs calls
             return
 
         repo_group_name = info["match"]["repo_group_name"]
 
         repo_group_name_parts = repo_group_name.split("/")
         repo_group_slugs = [
             x for x in [repo_name_slug(x) for x in repo_group_name_parts]
         ]
         if repo_group_name_parts != repo_group_slugs:
             # short-skip if the repo-name doesn't follow slug rule
             log.warning(
                 "repo_group_name: %s is different than slug %s",
                 repo_group_name_parts,
                 repo_group_slugs,
             )
             return False
 
         repo_group_model = repo_group.RepoGroupModel()
         by_name_match = repo_group_model.get_by_group_name(repo_group_name, cache=False)
 
         if by_name_match:
             # register this as request object we can re-use later
             request.db_repo_group = by_name_match
             request.db_repo_group_name = request.db_repo_group.group_name
             return True
 
         return False
 
 
 class UserGroupRoutePredicate(object):
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         return f"user_group_route = {self.val}"
 
     phash = text
 
     def __call__(self, info, request):
         if hasattr(request, "vcs_call"):
             # skip vcs calls
             return
 
         user_group_id = info["match"]["user_group_id"]
         user_group_model = user_group.UserGroup()
         by_id_match = user_group_model.get(user_group_id, cache=False)
 
         if by_id_match:
             # register this as request object we can re-use later
             request.db_user_group = by_id_match
             return True
 
         return False
 
 
 class UserRoutePredicateBase(object):
     supports_default = None
 
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         raise NotImplementedError()
 
     def __call__(self, info, request):
         if hasattr(request, "vcs_call"):
             # skip vcs calls
             return
 
         user_id = info["match"]["user_id"]
         user_model = user.User()
         by_id_match = user_model.get(user_id, cache=False)
 
         if by_id_match:
             # register this as request object we can re-use later
             request.db_user = by_id_match
             request.db_user_supports_default = self.supports_default
             return True
 
         return False
 
 
 class UserRoutePredicate(UserRoutePredicateBase):
     supports_default = False
 
     def text(self):
         return f"user_route = {self.val}"
 
     phash = text
 
 
 class UserRouteWithDefaultPredicate(UserRoutePredicateBase):
     supports_default = True
 
     def text(self):
         return f"user_with_default_route = {self.val}"
 
     phash = text
 
 
 def includeme(config):
     config.add_route_predicate("repo_route", RepoRoutePredicate)
     config.add_route_predicate("repo_accepted_types", RepoTypeRoutePredicate)
     config.add_route_predicate(
         "repo_forbid_when_archived", RepoForbidArchivedRoutePredicate
     )
     config.add_route_predicate("repo_group_route", RepoGroupRoutePredicate)
     config.add_route_predicate("user_group_route", UserGroupRoutePredicate)
     config.add_route_predicate("user_route_with_default", UserRouteWithDefaultPredicate)
     config.add_route_predicate("user_route", UserRoutePredicate)

rhodecode/apps/admin/tests/test_admin_repos.py

@@ -1,497 +1,497 b''
 
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import urllib.request
 import urllib.parse
 import urllib.error
 
 import mock
 import pytest
 
 from rhodecode.apps._base import ADMIN_PREFIX
 from rhodecode.lib import auth
 from rhodecode.lib.utils2 import safe_str
 from rhodecode.lib import helpers as h
 from rhodecode.model.db import (
     Repository, RepoGroup, UserRepoToPerm, User, Permission)
 from rhodecode.model.meta import Session
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.user import UserModel
 from rhodecode.tests import (
     login_user_session, assert_session_flash, TEST_USER_ADMIN_LOGIN,
     TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS)
 from rhodecode.tests.fixture import Fixture, error_function
 from rhodecode.tests.utils import repo_on_filesystem
 from rhodecode.tests.routes import route_path
 
 fixture = Fixture()
 
 
 def _get_permission_for_user(user, repo):
     perm = UserRepoToPerm.query()\
         .filter(UserRepoToPerm.repository ==
                 Repository.get_by_repo_name(repo))\
         .filter(UserRepoToPerm.user == User.get_by_username(user))\
         .all()
     return perm
 
 
 @pytest.mark.usefixtures("app")
 class TestAdminRepos(object):
 
     def test_repo_list(self, autologin_user, user_util, xhr_header):
         repo = user_util.create_repo()
         repo_name = repo.repo_name
         response = self.app.get(
             route_path('repos_data'), status=200,
             extra_environ=xhr_header)
 
         response.mustcontain(repo_name)
 
     def test_create_page_restricted_to_single_backend(self, autologin_user, backend):
         with mock.patch('rhodecode.BACKENDS', {'git': 'git'}):
             response = self.app.get(route_path('repo_new'), status=200)
         assert_response = response.assert_response()
         element = assert_response.get_element('[name=repo_type]')
         assert element.get('value') == 'git'
 
     def test_create_page_non_restricted_backends(self, autologin_user, backend):
         response = self.app.get(route_path('repo_new'), status=200)
         assert_response = response.assert_response()
         assert ['hg', 'git', 'svn'] == [x.get('value') for x in assert_response.get_elements('[name=repo_type]')]
 
     @pytest.mark.parametrize(
         "suffix", ['', 'xxa'], ids=['', 'non-ascii'])
     def test_create(self, autologin_user, backend, suffix, csrf_token):
         repo_name_unicode = backend.new_repo_name(suffix=suffix)
         repo_name = repo_name_unicode
 
         description_unicode = 'description for newly created repo' + suffix
         description = description_unicode
 
         response = self.app.post(
             route_path('repo_create'),
             fixture._get_repo_create_params(
                 repo_private=False,
                 repo_name=repo_name,
                 repo_type=backend.alias,
                 repo_description=description,
                 csrf_token=csrf_token),
             status=302)
 
         self.assert_repository_is_created_correctly(
             repo_name, description, backend)
 
     def test_create_numeric_name(self, autologin_user, backend, csrf_token):
         numeric_repo = '1234'
         repo_name = numeric_repo
         description = 'description for newly created repo' + numeric_repo
         self.app.post(
             route_path('repo_create'),
             fixture._get_repo_create_params(
                 repo_private=False,
                 repo_name=repo_name,
                 repo_type=backend.alias,
                 repo_description=description,
                 csrf_token=csrf_token))
 
         self.assert_repository_is_created_correctly(
             repo_name, description, backend)
 
     @pytest.mark.parametrize("suffix", ['', '_ąćę'], ids=['', 'non-ascii'])
     def test_create_in_group(
             self, autologin_user, backend, suffix, csrf_token):
         # create GROUP
         group_name = f'sometest_{backend.alias}'
         gr = RepoGroupModel().create(group_name=group_name,
                                      group_description='test',
                                      owner=TEST_USER_ADMIN_LOGIN)
         Session().commit()
 
         repo_name = f'ingroup{suffix}'
         repo_name_full = RepoGroup.url_sep().join([group_name, repo_name])
         description = 'description for newly created repo'
 
         self.app.post(
             route_path('repo_create'),
             fixture._get_repo_create_params(
                 repo_private=False,
                 repo_name=safe_str(repo_name),
                 repo_type=backend.alias,
                 repo_description=description,
                 repo_group=gr.group_id,
                 csrf_token=csrf_token))
 
         # TODO: johbo: Cleanup work to fixture
         try:
             self.assert_repository_is_created_correctly(
                 repo_name_full, description, backend)
 
             new_repo = RepoModel().get_by_repo_name(repo_name_full)
             inherited_perms = UserRepoToPerm.query().filter(
                 UserRepoToPerm.repository_id == new_repo.repo_id).all()
             assert len(inherited_perms) == 1
         finally:
             RepoModel().delete(repo_name_full)
             RepoGroupModel().delete(group_name)
             Session().commit()
 
     def test_create_in_group_numeric_name(
             self, autologin_user, backend, csrf_token):
         # create GROUP
         group_name = 'sometest_%s' % backend.alias
         gr = RepoGroupModel().create(group_name=group_name,
                                      group_description='test',
                                      owner=TEST_USER_ADMIN_LOGIN)
         Session().commit()
 
         repo_name = '12345'
         repo_name_full = RepoGroup.url_sep().join([group_name, repo_name])
         description = 'description for newly created repo'
         self.app.post(
             route_path('repo_create'),
             fixture._get_repo_create_params(
                 repo_private=False,
                 repo_name=repo_name,
                 repo_type=backend.alias,
                 repo_description=description,
                 repo_group=gr.group_id,
                 csrf_token=csrf_token))
 
         # TODO: johbo: Cleanup work to fixture
         try:
             self.assert_repository_is_created_correctly(
                 repo_name_full, description, backend)
 
             new_repo = RepoModel().get_by_repo_name(repo_name_full)
             inherited_perms = UserRepoToPerm.query()\
                 .filter(UserRepoToPerm.repository_id == new_repo.repo_id).all()
             assert len(inherited_perms) == 1
         finally:
             RepoModel().delete(repo_name_full)
             RepoGroupModel().delete(group_name)
             Session().commit()
 
     def test_create_in_group_without_needed_permissions(self, backend):
         session = login_user_session(
             self.app, TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS)
         csrf_token = auth.get_csrf_token(session)
         # revoke
         user_model = UserModel()
         # disable fork and create on default user
         user_model.revoke_perm(User.DEFAULT_USER, 'hg.create.repository')
         user_model.grant_perm(User.DEFAULT_USER, 'hg.create.none')
         user_model.revoke_perm(User.DEFAULT_USER, 'hg.fork.repository')
         user_model.grant_perm(User.DEFAULT_USER, 'hg.fork.none')
 
         # disable on regular user
         user_model.revoke_perm(TEST_USER_REGULAR_LOGIN, 'hg.create.repository')
         user_model.grant_perm(TEST_USER_REGULAR_LOGIN, 'hg.create.none')
         user_model.revoke_perm(TEST_USER_REGULAR_LOGIN, 'hg.fork.repository')
         user_model.grant_perm(TEST_USER_REGULAR_LOGIN, 'hg.fork.none')
         Session().commit()
 
         # create GROUP
         group_name = 'reg_sometest_%s' % backend.alias
         gr = RepoGroupModel().create(group_name=group_name,
                                      group_description='test',
                                      owner=TEST_USER_ADMIN_LOGIN)
         Session().commit()
         repo_group_id = gr.group_id
 
         group_name_allowed = 'reg_sometest_allowed_%s' % backend.alias
         gr_allowed = RepoGroupModel().create(
             group_name=group_name_allowed,
             group_description='test',
             owner=TEST_USER_REGULAR_LOGIN)
         allowed_repo_group_id = gr_allowed.group_id
         Session().commit()
 
         repo_name = 'ingroup'
         description = 'description for newly created repo'
         response = self.app.post(
             route_path('repo_create'),
             fixture._get_repo_create_params(
                 repo_private=False,
                 repo_name=repo_name,
                 repo_type=backend.alias,
                 repo_description=description,
                 repo_group=repo_group_id,
                 csrf_token=csrf_token))
 
         response.mustcontain('Invalid value')
 
         # user is allowed to create in this group
         repo_name = 'ingroup'
         repo_name_full = RepoGroup.url_sep().join(
             [group_name_allowed, repo_name])
         description = 'description for newly created repo'
         response = self.app.post(
             route_path('repo_create'),
             fixture._get_repo_create_params(
                 repo_private=False,
                 repo_name=repo_name,
                 repo_type=backend.alias,
                 repo_description=description,
                 repo_group=allowed_repo_group_id,
                 csrf_token=csrf_token))
 
         # TODO: johbo: Cleanup in pytest fixture
         try:
             self.assert_repository_is_created_correctly(
                 repo_name_full, description, backend)
 
             new_repo = RepoModel().get_by_repo_name(repo_name_full)
             inherited_perms = UserRepoToPerm.query().filter(
                 UserRepoToPerm.repository_id == new_repo.repo_id).all()
             assert len(inherited_perms) == 1
 
             assert repo_on_filesystem(repo_name_full)
         finally:
             RepoModel().delete(repo_name_full)
             RepoGroupModel().delete(group_name)
             RepoGroupModel().delete(group_name_allowed)
             Session().commit()
 
     def test_create_in_group_inherit_permissions(self, autologin_user, backend,
                                                  csrf_token):
         # create GROUP
         group_name = 'sometest_%s' % backend.alias
         gr = RepoGroupModel().create(group_name=group_name,
                                      group_description='test',
                                      owner=TEST_USER_ADMIN_LOGIN)
         perm = Permission.get_by_key('repository.write')
         RepoGroupModel().grant_user_permission(
             gr, TEST_USER_REGULAR_LOGIN, perm)
 
         # add repo permissions
         Session().commit()
         repo_group_id = gr.group_id
         repo_name = 'ingroup_inherited_%s' % backend.alias
         repo_name_full = RepoGroup.url_sep().join([group_name, repo_name])
         description = 'description for newly created repo'
         self.app.post(
             route_path('repo_create'),
             fixture._get_repo_create_params(
                 repo_private=False,
                 repo_name=repo_name,
                 repo_type=backend.alias,
                 repo_description=description,
                 repo_group=repo_group_id,
                 repo_copy_permissions=True,
                 csrf_token=csrf_token))
 
         # TODO: johbo: Cleanup to pytest fixture
         try:
             self.assert_repository_is_created_correctly(
                 repo_name_full, description, backend)
         except Exception:
             RepoGroupModel().delete(group_name)
             Session().commit()
             raise
 
         # check if inherited permissions are applied
         new_repo = RepoModel().get_by_repo_name(repo_name_full)
         inherited_perms = UserRepoToPerm.query().filter(
             UserRepoToPerm.repository_id == new_repo.repo_id).all()
         assert len(inherited_perms) == 2
 
         assert TEST_USER_REGULAR_LOGIN in [
             x.user.username for x in inherited_perms]
         assert 'repository.write' in [
             x.permission.permission_name for x in inherited_perms]
 
         RepoModel().delete(repo_name_full)
         RepoGroupModel().delete(group_name)
         Session().commit()
 
     @pytest.mark.xfail_backends(
         "git", "hg", reason="Missing reposerver support")
     def test_create_with_clone_uri(self, autologin_user, backend, reposerver,
                                    csrf_token):
         source_repo = backend.create_repo(number_of_commits=2)
         source_repo_name = source_repo.repo_name
         reposerver.serve(source_repo.scm_instance())
 
         repo_name = backend.new_repo_name()
         response = self.app.post(
             route_path('repo_create'),
             fixture._get_repo_create_params(
                 repo_private=False,
                 repo_name=repo_name,
                 repo_type=backend.alias,
                 repo_description='',
                 clone_uri=reposerver.url,
                 csrf_token=csrf_token),
             status=302)
 
         # Should be redirected to the creating page
         response.mustcontain('repo_creating')
 
         # Expecting that both repositories have same history
         source_repo = RepoModel().get_by_repo_name(source_repo_name)
         source_vcs = source_repo.scm_instance()
         repo = RepoModel().get_by_repo_name(repo_name)
         repo_vcs = repo.scm_instance()
         assert source_vcs[0].message == repo_vcs[0].message
         assert source_vcs.count() == repo_vcs.count()
         assert source_vcs.commit_ids == repo_vcs.commit_ids
 
     @pytest.mark.xfail_backends("svn", reason="Depends on import support")
     def test_create_remote_repo_wrong_clone_uri(self, autologin_user, backend,
                                                 csrf_token):
         repo_name = backend.new_repo_name()
         description = 'description for newly created repo'
         response = self.app.post(
             route_path('repo_create'),
             fixture._get_repo_create_params(
                 repo_private=False,
                 repo_name=repo_name,
                 repo_type=backend.alias,
                 repo_description=description,
                 clone_uri='http://repo.invalid/repo',
                 csrf_token=csrf_token))
         response.mustcontain('invalid clone url')
 
     @pytest.mark.xfail_backends("svn", reason="Depends on import support")
     def test_create_remote_repo_wrong_clone_uri_hg_svn(
             self, autologin_user, backend, csrf_token):
         repo_name = backend.new_repo_name()
         description = 'description for newly created repo'
         response = self.app.post(
             route_path('repo_create'),
             fixture._get_repo_create_params(
                 repo_private=False,
                 repo_name=repo_name,
                 repo_type=backend.alias,
                 repo_description=description,
                 clone_uri='svn+http://svn.invalid/repo',
                 csrf_token=csrf_token))
         response.mustcontain('invalid clone url')
 
     def test_create_with_git_suffix(
             self, autologin_user, backend, csrf_token):
         repo_name = backend.new_repo_name() + ".git"
         description = 'description for newly created repo'
         response = self.app.post(
             route_path('repo_create'),
             fixture._get_repo_create_params(
                 repo_private=False,
                 repo_name=repo_name,
                 repo_type=backend.alias,
                 repo_description=description,
                 csrf_token=csrf_token))
         response.mustcontain('Repository name cannot end with .git')
 
     def test_default_user_cannot_access_private_repo_in_a_group(
             self, autologin_user, user_util, backend):
 
         group = user_util.create_repo_group()
 
         repo = backend.create_repo(
             repo_private=True, repo_group=group, repo_copy_permissions=True)
 
         permissions = _get_permission_for_user(
             user='default', repo=repo.repo_name)
         assert len(permissions) == 1
         assert permissions[0].permission.permission_name == 'repository.none'
         assert permissions[0].repository.private is True
 
     def test_create_on_top_level_without_permissions(self, backend):
         session = login_user_session(
             self.app, TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS)
         csrf_token = auth.get_csrf_token(session)
 
         # revoke
         user_model = UserModel()
         # disable fork and create on default user
         user_model.revoke_perm(User.DEFAULT_USER, 'hg.create.repository')
         user_model.grant_perm(User.DEFAULT_USER, 'hg.create.none')
         user_model.revoke_perm(User.DEFAULT_USER, 'hg.fork.repository')
         user_model.grant_perm(User.DEFAULT_USER, 'hg.fork.none')
 
         # disable on regular user
         user_model.revoke_perm(TEST_USER_REGULAR_LOGIN, 'hg.create.repository')
         user_model.grant_perm(TEST_USER_REGULAR_LOGIN, 'hg.create.none')
         user_model.revoke_perm(TEST_USER_REGULAR_LOGIN, 'hg.fork.repository')
         user_model.grant_perm(TEST_USER_REGULAR_LOGIN, 'hg.fork.none')
         Session().commit()
 
         repo_name = backend.new_repo_name()
         description = 'description for newly created repo'
         response = self.app.post(
             route_path('repo_create'),
             fixture._get_repo_create_params(
                 repo_private=False,
                 repo_name=repo_name,
                 repo_type=backend.alias,
                 repo_description=description,
                 csrf_token=csrf_token))
 
         response.mustcontain(
-            u"You do not have the permission to store repositories in "
-            u"the root location.")
+            "You do not have the permission to store repositories in "
+            "the root location.")
 
     @mock.patch.object(RepoModel, '_create_filesystem_repo', error_function)
     def test_create_repo_when_filesystem_op_fails(
             self, autologin_user, backend, csrf_token):
         repo_name = backend.new_repo_name()
         description = 'description for newly created repo'
 
         response = self.app.post(
             route_path('repo_create'),
             fixture._get_repo_create_params(
                 repo_private=False,
                 repo_name=repo_name,
                 repo_type=backend.alias,
                 repo_description=description,
                 csrf_token=csrf_token))
 
         assert_session_flash(
             response, 'Error creating repository %s' % repo_name)
         # repo must not be in db
         assert backend.repo is None
         # repo must not be in filesystem !
         assert not repo_on_filesystem(repo_name)
 
     def assert_repository_is_created_correctly(self, repo_name, description, backend):
         url_quoted_repo_name = urllib.parse.quote(repo_name)
 
         # run the check page that triggers the flash message
         response = self.app.get(
             route_path('repo_creating_check', repo_name=repo_name))
         assert response.json == {'result': True}
 
         flash_msg = 'Created repository <a href="/{}">{}</a>'.format(url_quoted_repo_name, repo_name)
         assert_session_flash(response, flash_msg)
 
         # test if the repo was created in the database
         new_repo = RepoModel().get_by_repo_name(repo_name)
 
         assert new_repo.repo_name == repo_name
         assert new_repo.description == description
 
         # test if the repository is visible in the list ?
         response = self.app.get(
             h.route_path('repo_summary', repo_name=repo_name))
         response.mustcontain(repo_name)
         response.mustcontain(backend.alias)
 
         assert repo_on_filesystem(repo_name)

rhodecode/apps/admin/tests/test_admin_settings.py

@@ -1,678 +1,678 b''
 
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import mock
 import pytest
 
 import rhodecode
 from rhodecode.apps._base import ADMIN_PREFIX
 from rhodecode.lib.hash_utils import md5_safe
 from rhodecode.model.db import RhodeCodeUi
 from rhodecode.model.meta import Session
 from rhodecode.model.settings import SettingsModel, IssueTrackerSettingsModel
 from rhodecode.tests import assert_session_flash
 from rhodecode.tests.routes import route_path
 
 
 UPDATE_DATA_QUALNAME = 'rhodecode.model.update.UpdateModel.get_update_data'
 
 
 @pytest.mark.usefixtures('autologin_user', 'app')
 class TestAdminSettingsController(object):
 
     @pytest.mark.parametrize('urlname', [
         'admin_settings_vcs',
         'admin_settings_mapping',
         'admin_settings_global',
         'admin_settings_visual',
         'admin_settings_email',
         'admin_settings_hooks',
         'admin_settings_search',
     ])
     def test_simple_get(self, urlname):
         self.app.get(route_path(urlname))
 
     def test_create_custom_hook(self, csrf_token):
         response = self.app.post(
             route_path('admin_settings_hooks_update'),
             params={
                 'new_hook_ui_key': 'test_hooks_1',
                 'new_hook_ui_value': 'cd /tmp',
                 'csrf_token': csrf_token})
 
         response = response.follow()
         response.mustcontain('test_hooks_1')
         response.mustcontain('cd /tmp')
 
     def test_create_custom_hook_delete(self, csrf_token):
         response = self.app.post(
             route_path('admin_settings_hooks_update'),
             params={
                 'new_hook_ui_key': 'test_hooks_2',
                 'new_hook_ui_value': 'cd /tmp2',
                 'csrf_token': csrf_token})
 
         response = response.follow()
         response.mustcontain('test_hooks_2')
         response.mustcontain('cd /tmp2')
 
         hook_id = SettingsModel().get_ui_by_key('test_hooks_2').ui_id
 
         # delete
         self.app.post(
             route_path('admin_settings_hooks_delete'),
             params={'hook_id': hook_id, 'csrf_token': csrf_token})
         response = self.app.get(route_path('admin_settings_hooks'))
         response.mustcontain(no=['test_hooks_2'])
         response.mustcontain(no=['cd /tmp2'])
 
 
 @pytest.mark.usefixtures('autologin_user', 'app')
 class TestAdminSettingsGlobal(object):
 
     def test_pre_post_code_code_active(self, csrf_token):
         pre_code = 'rc-pre-code-187652122'
         post_code = 'rc-postcode-98165231'
 
         response = self.post_and_verify_settings({
             'rhodecode_pre_code': pre_code,
             'rhodecode_post_code': post_code,
             'csrf_token': csrf_token,
         })
 
         response = response.follow()
         response.mustcontain(pre_code, post_code)
 
     def test_pre_post_code_code_inactive(self, csrf_token):
         pre_code = 'rc-pre-code-187652122'
         post_code = 'rc-postcode-98165231'
         response = self.post_and_verify_settings({
             'rhodecode_pre_code': '',
             'rhodecode_post_code': '',
             'csrf_token': csrf_token,
         })
 
         response = response.follow()
         response.mustcontain(no=[pre_code, post_code])
 
     def test_captcha_activate(self, csrf_token):
         self.post_and_verify_settings({
             'rhodecode_captcha_private_key': '1234567890',
             'rhodecode_captcha_public_key': '1234567890',
             'csrf_token': csrf_token,
         })
 
         response = self.app.get(ADMIN_PREFIX + '/register')
         response.mustcontain('captcha')
 
     def test_captcha_deactivate(self, csrf_token):
         self.post_and_verify_settings({
             'rhodecode_captcha_private_key': '',
             'rhodecode_captcha_public_key': '1234567890',
             'csrf_token': csrf_token,
         })
 
         response = self.app.get(ADMIN_PREFIX + '/register')
         response.mustcontain(no=['captcha'])
 
     def test_title_change(self, csrf_token):
         old_title = 'RhodeCode'
 
         for new_title in ['Changed', 'Żółwik', old_title]:
             response = self.post_and_verify_settings({
                 'rhodecode_title': new_title,
                 'csrf_token': csrf_token,
             })
 
             response = response.follow()
             response.mustcontain(new_title)
 
     def post_and_verify_settings(self, settings):
         old_title = 'RhodeCode'
         old_realm = 'RhodeCode authentication'
         params = {
             'rhodecode_title': old_title,
             'rhodecode_realm': old_realm,
             'rhodecode_pre_code': '',
             'rhodecode_post_code': '',
             'rhodecode_captcha_private_key': '',
             'rhodecode_captcha_public_key': '',
             'rhodecode_create_personal_repo_group': False,
             'rhodecode_personal_repo_group_pattern': '${username}',
         }
         params.update(settings)
         response = self.app.post(
             route_path('admin_settings_global_update'), params=params)
 
         assert_session_flash(response, 'Updated application settings')
 
         app_settings = SettingsModel().get_all_settings()
         del settings['csrf_token']
         for key, value in settings.items():
             assert app_settings[key] == value
 
         return response
 
 
 @pytest.mark.usefixtures('autologin_user', 'app')
 class TestAdminSettingsVcs(object):
 
     def test_contains_svn_default_patterns(self):
         response = self.app.get(route_path('admin_settings_vcs'))
         expected_patterns = [
             '/trunk',
             '/branches/*',
             '/tags/*',
         ]
         for pattern in expected_patterns:
             response.mustcontain(pattern)
 
     def test_add_new_svn_branch_and_tag_pattern(
             self, backend_svn, form_defaults, disable_sql_cache,
             csrf_token):
         form_defaults.update({
             'new_svn_branch': '/exp/branches/*',
             'new_svn_tag': '/important_tags/*',
             'csrf_token': csrf_token,
         })
 
         response = self.app.post(
             route_path('admin_settings_vcs_update'),
             params=form_defaults, status=302)
         response = response.follow()
 
         # Expect to find the new values on the page
         response.mustcontain('/exp/branches/*')
         response.mustcontain('/important_tags/*')
 
         # Expect that those patterns are used to match branches and tags now
         repo = backend_svn['svn-simple-layout'].scm_instance()
         assert 'exp/branches/exp-sphinx-docs' in repo.branches
         assert 'important_tags/v0.5' in repo.tags
 
     def test_add_same_svn_value_twice_shows_an_error_message(
             self, form_defaults, csrf_token, settings_util):
         settings_util.create_rhodecode_ui('vcs_svn_branch', '/test')
         settings_util.create_rhodecode_ui('vcs_svn_tag', '/test')
 
         response = self.app.post(
             route_path('admin_settings_vcs_update'),
             params={
                 'paths_root_path': form_defaults['paths_root_path'],
                 'new_svn_branch': '/test',
                 'new_svn_tag': '/test',
                 'csrf_token': csrf_token,
             },
             status=200)
 
         response.mustcontain("Pattern already exists")
         response.mustcontain("Some form inputs contain invalid data.")
 
     @pytest.mark.parametrize('section', [
         'vcs_svn_branch',
         'vcs_svn_tag',
     ])
     def test_delete_svn_patterns(
             self, section, csrf_token, settings_util):
         setting = settings_util.create_rhodecode_ui(
             section, '/test_delete', cleanup=False)
 
         self.app.post(
             route_path('admin_settings_vcs_svn_pattern_delete'),
             params={
                 'delete_svn_pattern': setting.ui_id,
                 'csrf_token': csrf_token},
             headers={'X-REQUESTED-WITH': 'XMLHttpRequest'})
 
     @pytest.mark.parametrize('section', [
         'vcs_svn_branch',
         'vcs_svn_tag',
     ])
     def test_delete_svn_patterns_raises_404_when_no_xhr(
             self, section, csrf_token, settings_util):
         setting = settings_util.create_rhodecode_ui(section, '/test_delete')
 
         self.app.post(
             route_path('admin_settings_vcs_svn_pattern_delete'),
             params={
                 'delete_svn_pattern': setting.ui_id,
                 'csrf_token': csrf_token},
             status=404)
 
     def test_extensions_hgevolve(self, form_defaults, csrf_token):
         form_defaults.update({
             'csrf_token': csrf_token,
             'extensions_evolve': 'True',
         })
         response = self.app.post(
             route_path('admin_settings_vcs_update'),
             params=form_defaults,
             status=302)
 
         response = response.follow()
         extensions_input = (
             '<input id="extensions_evolve" '
             'name="extensions_evolve" type="checkbox" '
             'value="True" checked="checked" />')
         response.mustcontain(extensions_input)
 
     def test_has_a_section_for_pull_request_settings(self):
         response = self.app.get(route_path('admin_settings_vcs'))
         response.mustcontain('Pull Request Settings')
 
     def test_has_an_input_for_invalidation_of_inline_comments(self):
         response = self.app.get(route_path('admin_settings_vcs'))
         assert_response = response.assert_response()
         assert_response.one_element_exists(
             '[name=rhodecode_use_outdated_comments]')
 
     @pytest.mark.parametrize('new_value', [True, False])
     def test_allows_to_change_invalidation_of_inline_comments(
             self, form_defaults, csrf_token, new_value):
         setting_key = 'use_outdated_comments'
         setting = SettingsModel().create_or_update_setting(
             setting_key, not new_value, 'bool')
         Session().add(setting)
         Session().commit()
 
         form_defaults.update({
             'csrf_token': csrf_token,
             'rhodecode_use_outdated_comments': str(new_value),
         })
         response = self.app.post(
             route_path('admin_settings_vcs_update'),
             params=form_defaults,
             status=302)
         response = response.follow()
         setting = SettingsModel().get_setting_by_name(setting_key)
         assert setting.app_settings_value is new_value
 
     @pytest.mark.parametrize('new_value', [True, False])
     def test_allows_to_change_hg_rebase_merge_strategy(
             self, form_defaults, csrf_token, new_value):
         setting_key = 'hg_use_rebase_for_merging'
 
         form_defaults.update({
             'csrf_token': csrf_token,
             'rhodecode_' + setting_key: str(new_value),
         })
 
         with mock.patch.dict(
                 rhodecode.CONFIG, {'labs_settings_active': 'true'}):
             self.app.post(
                 route_path('admin_settings_vcs_update'),
                 params=form_defaults,
                 status=302)
 
         setting = SettingsModel().get_setting_by_name(setting_key)
         assert setting.app_settings_value is new_value
 
     @pytest.fixture()
     def disable_sql_cache(self, request):
         # patch _do_orm_execute so it returns None similar like if we don't use a cached query
         patcher = mock.patch(
             'rhodecode.lib.caching_query.ORMCache._do_orm_execute', return_value=None)
         request.addfinalizer(patcher.stop)
         patcher.start()
 
     @pytest.fixture()
     def form_defaults(self):
         from rhodecode.apps.admin.views.settings import AdminSettingsView
         return AdminSettingsView._form_defaults()
 
     # TODO: johbo: What we really want is to checkpoint before a test run and
     # reset the session afterwards.
     @pytest.fixture(scope='class', autouse=True)
     def cleanup_settings(self, request, baseapp):
         ui_id = RhodeCodeUi.ui_id
         original_ids = [r.ui_id for r in RhodeCodeUi.query().with_entities(ui_id)]
 
         @request.addfinalizer
         def cleanup():
             RhodeCodeUi.query().filter(
                 ui_id.notin_(original_ids)).delete(False)
 
 
 @pytest.mark.usefixtures('autologin_user', 'app')
 class TestLabsSettings(object):
     def test_get_settings_page_disabled(self):
         with mock.patch.dict(
                 rhodecode.CONFIG, {'labs_settings_active': 'false'}):
 
             response = self.app.get(
                 route_path('admin_settings_labs'), status=302)
 
         assert response.location.endswith(route_path('admin_settings'))
 
     def test_get_settings_page_enabled(self):
         from rhodecode.apps.admin.views import settings
         lab_settings = [
             settings.LabSetting(
                 key='rhodecode_bool',
                 type='bool',
                 group='bool group',
                 label='bool label',
                 help='bool help'
             ),
             settings.LabSetting(
                 key='rhodecode_text',
                 type='unicode',
                 group='text group',
                 label='text label',
                 help='text help'
             ),
         ]
         with mock.patch.dict(rhodecode.CONFIG,
                              {'labs_settings_active': 'true'}):
             with mock.patch.object(settings, '_LAB_SETTINGS', lab_settings):
                 response = self.app.get(route_path('admin_settings_labs'))
 
         assert '<label>bool group:</label>' in response
         assert '<label for="rhodecode_bool">bool label</label>' in response
         assert '<p class="help-block">bool help</p>' in response
         assert 'name="rhodecode_bool" type="checkbox"' in response
 
         assert '<label>text group:</label>' in response
         assert '<label for="rhodecode_text">text label</label>' in response
         assert '<p class="help-block">text help</p>' in response
         assert 'name="rhodecode_text" size="60" type="text"' in response
 
 
 @pytest.mark.usefixtures('app')
 class TestOpenSourceLicenses(object):
 
     def test_records_are_displayed(self, autologin_user):
         sample_licenses = [
             {
                 "license": [
                     {
                         "fullName": "BSD 4-clause \"Original\" or \"Old\" License",
                         "shortName": "bsdOriginal",
                         "spdxId": "BSD-4-Clause",
                         "url": "http://spdx.org/licenses/BSD-4-Clause.html"
                     }
                 ],
                 "name": "python2.7-coverage-3.7.1"
             },
             {
                 "license": [
                     {
                         "fullName": "MIT License",
                         "shortName": "mit",
                         "spdxId": "MIT",
                         "url": "http://spdx.org/licenses/MIT.html"
                     }
                 ],
                 "name": "python2.7-bootstrapped-pip-9.0.1"
             },
         ]
         read_licenses_patch = mock.patch(
             'rhodecode.apps.admin.views.open_source_licenses.read_opensource_licenses',
             return_value=sample_licenses)
         with read_licenses_patch:
             response = self.app.get(
                 route_path('admin_settings_open_source'), status=200)
 
         assert_response = response.assert_response()
         assert_response.element_contains(
             '.panel-heading', 'Licenses of Third Party Packages')
         for license_data in sample_licenses:
             response.mustcontain(license_data["license"][0]["spdxId"])
             assert_response.element_contains('.panel-body', license_data["name"])
 
     def test_records_can_be_read(self, autologin_user):
         response = self.app.get(
             route_path('admin_settings_open_source'), status=200)
         assert_response = response.assert_response()
         assert_response.element_contains(
             '.panel-heading', 'Licenses of Third Party Packages')
 
     def test_forbidden_when_normal_user(self, autologin_regular_user):
         self.app.get(
             route_path('admin_settings_open_source'), status=404)
 
 
 @pytest.mark.usefixtures('app')
 class TestUserSessions(object):
 
     def test_forbidden_when_normal_user(self, autologin_regular_user):
         self.app.get(route_path('admin_settings_sessions'), status=404)
 
     def test_show_sessions_page(self, autologin_user):
         response = self.app.get(route_path('admin_settings_sessions'), status=200)
         response.mustcontain('file')
 
     def test_cleanup_old_sessions(self, autologin_user, csrf_token):
 
         post_data = {
             'csrf_token': csrf_token,
             'expire_days': '60'
         }
         response = self.app.post(
             route_path('admin_settings_sessions_cleanup'), params=post_data,
             status=302)
         assert_session_flash(response, 'Cleaned up old sessions')
 
 
 @pytest.mark.usefixtures('app')
 class TestAdminSystemInfo(object):
 
     def test_forbidden_when_normal_user(self, autologin_regular_user):
         self.app.get(route_path('admin_settings_system'), status=404)
 
     def test_system_info_page(self, autologin_user):
         response = self.app.get(route_path('admin_settings_system'))
         response.mustcontain('RhodeCode Community Edition, version {}'.format(
             rhodecode.__version__))
 
     def test_system_update_new_version(self, autologin_user):
         update_data = {
             'versions': [
                 {
-                    'version': '100.3.1415926535',
+                    'version': '100.0.0',
                     'general': 'The latest version we are ever going to ship'
                 },
                 {
                     'version': '0.0.0',
                     'general': 'The first version we ever shipped'
                 }
             ]
         }
         with mock.patch(UPDATE_DATA_QUALNAME, return_value=update_data):
             response = self.app.get(route_path('admin_settings_system_update'))
             response.mustcontain('A <b>new version</b> is available')
 
     def test_system_update_nothing_new(self, autologin_user):
         update_data = {
             'versions': [
                 {
-                    'version': '0.0.0',
+                    'version': '4.0.0',
                     'general': 'The first version we ever shipped'
                 }
             ]
         }
+        text = f"Your current version, {rhodecode.__version__}, is up-to-date as it is equal to or newer than the latest available version, 4.0.0."
         with mock.patch(UPDATE_DATA_QUALNAME, return_value=update_data):
             response = self.app.get(route_path('admin_settings_system_update'))
-            response.mustcontain(
-                'This instance is already running the <b>latest</b> stable version')
+            response.mustcontain(text)
 
     def test_system_update_bad_response(self, autologin_user):
         with mock.patch(UPDATE_DATA_QUALNAME, side_effect=ValueError('foo')):
             response = self.app.get(route_path('admin_settings_system_update'))
             response.mustcontain(
                 'Bad data sent from update server')
 
 
 @pytest.mark.usefixtures("app")
 class TestAdminSettingsIssueTracker(object):
     RC_PREFIX = 'rhodecode_'
     SHORT_PATTERN_KEY = 'issuetracker_pat_'
     PATTERN_KEY = RC_PREFIX + SHORT_PATTERN_KEY
     DESC_KEY = RC_PREFIX + 'issuetracker_desc_'
 
     def test_issuetracker_index(self, autologin_user):
         response = self.app.get(route_path('admin_settings_issuetracker'))
         assert response.status_code == 200
 
     def test_add_empty_issuetracker_pattern(
             self, request, autologin_user, csrf_token):
         post_url = route_path('admin_settings_issuetracker_update')
         post_data = {
             'csrf_token': csrf_token
         }
         self.app.post(post_url, post_data, status=302)
 
     def test_add_issuetracker_pattern(
             self, request, autologin_user, csrf_token):
         pattern = 'issuetracker_pat'
         another_pattern = pattern+'1'
         post_url = route_path('admin_settings_issuetracker_update')
         post_data = {
             'new_pattern_pattern_0': pattern,
             'new_pattern_url_0': 'http://url',
             'new_pattern_prefix_0': 'prefix',
             'new_pattern_description_0': 'description',
             'new_pattern_pattern_1': another_pattern,
             'new_pattern_url_1': 'https://url1',
             'new_pattern_prefix_1': 'prefix1',
             'new_pattern_description_1': 'description1',
             'csrf_token': csrf_token
         }
         self.app.post(post_url, post_data, status=302)
         settings = SettingsModel().get_all_settings()
         self.uid = md5_safe(pattern)
         assert settings[self.PATTERN_KEY+self.uid] == pattern
         self.another_uid = md5_safe(another_pattern)
         assert settings[self.PATTERN_KEY+self.another_uid] == another_pattern
 
         @request.addfinalizer
         def cleanup():
             defaults = SettingsModel().get_all_settings()
 
             entries = [name for name in defaults if (
                 (self.uid in name) or (self.another_uid in name))]
             start = len(self.RC_PREFIX)
             for del_key in entries:
                 # TODO: anderson: get_by_name needs name without prefix
                 entry = SettingsModel().get_setting_by_name(del_key[start:])
                 Session().delete(entry)
 
             Session().commit()
 
     def test_edit_issuetracker_pattern(
             self, autologin_user, backend, csrf_token, request):
 
         old_pattern = 'issuetracker_pat1'
         old_uid = md5_safe(old_pattern)
 
         post_url = route_path('admin_settings_issuetracker_update')
         post_data = {
             'new_pattern_pattern_0': old_pattern,
             'new_pattern_url_0': 'http://url',
             'new_pattern_prefix_0': 'prefix',
             'new_pattern_description_0': 'description',
 
             'csrf_token': csrf_token
         }
         self.app.post(post_url, post_data, status=302)
 
         new_pattern = 'issuetracker_pat1_edited'
         self.new_uid = md5_safe(new_pattern)
 
         post_url = route_path('admin_settings_issuetracker_update')
         post_data = {
             'new_pattern_pattern_{}'.format(old_uid): new_pattern,
             'new_pattern_url_{}'.format(old_uid): 'https://url_edited',
             'new_pattern_prefix_{}'.format(old_uid): 'prefix_edited',
             'new_pattern_description_{}'.format(old_uid): 'description_edited',
             'uid': old_uid,
             'csrf_token': csrf_token
         }
         self.app.post(post_url, post_data, status=302)
 
         settings = SettingsModel().get_all_settings()
         assert settings[self.PATTERN_KEY+self.new_uid] == new_pattern
         assert settings[self.DESC_KEY + self.new_uid] == 'description_edited'
         assert self.PATTERN_KEY+old_uid not in settings
 
         @request.addfinalizer
         def cleanup():
             IssueTrackerSettingsModel().delete_entries(old_uid)
             IssueTrackerSettingsModel().delete_entries(self.new_uid)
 
     def test_replace_issuetracker_pattern_description(
             self, autologin_user, csrf_token, request, settings_util):
         prefix = 'issuetracker'
         pattern = 'issuetracker_pat'
         self.uid = md5_safe(pattern)
         pattern_key = '_'.join([prefix, 'pat', self.uid])
         rc_pattern_key = '_'.join(['rhodecode', pattern_key])
         desc_key = '_'.join([prefix, 'desc', self.uid])
         rc_desc_key = '_'.join(['rhodecode', desc_key])
         new_description = 'new_description'
 
         settings_util.create_rhodecode_setting(
             pattern_key, pattern, 'unicode', cleanup=False)
         settings_util.create_rhodecode_setting(
             desc_key, 'old description', 'unicode', cleanup=False)
 
         post_url = route_path('admin_settings_issuetracker_update')
         post_data = {
             'new_pattern_pattern_0': pattern,
             'new_pattern_url_0': 'https://url',
             'new_pattern_prefix_0': 'prefix',
             'new_pattern_description_0': new_description,
             'uid': self.uid,
             'csrf_token': csrf_token
         }
         self.app.post(post_url, post_data, status=302)
         settings = SettingsModel().get_all_settings()
         assert settings[rc_pattern_key] == pattern
         assert settings[rc_desc_key] == new_description
 
         @request.addfinalizer
         def cleanup():
             IssueTrackerSettingsModel().delete_entries(self.uid)
 
     def test_delete_issuetracker_pattern(
             self, autologin_user, backend, csrf_token, settings_util, xhr_header):
 
         old_pattern = 'issuetracker_pat_deleted'
         old_uid = md5_safe(old_pattern)
 
         post_url = route_path('admin_settings_issuetracker_update')
         post_data = {
             'new_pattern_pattern_0': old_pattern,
             'new_pattern_url_0': 'http://url',
             'new_pattern_prefix_0': 'prefix',
             'new_pattern_description_0': 'description',
 
             'csrf_token': csrf_token
         }
         self.app.post(post_url, post_data, status=302)
 
         post_url = route_path('admin_settings_issuetracker_delete')
         post_data = {
             'uid': old_uid,
             'csrf_token': csrf_token
         }
         self.app.post(post_url, post_data, extra_environ=xhr_header, status=200)
         settings = SettingsModel().get_all_settings()
         assert self.PATTERN_KEY+old_uid not in settings
         assert self.DESC_KEY + old_uid not in settings

rhodecode/apps/admin/views/permissions.py

@@ -1,477 +1,477 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import re
 import logging
 import formencode
 import formencode.htmlfill
 import datetime
 from pyramid.interfaces import IRoutesMapper
 
 from pyramid.httpexceptions import HTTPFound
 from pyramid.renderers import render
 from pyramid.response import Response
 
 from rhodecode.apps._base import BaseAppView, DataGridAppView
-from rhodecode.apps.ssh_support import SshKeyFileChangeEvent
+from rhodecode.apps.ssh_support.events import SshKeyFileChangeEvent
 from rhodecode import events
 
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import (
     LoginRequired, HasPermissionAllDecorator, CSRFRequired)
 from rhodecode.lib.utils2 import aslist, safe_str
 from rhodecode.model.db import (
     or_, coalesce, User, UserIpMap, UserSshKeys)
 from rhodecode.model.forms import (
     ApplicationPermissionsForm, ObjectPermissionsForm, UserPermissionsForm)
 from rhodecode.model.meta import Session
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.settings import SettingsModel
 
 
 log = logging.getLogger(__name__)
 
 
 class AdminPermissionsView(BaseAppView, DataGridAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         PermissionModel().set_global_permission_choices(
             c, gettext_translator=self.request.translate)
         return c
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def permissions_application(self):
         c = self.load_default_context()
         c.active = 'application'
 
         c.user = User.get_default_user(refresh=True)
 
         app_settings = c.rc_config
 
         defaults = {
             'anonymous': c.user.active,
             'default_register_message': app_settings.get(
                 'rhodecode_register_message')
         }
         defaults.update(c.user.get_default_perms())
 
         data = render('rhodecode:templates/admin/permissions/permissions.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def permissions_application_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'application'
 
         _form = ApplicationPermissionsForm(
             self.request.translate,
             [x[0] for x in c.register_choices],
             [x[0] for x in c.password_reset_choices],
             [x[0] for x in c.extern_activate_choices])()
 
         try:
             form_result = _form.to_python(dict(self.request.POST))
             form_result.update({'perm_user_name': User.DEFAULT_USER})
             PermissionModel().update_application_permissions(form_result)
 
             settings = [
                 ('register_message', 'default_register_message'),
             ]
             for setting, form_key in settings:
                 sett = SettingsModel().create_or_update_setting(
                     setting, form_result[form_key])
                 Session().add(sett)
 
             Session().commit()
             h.flash(_('Application permissions updated successfully'),
                     category='success')
 
         except formencode.Invalid as errors:
             defaults = errors.value
 
             data = render(
                 'rhodecode:templates/admin/permissions/permissions.mako',
                 self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=defaults,
                 errors=errors.unpack_errors() or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
 
         except Exception:
             log.exception("Exception during update of permissions")
             h.flash(_('Error occurred during update of permissions'),
                     category='error')
 
         affected_user_ids = [User.get_default_user_id()]
         PermissionModel().trigger_permission_flush(affected_user_ids)
 
         raise HTTPFound(h.route_path('admin_permissions_application'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def permissions_objects(self):
         c = self.load_default_context()
         c.active = 'objects'
 
         c.user = User.get_default_user(refresh=True)
         defaults = {}
         defaults.update(c.user.get_default_perms())
 
         data = render(
             'rhodecode:templates/admin/permissions/permissions.mako',
             self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def permissions_objects_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'objects'
 
         _form = ObjectPermissionsForm(
             self.request.translate,
             [x[0] for x in c.repo_perms_choices],
             [x[0] for x in c.group_perms_choices],
             [x[0] for x in c.user_group_perms_choices],
         )()
 
         try:
             form_result = _form.to_python(dict(self.request.POST))
             form_result.update({'perm_user_name': User.DEFAULT_USER})
             PermissionModel().update_object_permissions(form_result)
 
             Session().commit()
             h.flash(_('Object permissions updated successfully'),
                     category='success')
 
         except formencode.Invalid as errors:
             defaults = errors.value
 
             data = render(
                 'rhodecode:templates/admin/permissions/permissions.mako',
                 self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=defaults,
                 errors=errors.unpack_errors() or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
         except Exception:
             log.exception("Exception during update of permissions")
             h.flash(_('Error occurred during update of permissions'),
                     category='error')
 
         affected_user_ids = [User.get_default_user_id()]
         PermissionModel().trigger_permission_flush(affected_user_ids)
 
         raise HTTPFound(h.route_path('admin_permissions_object'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def permissions_branch(self):
         c = self.load_default_context()
         c.active = 'branch'
 
         c.user = User.get_default_user(refresh=True)
         defaults = {}
         defaults.update(c.user.get_default_perms())
 
         data = render(
             'rhodecode:templates/admin/permissions/permissions.mako',
             self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def permissions_global(self):
         c = self.load_default_context()
         c.active = 'global'
 
         c.user = User.get_default_user(refresh=True)
         defaults = {}
         defaults.update(c.user.get_default_perms())
 
         data = render(
             'rhodecode:templates/admin/permissions/permissions.mako',
             self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def permissions_global_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'global'
 
         _form = UserPermissionsForm(
             self.request.translate,
             [x[0] for x in c.repo_create_choices],
             [x[0] for x in c.repo_create_on_write_choices],
             [x[0] for x in c.repo_group_create_choices],
             [x[0] for x in c.user_group_create_choices],
             [x[0] for x in c.fork_choices],
             [x[0] for x in c.inherit_default_permission_choices])()
 
         try:
             form_result = _form.to_python(dict(self.request.POST))
             form_result.update({'perm_user_name': User.DEFAULT_USER})
             PermissionModel().update_user_permissions(form_result)
 
             Session().commit()
             h.flash(_('Global permissions updated successfully'),
                     category='success')
 
         except formencode.Invalid as errors:
             defaults = errors.value
 
             data = render(
                 'rhodecode:templates/admin/permissions/permissions.mako',
                 self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=defaults,
                 errors=errors.unpack_errors() or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
         except Exception:
             log.exception("Exception during update of permissions")
             h.flash(_('Error occurred during update of permissions'),
                     category='error')
 
         affected_user_ids = [User.get_default_user_id()]
         PermissionModel().trigger_permission_flush(affected_user_ids)
 
         raise HTTPFound(h.route_path('admin_permissions_global'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def permissions_ips(self):
         c = self.load_default_context()
         c.active = 'ips'
 
         c.user = User.get_default_user(refresh=True)
         c.user_ip_map = (
             UserIpMap.query().filter(UserIpMap.user == c.user).all())
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def permissions_overview(self):
         c = self.load_default_context()
         c.active = 'perms'
 
         c.user = User.get_default_user(refresh=True)
         c.perm_user = c.user.AuthUser()
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def auth_token_access(self):
         from rhodecode import CONFIG
 
         c = self.load_default_context()
         c.active = 'auth_token_access'
 
         c.user = User.get_default_user(refresh=True)
         c.perm_user = c.user.AuthUser()
 
         mapper = self.request.registry.queryUtility(IRoutesMapper)
         c.view_data = []
 
         _argument_prog = re.compile(r'\{(.*?)\}|:\((.*)\)')
         introspector = self.request.registry.introspector
 
         view_intr = {}
         for view_data in introspector.get_category('views'):
             intr = view_data['introspectable']
 
             if 'route_name' in intr and intr['attr']:
                 view_intr[intr['route_name']] = '{}:{}'.format(
                     str(intr['derived_callable'].__name__), intr['attr']
                 )
 
         c.whitelist_key = 'api_access_controllers_whitelist'
         c.whitelist_file = CONFIG.get('__file__')
         whitelist_views = aslist(
             CONFIG.get(c.whitelist_key), sep=',')
 
         for route_info in mapper.get_routes():
             if not route_info.name.startswith('__'):
                 routepath = route_info.pattern
 
                 def replace(matchobj):
                     if matchobj.group(1):
                         return "{%s}" % matchobj.group(1).split(':')[0]
                     else:
                         return "{%s}" % matchobj.group(2)
 
                 routepath = _argument_prog.sub(replace, routepath)
 
                 if not routepath.startswith('/'):
                     routepath = '/' + routepath
 
                 view_fqn = view_intr.get(route_info.name, 'NOT AVAILABLE')
                 active = view_fqn in whitelist_views
                 c.view_data.append((route_info.name, view_fqn, routepath, active))
 
         c.whitelist_views = whitelist_views
         return self._get_template_context(c)
 
     def ssh_enabled(self):
         return self.request.registry.settings.get(
             'ssh.generate_authorized_keyfile')
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def ssh_keys(self):
         c = self.load_default_context()
         c.active = 'ssh_keys'
         c.ssh_enabled = self.ssh_enabled()
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def ssh_keys_data(self):
         _ = self.request.translate
         self.load_default_context()
         column_map = {
             'fingerprint': 'ssh_key_fingerprint',
             'username': User.username
         }
         draw, start, limit = self._extract_chunk(self.request)
         search_q, order_by, order_dir = self._extract_ordering(
             self.request, column_map=column_map)
 
         ssh_keys_data_total_count = UserSshKeys.query()\
             .count()
 
         # json generate
         base_q = UserSshKeys.query().join(UserSshKeys.user)
 
         if search_q:
             like_expression = f'%{safe_str(search_q)}%'
             base_q = base_q.filter(or_(
                 User.username.ilike(like_expression),
                 UserSshKeys.ssh_key_fingerprint.ilike(like_expression),
             ))
 
         users_data_total_filtered_count = base_q.count()
 
         sort_col = self._get_order_col(order_by, UserSshKeys)
         if sort_col:
             if order_dir == 'asc':
                 # handle null values properly to order by NULL last
                 if order_by in ['created_on']:
                     sort_col = coalesce(sort_col, datetime.date.max)
                 sort_col = sort_col.asc()
             else:
                 # handle null values properly to order by NULL last
                 if order_by in ['created_on']:
                     sort_col = coalesce(sort_col, datetime.date.min)
                 sort_col = sort_col.desc()
 
         base_q = base_q.order_by(sort_col)
         base_q = base_q.offset(start).limit(limit)
 
         ssh_keys = base_q.all()
 
         ssh_keys_data = []
         for ssh_key in ssh_keys:
             ssh_keys_data.append({
                 "username": h.gravatar_with_user(self.request, ssh_key.user.username),
                 "fingerprint": ssh_key.ssh_key_fingerprint,
                 "description": ssh_key.description,
                 "created_on": h.format_date(ssh_key.created_on),
                 "accessed_on": h.format_date(ssh_key.accessed_on),
                 "action": h.link_to(
                     _('Edit'), h.route_path('edit_user_ssh_keys',
                                             user_id=ssh_key.user.user_id))
             })
 
         data = ({
             'draw': draw,
             'data': ssh_keys_data,
             'recordsTotal': ssh_keys_data_total_count,
             'recordsFiltered': users_data_total_filtered_count,
         })
 
         return data
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def ssh_keys_update(self):
         _ = self.request.translate
         self.load_default_context()
 
         ssh_enabled = self.ssh_enabled()
         key_file = self.request.registry.settings.get(
             'ssh.authorized_keys_file_path')
         if ssh_enabled:
             events.trigger(SshKeyFileChangeEvent(), self.request.registry)
             h.flash(_('Updated SSH keys file: {}').format(key_file),
                     category='success')
         else:
             h.flash(_('SSH key support is disabled in .ini file'),
                     category='warning')
 
         raise HTTPFound(h.route_path('admin_permissions_ssh_keys'))

rhodecode/apps/admin/views/settings.py

@@ -1,714 +1,708 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 import logging
 import collections
 
 import datetime
 import formencode
 import formencode.htmlfill
 
 import rhodecode
 
 from pyramid.httpexceptions import HTTPFound, HTTPNotFound
 from pyramid.renderers import render
 from pyramid.response import Response
 
 from rhodecode.apps._base import BaseAppView
 from rhodecode.apps._base.navigation import navigation_list
-from rhodecode.apps.svn_support.config_keys import generate_config
+from rhodecode.apps.svn_support import config_keys
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import (
     LoginRequired, HasPermissionAllDecorator, CSRFRequired)
 from rhodecode.lib.celerylib import tasks, run_task
 from rhodecode.lib.str_utils import safe_str
-from rhodecode.lib.utils import repo2db_mapper
+from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_repo_store_path
 from rhodecode.lib.utils2 import str2bool, AttributeDict
 from rhodecode.lib.index import searcher_from_config
 
 from rhodecode.model.db import RhodeCodeUi, Repository
 from rhodecode.model.forms import (ApplicationSettingsForm,
     ApplicationUiSettingsForm, ApplicationVisualisationForm,
     LabsSettingsForm, IssueTrackerPatternsForm)
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.repo_group import RepoGroupModel
 
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.notification import EmailNotificationModel
 from rhodecode.model.meta import Session
 from rhodecode.model.settings import (
     IssueTrackerSettingsModel, VcsSettingsModel, SettingNotFound,
     SettingsModel)
 
 
 log = logging.getLogger(__name__)
 
 
 class AdminSettingsView(BaseAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         c.labs_active = str2bool(
             rhodecode.CONFIG.get('labs_settings_active', 'true'))
         c.navlist = navigation_list(self.request)
         return c
 
     @classmethod
     def _get_ui_settings(cls):
         ret = RhodeCodeUi.query().all()
 
         if not ret:
             raise Exception('Could not get application ui settings !')
         settings = {}
         for each in ret:
             k = each.ui_key
             v = each.ui_value
             if k == '/':
                 k = 'root_path'
 
             if k in ['push_ssl', 'publish', 'enabled']:
                 v = str2bool(v)
 
             if k.find('.') != -1:
                 k = k.replace('.', '_')
 
             if each.ui_section in ['hooks', 'extensions']:
                 v = each.ui_active
 
             settings[each.ui_section + '_' + k] = v
         return settings
 
     @classmethod
     def _form_defaults(cls):
         defaults = SettingsModel().get_all_settings()
         defaults.update(cls._get_ui_settings())
 
         defaults.update({
             'new_svn_branch': '',
             'new_svn_tag': '',
         })
         return defaults
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def settings_vcs(self):
         c = self.load_default_context()
         c.active = 'vcs'
         model = VcsSettingsModel()
         c.svn_branch_patterns = model.get_global_svn_branch_patterns()
         c.svn_tag_patterns = model.get_global_svn_tag_patterns()
-
-        settings = self.request.registry.settings
-        c.svn_proxy_generate_config = settings[generate_config]
-
+        c.svn_generate_config = rhodecode.ConfigGet().get_bool(config_keys.generate_config)
+        c.svn_config_path = rhodecode.ConfigGet().get_str(config_keys.config_file_path)
         defaults = self._form_defaults()
 
         model.create_largeobjects_dirs_if_needed(defaults['paths_root_path'])
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def settings_vcs_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'vcs'
 
         model = VcsSettingsModel()
         c.svn_branch_patterns = model.get_global_svn_branch_patterns()
         c.svn_tag_patterns = model.get_global_svn_tag_patterns()
 
-        settings = self.request.registry.settings
-        c.svn_proxy_generate_config = settings[generate_config]
-
+        c.svn_generate_config = rhodecode.ConfigGet().get_bool(config_keys.generate_config)
+        c.svn_config_path = rhodecode.ConfigGet().get_str(config_keys.config_file_path)
         application_form = ApplicationUiSettingsForm(self.request.translate)()
 
         try:
             form_result = application_form.to_python(dict(self.request.POST))
         except formencode.Invalid as errors:
             h.flash(
                 _("Some form inputs contain invalid data."),
                 category='error')
             data = render('rhodecode:templates/admin/settings/settings.mako',
                           self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.unpack_errors() or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
 
         try:
-            if c.visual.allow_repo_location_change:
-                model.update_global_path_setting(form_result['paths_root_path'])
-
             model.update_global_ssl_setting(form_result['web_push_ssl'])
             model.update_global_hook_settings(form_result)
 
             model.create_or_update_global_svn_settings(form_result)
             model.create_or_update_global_hg_settings(form_result)
             model.create_or_update_global_git_settings(form_result)
             model.create_or_update_global_pr_settings(form_result)
         except Exception:
             log.exception("Exception while updating settings")
             h.flash(_('Error occurred during updating '
                       'application settings'), category='error')
         else:
             Session().commit()
             h.flash(_('Updated VCS settings'), category='success')
             raise HTTPFound(h.route_path('admin_settings_vcs'))
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def settings_vcs_delete_svn_pattern(self):
         delete_pattern_id = self.request.POST.get('delete_svn_pattern')
         model = VcsSettingsModel()
         try:
             model.delete_global_svn_pattern(delete_pattern_id)
         except SettingNotFound:
             log.exception(
                 'Failed to delete svn_pattern with id %s', delete_pattern_id)
             raise HTTPNotFound()
 
         Session().commit()
         return True
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def settings_mapping(self):
         c = self.load_default_context()
         c.active = 'mapping'
-        c.storage_path = VcsSettingsModel().get_repos_location()
+        c.storage_path = get_rhodecode_repo_store_path()
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def settings_mapping_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'mapping'
         rm_obsolete = self.request.POST.get('destroy', False)
         invalidate_cache = self.request.POST.get('invalidate', False)
         log.debug('rescanning repo location with destroy obsolete=%s', rm_obsolete)
 
         if invalidate_cache:
             log.debug('invalidating all repositories cache')
             for repo in Repository.get_all():
                 ScmModel().mark_for_invalidation(repo.repo_name, delete=True)
 
         filesystem_repos = ScmModel().repo_scan()
         added, removed = repo2db_mapper(filesystem_repos, rm_obsolete, force_hooks_rebuild=True)
         PermissionModel().trigger_permission_flush()
 
         def _repr(rm_repo):
             return ', '.join(map(safe_str, rm_repo)) or '-'
 
         h.flash(_('Repositories successfully '
                   'rescanned added: %s ; removed: %s') %
                 (_repr(added), _repr(removed)),
                 category='success')
         raise HTTPFound(h.route_path('admin_settings_mapping'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def settings_global(self):
         c = self.load_default_context()
         c.active = 'global'
         c.personal_repo_group_default_pattern = RepoGroupModel()\
             .get_personal_group_name_pattern()
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def settings_global_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'global'
         c.personal_repo_group_default_pattern = RepoGroupModel()\
             .get_personal_group_name_pattern()
         application_form = ApplicationSettingsForm(self.request.translate)()
         try:
             form_result = application_form.to_python(dict(self.request.POST))
         except formencode.Invalid as errors:
             h.flash(
                 _("Some form inputs contain invalid data."),
                 category='error')
             data = render('rhodecode:templates/admin/settings/settings.mako',
                           self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.unpack_errors() or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
 
         settings = [
             ('title', 'rhodecode_title', 'unicode'),
             ('realm', 'rhodecode_realm', 'unicode'),
             ('pre_code', 'rhodecode_pre_code', 'unicode'),
             ('post_code', 'rhodecode_post_code', 'unicode'),
             ('captcha_public_key', 'rhodecode_captcha_public_key', 'unicode'),
             ('captcha_private_key', 'rhodecode_captcha_private_key', 'unicode'),
             ('create_personal_repo_group', 'rhodecode_create_personal_repo_group', 'bool'),
             ('personal_repo_group_pattern', 'rhodecode_personal_repo_group_pattern', 'unicode'),
         ]
 
         try:
             for setting, form_key, type_ in settings:
                 sett = SettingsModel().create_or_update_setting(
                     setting, form_result[form_key], type_)
                 Session().add(sett)
 
             Session().commit()
             SettingsModel().invalidate_settings_cache()
             h.flash(_('Updated application settings'), category='success')
         except Exception:
             log.exception("Exception while updating application settings")
             h.flash(
                 _('Error occurred during updating application settings'),
                 category='error')
 
         raise HTTPFound(h.route_path('admin_settings_global'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def settings_visual(self):
         c = self.load_default_context()
         c.active = 'visual'
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def settings_visual_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'visual'
         application_form = ApplicationVisualisationForm(self.request.translate)()
         try:
             form_result = application_form.to_python(dict(self.request.POST))
         except formencode.Invalid as errors:
             h.flash(
                 _("Some form inputs contain invalid data."),
                 category='error')
             data = render('rhodecode:templates/admin/settings/settings.mako',
                           self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.unpack_errors() or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
 
         try:
             settings = [
                 ('show_public_icon', 'rhodecode_show_public_icon', 'bool'),
                 ('show_private_icon', 'rhodecode_show_private_icon', 'bool'),
                 ('stylify_metatags', 'rhodecode_stylify_metatags', 'bool'),
                 ('repository_fields', 'rhodecode_repository_fields', 'bool'),
                 ('dashboard_items', 'rhodecode_dashboard_items', 'int'),
                 ('admin_grid_items', 'rhodecode_admin_grid_items', 'int'),
                 ('show_version', 'rhodecode_show_version', 'bool'),
                 ('use_gravatar', 'rhodecode_use_gravatar', 'bool'),
                 ('markup_renderer', 'rhodecode_markup_renderer', 'unicode'),
                 ('gravatar_url', 'rhodecode_gravatar_url', 'unicode'),
                 ('clone_uri_tmpl', 'rhodecode_clone_uri_tmpl', 'unicode'),
                 ('clone_uri_id_tmpl', 'rhodecode_clone_uri_id_tmpl', 'unicode'),
                 ('clone_uri_ssh_tmpl', 'rhodecode_clone_uri_ssh_tmpl', 'unicode'),
                 ('support_url', 'rhodecode_support_url', 'unicode'),
                 ('show_revision_number', 'rhodecode_show_revision_number', 'bool'),
                 ('show_sha_length', 'rhodecode_show_sha_length', 'int'),
             ]
             for setting, form_key, type_ in settings:
                 sett = SettingsModel().create_or_update_setting(
                     setting, form_result[form_key], type_)
                 Session().add(sett)
 
             Session().commit()
             SettingsModel().invalidate_settings_cache()
             h.flash(_('Updated visualisation settings'), category='success')
         except Exception:
             log.exception("Exception updating visualization settings")
             h.flash(_('Error occurred during updating '
                       'visualisation settings'),
                     category='error')
 
         raise HTTPFound(h.route_path('admin_settings_visual'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def settings_issuetracker(self):
         c = self.load_default_context()
         c.active = 'issuetracker'
         defaults = c.rc_config
 
         entry_key = 'rhodecode_issuetracker_pat_'
 
         c.issuetracker_entries = {}
         for k, v in defaults.items():
             if k.startswith(entry_key):
                 uid = k[len(entry_key):]
                 c.issuetracker_entries[uid] = None
 
         for uid in c.issuetracker_entries:
             c.issuetracker_entries[uid] = AttributeDict({
                 'pat': defaults.get('rhodecode_issuetracker_pat_' + uid),
                 'url': defaults.get('rhodecode_issuetracker_url_' + uid),
                 'pref': defaults.get('rhodecode_issuetracker_pref_' + uid),
                 'desc': defaults.get('rhodecode_issuetracker_desc_' + uid),
             })
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def settings_issuetracker_test(self):
         error_container = []
 
         urlified_commit = h.urlify_commit_message(
             self.request.POST.get('test_text', ''),
             'repo_group/test_repo1', error_container=error_container)
         if error_container:
             def converter(inp):
                 return h.html_escape(inp)
 
             return 'ERRORS: ' + '\n'.join(map(converter, error_container))
 
         return urlified_commit
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def settings_issuetracker_update(self):
         _ = self.request.translate
         self.load_default_context()
         settings_model = IssueTrackerSettingsModel()
 
         try:
             form = IssueTrackerPatternsForm(self.request.translate)()
             data = form.to_python(self.request.POST)
         except formencode.Invalid as errors:
             log.exception('Failed to add new pattern')
             error = errors
             h.flash(_(f'Invalid issue tracker pattern: {error}'),
                     category='error')
             raise HTTPFound(h.route_path('admin_settings_issuetracker'))
 
         if data:
             for uid in data.get('delete_patterns', []):
                 settings_model.delete_entries(uid)
 
             for pattern in data.get('patterns', []):
                 for setting, value, type_ in pattern:
                     sett = settings_model.create_or_update_setting(
                         setting, value, type_)
                     Session().add(sett)
 
                 Session().commit()
 
             SettingsModel().invalidate_settings_cache()
             h.flash(_('Updated issue tracker entries'), category='success')
         raise HTTPFound(h.route_path('admin_settings_issuetracker'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def settings_issuetracker_delete(self):
         _ = self.request.translate
         self.load_default_context()
         uid = self.request.POST.get('uid')
         try:
             IssueTrackerSettingsModel().delete_entries(uid)
         except Exception:
             log.exception('Failed to delete issue tracker setting %s', uid)
             raise HTTPNotFound()
 
         SettingsModel().invalidate_settings_cache()
         h.flash(_('Removed issue tracker entry.'), category='success')
 
         return {'deleted': uid}
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def settings_email(self):
         c = self.load_default_context()
         c.active = 'email'
         c.rhodecode_ini = rhodecode.CONFIG
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def settings_email_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'email'
 
         test_email = self.request.POST.get('test_email')
 
         if not test_email:
             h.flash(_('Please enter email address'), category='error')
             raise HTTPFound(h.route_path('admin_settings_email'))
 
         email_kwargs = {
             'date': datetime.datetime.now(),
             'user': self._rhodecode_db_user
         }
 
         (subject, email_body, email_body_plaintext) = EmailNotificationModel().render_email(
             EmailNotificationModel.TYPE_EMAIL_TEST, **email_kwargs)
 
         recipients = [test_email] if test_email else None
 
         run_task(tasks.send_email, recipients, subject,
                  email_body_plaintext, email_body)
 
         h.flash(_('Send email task created'), category='success')
         raise HTTPFound(h.route_path('admin_settings_email'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def settings_hooks(self):
         c = self.load_default_context()
         c.active = 'hooks'
 
         model = SettingsModel()
         c.hooks = model.get_builtin_hooks()
         c.custom_hooks = model.get_custom_hooks()
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def settings_hooks_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'hooks'
         if c.visual.allow_custom_hooks_settings:
             ui_key = self.request.POST.get('new_hook_ui_key')
             ui_value = self.request.POST.get('new_hook_ui_value')
 
             hook_id = self.request.POST.get('hook_id')
             new_hook = False
 
             model = SettingsModel()
             try:
                 if ui_value and ui_key:
                     model.create_or_update_hook(ui_key, ui_value)
                     h.flash(_('Added new hook'), category='success')
                     new_hook = True
                 elif hook_id:
                     RhodeCodeUi.delete(hook_id)
                     Session().commit()
 
                 # check for edits
                 update = False
                 _d = self.request.POST.dict_of_lists()
                 for k, v in zip(_d.get('hook_ui_key', []),
                                 _d.get('hook_ui_value_new', [])):
                     model.create_or_update_hook(k, v)
                     update = True
 
                 if update and not new_hook:
                     h.flash(_('Updated hooks'), category='success')
                 Session().commit()
             except Exception:
                 log.exception("Exception during hook creation")
                 h.flash(_('Error occurred during hook creation'),
                         category='error')
 
         raise HTTPFound(h.route_path('admin_settings_hooks'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def settings_search(self):
         c = self.load_default_context()
         c.active = 'search'
 
         c.searcher = searcher_from_config(self.request.registry.settings)
         c.statistics = c.searcher.statistics(self.request.translate)
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def settings_labs(self):
         c = self.load_default_context()
         if not c.labs_active:
             raise HTTPFound(h.route_path('admin_settings'))
 
         c.active = 'labs'
         c.lab_settings = _LAB_SETTINGS
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def settings_labs_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'labs'
 
         application_form = LabsSettingsForm(self.request.translate)()
         try:
             form_result = application_form.to_python(dict(self.request.POST))
         except formencode.Invalid as errors:
             h.flash(
                 _("Some form inputs contain invalid data."),
                 category='error')
             data = render('rhodecode:templates/admin/settings/settings.mako',
                           self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.unpack_errors() or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
 
         try:
             session = Session()
             for setting in _LAB_SETTINGS:
                 setting_name = setting.key[len('rhodecode_'):]
                 sett = SettingsModel().create_or_update_setting(
                     setting_name, form_result[setting.key], setting.type)
                 session.add(sett)
 
         except Exception:
             log.exception('Exception while updating lab settings')
             h.flash(_('Error occurred during updating labs settings'),
                     category='error')
         else:
             Session().commit()
             SettingsModel().invalidate_settings_cache()
             h.flash(_('Updated Labs settings'), category='success')
             raise HTTPFound(h.route_path('admin_settings_labs'))
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
 
 # :param key: name of the setting including the 'rhodecode_' prefix
 # :param type: the RhodeCodeSetting type to use.
 # :param group: the i18ned group in which we should dispaly this setting
 # :param label: the i18ned label we should display for this setting
 # :param help: the i18ned help we should dispaly for this setting
 LabSetting = collections.namedtuple(
     'LabSetting', ('key', 'type', 'group', 'label', 'help'))
 
 
 # This list has to be kept in sync with the form
 # rhodecode.model.forms.LabsSettingsForm.
 _LAB_SETTINGS = [
 
 ]

rhodecode/apps/admin/views/system_info.py

@@ -1,237 +1,243 b''
 
 
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import urllib.request
 import urllib.error
 import urllib.parse
 import os
 
 import rhodecode
 from rhodecode.apps._base import BaseAppView
 from rhodecode.apps._base.navigation import navigation_list
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import (LoginRequired, HasPermissionAllDecorator)
 from rhodecode.lib.utils2 import str2bool
 from rhodecode.lib import system_info
 from rhodecode.model.update import UpdateModel
 
 log = logging.getLogger(__name__)
 
 
 class AdminSystemInfoSettingsView(BaseAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         return c
 
     def get_env_data(self):
         black_list = [
             'NIX_LDFLAGS',
             'NIX_CFLAGS_COMPILE',
             'propagatedBuildInputs',
             'propagatedNativeBuildInputs',
             'postInstall',
             'buildInputs',
             'buildPhase',
             'preShellHook',
             'preShellHook',
             'preCheck',
             'preBuild',
             'postShellHook',
             'postFixup',
             'postCheck',
             'nativeBuildInputs',
             'installPhase',
             'installCheckPhase',
             'checkPhase',
             'configurePhase',
             'shellHook'
         ]
         secret_list = [
             'RHODECODE_USER_PASS'
         ]
 
         for k, v in sorted(os.environ.items()):
             if k in black_list:
                 continue
             if k in secret_list:
                 v = '*****'
             yield k, v
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def settings_system_info(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         c.active = 'system'
         c.navlist = navigation_list(self.request)
 
         # TODO(marcink), figure out how to allow only selected users to do this
         c.allowed_to_snapshot = self._rhodecode_user.admin
 
         snapshot = str2bool(self.request.params.get('snapshot'))
 
         c.rhodecode_update_url = UpdateModel().get_update_url()
         c.env_data = self.get_env_data()
         server_info = system_info.get_system_info(self.request.environ)
 
         for key, val in server_info.items():
             setattr(c, key, val)
 
         def val(name, subkey='human_value'):
             return server_info[name][subkey]
 
         def state(name):
             return server_info[name]['state']
 
         def val2(name):
             val = server_info[name]['human_value']
             state = server_info[name]['state']
             return val, state
 
         update_info_msg = _('Note: please make sure this server can '
                             'access `${url}` for the update link to work',
                             mapping=dict(url=c.rhodecode_update_url))
         version = UpdateModel().get_stored_version()
         is_outdated = UpdateModel().is_outdated(
             rhodecode.__version__, version)
         update_state = {
             'type': 'warning',
             'message': 'New version available: {}'.format(version)
             } \
             if is_outdated else {}
         c.data_items = [
             # update info
             (_('Update info'), h.literal(
                 '<span class="link" id="check_for_update" >%s.</span>' % (
                 _('Check for updates')) +
                 '<br/> <span >%s.</span>' % (update_info_msg)
             ), ''),
 
             # RhodeCode specific
             (_('RhodeCode Version'), val('rhodecode_app')['text'], state('rhodecode_app')),
             (_('Latest version'), version, update_state),
             (_('RhodeCode Base URL'), val('rhodecode_config')['config'].get('app.base_url'), state('rhodecode_config')),
             (_('RhodeCode Server IP'), val('server')['server_ip'], state('server')),
             (_('RhodeCode Server ID'), val('server')['server_id'], state('server')),
             (_('RhodeCode Configuration'), val('rhodecode_config')['path'], state('rhodecode_config')),
             (_('RhodeCode Certificate'), val('rhodecode_config')['cert_path'], state('rhodecode_config')),
             (_('Workers'), val('rhodecode_config')['config']['server:main'].get('workers', '?'), state('rhodecode_config')),
             (_('Worker Type'), val('rhodecode_config')['config']['server:main'].get('worker_class', 'sync'), state('rhodecode_config')),
             ('', '', ''),  # spacer
 
             # Database
             (_('Database'), val('database')['url'], state('database')),
             (_('Database version'), val('database')['version'], state('database')),
             ('', '', ''),  # spacer
 
             # Platform/Python
             (_('Platform'), val('platform')['name'], state('platform')),
             (_('Platform UUID'), val('platform')['uuid'], state('platform')),
             (_('Lang'), val('locale'), state('locale')),
             (_('Python version'), val('python')['version'], state('python')),
             (_('Python path'), val('python')['executable'], state('python')),
             ('', '', ''),  # spacer
 
             # Systems stats
             (_('CPU'), val('cpu')['text'], state('cpu')),
             (_('Load'), val('load')['text'], state('load')),
             (_('Memory'), val('memory')['text'], state('memory')),
             (_('Uptime'), val('uptime')['text'], state('uptime')),
             ('', '', ''),  # spacer
 
             # ulimit
             (_('Ulimit'), val('ulimit')['text'], state('ulimit')),
 
             # Repo storage
             (_('Storage location'), val('storage')['path'], state('storage')),
             (_('Storage info'), val('storage')['text'], state('storage')),
             (_('Storage inodes'), val('storage_inodes')['text'], state('storage_inodes')),
+            ('', '', ''),  # spacer
 
             (_('Gist storage location'), val('storage_gist')['path'], state('storage_gist')),
             (_('Gist storage info'), val('storage_gist')['text'], state('storage_gist')),
+            ('', '', ''),  # spacer
 
+            (_('Archive cache storage type'), val('storage_archive')['type'], state('storage_archive')),
             (_('Archive cache storage location'), val('storage_archive')['path'], state('storage_archive')),
             (_('Archive cache info'), val('storage_archive')['text'], state('storage_archive')),
+            ('', '', ''),  # spacer
 
             (_('Temp storage location'), val('storage_temp')['path'], state('storage_temp')),
             (_('Temp storage info'), val('storage_temp')['text'], state('storage_temp')),
+            ('', '', ''),  # spacer
 
             (_('Search info'), val('search')['text'], state('search')),
             (_('Search location'), val('search')['location'], state('search')),
             ('', '', ''),  # spacer
 
             # VCS specific
             (_('VCS Backends'), val('vcs_backends'), state('vcs_backends')),
             (_('VCS Server'), val('vcs_server')['text'], state('vcs_server')),
             (_('GIT'), val('git'), state('git')),
             (_('HG'), val('hg'), state('hg')),
             (_('SVN'), val('svn'), state('svn')),
 
         ]
 
         c.vcsserver_data_items = [
-            (k, v) for k,v in (val('vcs_server_config') or {}).items()
+            (k, v) for k, v in (val('vcs_server_config') or {}).items()
         ]
 
         if snapshot:
             if c.allowed_to_snapshot:
                 c.data_items.pop(0)  # remove server info
                 self.request.override_renderer = 'admin/settings/settings_system_snapshot.mako'
             else:
                 h.flash('You are not allowed to do this', category='warning')
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def settings_system_info_check_update(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         update_url = UpdateModel().get_update_url()
 
         def _err(s):
-            return '<div style="color:#ff8888; padding:4px 0px">{}</div>'.format(s)
+            return f'<div style="color:#ff8888; padding:4px 0px">{s}</div>'
+
         try:
             data = UpdateModel().get_update_data(update_url)
         except urllib.error.URLError as e:
             log.exception("Exception contacting upgrade server")
             self.request.override_renderer = 'string'
             return _err('Failed to contact upgrade server: %r' % e)
         except ValueError as e:
             log.exception("Bad data sent from update server")
             self.request.override_renderer = 'string'
             return _err('Bad data sent from update server')
 
         latest = data['versions'][0]
 
         c.update_url = update_url
         c.latest_data = latest
-        c.latest_ver = latest['version']
-        c.cur_ver = rhodecode.__version__
+        c.latest_ver = (latest['version'] or '').strip()
+        c.cur_ver = self.request.GET.get('ver') or rhodecode.__version__
         c.should_upgrade = False
 
-        is_oudated = UpdateModel().is_outdated(c.cur_ver, c.latest_ver)
-        if is_oudated:
+        is_outdated = UpdateModel().is_outdated(c.cur_ver, c.latest_ver)
+        if is_outdated:
             c.should_upgrade = True
         c.important_notices = latest['general']
         UpdateModel().store_version(latest['version'])
         return self._get_template_context(c)

rhodecode/apps/admin/views/users.py

@@ -1,1321 +1,1321 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import datetime
 import formencode
 import formencode.htmlfill
 
 from pyramid.httpexceptions import HTTPFound
 from pyramid.renderers import render
 from pyramid.response import Response
 
 from rhodecode import events
 from rhodecode.apps._base import BaseAppView, DataGridAppView, UserAppView
-from rhodecode.apps.ssh_support import SshKeyFileChangeEvent
+from rhodecode.apps.ssh_support.events import SshKeyFileChangeEvent
 from rhodecode.authentication.base import get_authn_registry, RhodeCodeExternalAuthPlugin
 from rhodecode.authentication.plugins import auth_rhodecode
 from rhodecode.events import trigger
 from rhodecode.model.db import true, UserNotice
 
 from rhodecode.lib import audit_logger, rc_cache, auth
 from rhodecode.lib.exceptions import (
     UserCreationError, UserOwnsReposException, UserOwnsRepoGroupsException,
     UserOwnsUserGroupsException, UserOwnsPullRequestsException,
     UserOwnsArtifactsException, DefaultUserException)
 from rhodecode.lib import ext_json
 from rhodecode.lib.auth import (
     LoginRequired, HasPermissionAllDecorator, CSRFRequired)
 from rhodecode.lib import helpers as h
 from rhodecode.lib.helpers import SqlPage
 from rhodecode.lib.utils2 import safe_int, safe_str, AttributeDict
 from rhodecode.model.auth_token import AuthTokenModel
 from rhodecode.model.forms import (
     UserForm, UserIndividualPermissionsForm, UserPermissionsForm,
     UserExtraEmailForm, UserExtraIpForm)
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.ssh_key import SshKeyModel
 from rhodecode.model.user import UserModel
 from rhodecode.model.user_group import UserGroupModel
 from rhodecode.model.db import (
     or_, coalesce,IntegrityError, User, UserGroup, UserIpMap, UserEmailMap,
     UserApiKeys, UserSshKeys, RepoGroup)
 from rhodecode.model.meta import Session
 
 log = logging.getLogger(__name__)
 
 
 class AdminUsersView(BaseAppView, DataGridAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         return c
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def users_list(self):
         c = self.load_default_context()
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def users_list_data(self):
         self.load_default_context()
         column_map = {
             'first_name': 'name',
             'last_name': 'lastname',
         }
         draw, start, limit = self._extract_chunk(self.request)
         search_q, order_by, order_dir = self._extract_ordering(
             self.request, column_map=column_map)
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/data_table/_dt_elements.mako')
 
         def user_actions(user_id, username):
             return _render("user_actions", user_id, username)
 
         users_data_total_count = User.query()\
             .filter(User.username != User.DEFAULT_USER) \
             .count()
 
         users_data_total_inactive_count = User.query()\
             .filter(User.username != User.DEFAULT_USER) \
             .filter(User.active != true())\
             .count()
 
         # json generate
         base_q = User.query().filter(User.username != User.DEFAULT_USER)
         base_inactive_q = base_q.filter(User.active != true())
 
         if search_q:
             like_expression = '%{}%'.format(safe_str(search_q))
             base_q = base_q.filter(or_(
                 User.username.ilike(like_expression),
                 User._email.ilike(like_expression),
                 User.name.ilike(like_expression),
                 User.lastname.ilike(like_expression),
             ))
             base_inactive_q = base_q.filter(User.active != true())
 
         users_data_total_filtered_count = base_q.count()
         users_data_total_filtered_inactive_count = base_inactive_q.count()
 
         sort_col = getattr(User, order_by, None)
         if sort_col:
             if order_dir == 'asc':
                 # handle null values properly to order by NULL last
                 if order_by in ['last_activity']:
                     sort_col = coalesce(sort_col, datetime.date.max)
                 sort_col = sort_col.asc()
             else:
                 # handle null values properly to order by NULL last
                 if order_by in ['last_activity']:
                     sort_col = coalesce(sort_col, datetime.date.min)
                 sort_col = sort_col.desc()
 
         base_q = base_q.order_by(sort_col)
         base_q = base_q.offset(start).limit(limit)
 
         users_list = base_q.all()
 
         users_data = []
         for user in users_list:
             users_data.append({
                 "username": h.gravatar_with_user(self.request, user.username),
                 "email": user.email,
                 "first_name": user.first_name,
                 "last_name": user.last_name,
                 "last_login": h.format_date(user.last_login),
                 "last_activity": h.format_date(user.last_activity),
                 "active": h.bool2icon(user.active),
                 "active_raw": user.active,
                 "admin": h.bool2icon(user.admin),
                 "extern_type": user.extern_type,
                 "extern_name": user.extern_name,
                 "action": user_actions(user.user_id, user.username),
             })
         data = ({
             'draw': draw,
             'data': users_data,
             'recordsTotal': users_data_total_count,
             'recordsFiltered': users_data_total_filtered_count,
             'recordsTotalInactive': users_data_total_inactive_count,
             'recordsFilteredInactive': users_data_total_filtered_inactive_count
         })
 
         return data
 
     def _set_personal_repo_group_template_vars(self, c_obj):
         DummyUser = AttributeDict({
             'username': '${username}',
             'user_id': '${user_id}',
         })
         c_obj.default_create_repo_group = RepoGroupModel() \
             .get_default_create_personal_repo_group()
         c_obj.personal_repo_group_name = RepoGroupModel() \
             .get_personal_group_name(DummyUser)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def users_new(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.default_extern_type = auth_rhodecode.RhodeCodeAuthPlugin.uid
         self._set_personal_repo_group_template_vars(c)
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def users_create(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.default_extern_type = auth_rhodecode.RhodeCodeAuthPlugin.uid
         user_model = UserModel()
         user_form = UserForm(self.request.translate)()
         try:
             form_result = user_form.to_python(dict(self.request.POST))
             user = user_model.create(form_result)
             Session().flush()
             creation_data = user.get_api_data()
             username = form_result['username']
 
             audit_logger.store_web(
                 'user.create', action_data={'data': creation_data},
                 user=c.rhodecode_user)
 
             user_link = h.link_to(
                 h.escape(username),
                 h.route_path('user_edit', user_id=user.user_id))
             h.flash(h.literal(_('Created user %(user_link)s')
                               % {'user_link': user_link}), category='success')
             Session().commit()
         except formencode.Invalid as errors:
             self._set_personal_repo_group_template_vars(c)
             data = render(
                 'rhodecode:templates/admin/users/user_add.mako',
                 self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
-                errors=errors.unpack_errors() or {},
+                errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
         except UserCreationError as e:
             h.flash(safe_str(e), 'error')
         except Exception:
             log.exception("Exception creation of user")
             h.flash(_('Error occurred during creation of user %s')
                     % self.request.POST.get('username'), category='error')
         raise HTTPFound(h.route_path('users'))
 
 
 class UsersView(UserAppView):
     ALLOW_SCOPED_TOKENS = False
     """
     This view has alternative version inside EE, if modified please take a look
     in there as well.
     """
 
     def get_auth_plugins(self):
         valid_plugins = []
         authn_registry = get_authn_registry(self.request.registry)
         for plugin in authn_registry.get_plugins_for_authentication():
             if isinstance(plugin, RhodeCodeExternalAuthPlugin):
                 valid_plugins.append(plugin)
             elif plugin.name == 'rhodecode':
                 valid_plugins.append(plugin)
 
         # extend our choices if user has set a bound plugin which isn't enabled at the
         # moment
         extern_type = self.db_user.extern_type
         if extern_type not in [x.uid for x in valid_plugins]:
             try:
                 plugin = authn_registry.get_plugin_by_uid(extern_type)
                 if plugin:
                     valid_plugins.append(plugin)
 
             except Exception:
                 log.exception(
                     f'Could not extend user plugins with `{extern_type}`')
         return valid_plugins
 
     def load_default_context(self):
         req = self.request
 
         c = self._get_local_tmpl_context()
         c.allow_scoped_tokens = self.ALLOW_SCOPED_TOKENS
         c.allowed_languages = [
             ('en', 'English (en)'),
             ('de', 'German (de)'),
             ('fr', 'French (fr)'),
             ('it', 'Italian (it)'),
             ('ja', 'Japanese (ja)'),
             ('pl', 'Polish (pl)'),
             ('pt', 'Portuguese (pt)'),
             ('ru', 'Russian (ru)'),
             ('zh', 'Chinese (zh)'),
         ]
 
         c.allowed_extern_types = [
             (x.uid, x.get_display_name()) for x in self.get_auth_plugins()
         ]
         perms = req.registry.settings.get('available_permissions')
         if not perms:
             # inject info about available permissions
             auth.set_available_permissions(req.registry.settings)
 
         c.available_permissions = req.registry.settings['available_permissions']
         PermissionModel().set_global_permission_choices(
             c, gettext_translator=req.translate)
 
         return c
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def user_update(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         c.active = 'profile'
         c.extern_type = c.user.extern_type
         c.extern_name = c.user.extern_name
         c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
         available_languages = [x[0] for x in c.allowed_languages]
         _form = UserForm(self.request.translate, edit=True,
                          available_languages=available_languages,
                          old_data={'user_id': user_id,
                                    'email': c.user.email})()
 
         c.edit_mode = self.request.POST.get('edit') == '1'
         form_result = {}
         old_values = c.user.get_api_data()
         try:
             form_result = _form.to_python(dict(self.request.POST))
             skip_attrs = ['extern_name']
             # TODO: plugin should define if username can be updated
 
             if c.extern_type != "rhodecode" and not c.edit_mode:
                 # forbid updating username for external accounts
                 skip_attrs.append('username')
 
             UserModel().update_user(
                 user_id, skip_attrs=skip_attrs, **form_result)
 
             audit_logger.store_web(
                 'user.edit', action_data={'old_data': old_values},
                 user=c.rhodecode_user)
 
             Session().commit()
             h.flash(_('User updated successfully'), category='success')
         except formencode.Invalid as errors:
             data = render(
                 'rhodecode:templates/admin/users/user_edit.mako',
                 self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.unpack_errors() or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
         except UserCreationError as e:
             h.flash(safe_str(e), 'error')
         except Exception:
             log.exception("Exception updating user")
             h.flash(_('Error occurred during update of user %s')
                     % form_result.get('username'), category='error')
         raise HTTPFound(h.route_path('user_edit', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def user_delete(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         _repos = len(c.user.repositories)
         _repo_groups = len(c.user.repository_groups)
         _user_groups = len(c.user.user_groups)
         _pull_requests = len(c.user.user_pull_requests)
         _artifacts = len(c.user.artifacts)
 
         handle_repos = None
         handle_repo_groups = None
         handle_user_groups = None
         handle_pull_requests = None
         handle_artifacts = None
 
         # calls for flash of handle based on handle case detach or delete
         def set_handle_flash_repos():
             handle = handle_repos
             if handle == 'detach':
                 h.flash(_('Detached %s repositories') % _repos,
                         category='success')
             elif handle == 'delete':
                 h.flash(_('Deleted %s repositories') % _repos,
                         category='success')
 
         def set_handle_flash_repo_groups():
             handle = handle_repo_groups
             if handle == 'detach':
                 h.flash(_('Detached %s repository groups') % _repo_groups,
                         category='success')
             elif handle == 'delete':
                 h.flash(_('Deleted %s repository groups') % _repo_groups,
                         category='success')
 
         def set_handle_flash_user_groups():
             handle = handle_user_groups
             if handle == 'detach':
                 h.flash(_('Detached %s user groups') % _user_groups,
                         category='success')
             elif handle == 'delete':
                 h.flash(_('Deleted %s user groups') % _user_groups,
                         category='success')
 
         def set_handle_flash_pull_requests():
             handle = handle_pull_requests
             if handle == 'detach':
                 h.flash(_('Detached %s pull requests') % _pull_requests,
                         category='success')
             elif handle == 'delete':
                 h.flash(_('Deleted %s pull requests') % _pull_requests,
                         category='success')
 
         def set_handle_flash_artifacts():
             handle = handle_artifacts
             if handle == 'detach':
                 h.flash(_('Detached %s artifacts') % _artifacts,
                         category='success')
             elif handle == 'delete':
                 h.flash(_('Deleted %s artifacts') % _artifacts,
                         category='success')
 
         handle_user = User.get_first_super_admin()
         handle_user_id = safe_int(self.request.POST.get('detach_user_id'))
         if handle_user_id:
             # NOTE(marcink): we get new owner for objects...
             handle_user = User.get_or_404(handle_user_id)
 
         if _repos and self.request.POST.get('user_repos'):
             handle_repos = self.request.POST['user_repos']
 
         if _repo_groups and self.request.POST.get('user_repo_groups'):
             handle_repo_groups = self.request.POST['user_repo_groups']
 
         if _user_groups and self.request.POST.get('user_user_groups'):
             handle_user_groups = self.request.POST['user_user_groups']
 
         if _pull_requests and self.request.POST.get('user_pull_requests'):
             handle_pull_requests = self.request.POST['user_pull_requests']
 
         if _artifacts and self.request.POST.get('user_artifacts'):
             handle_artifacts = self.request.POST['user_artifacts']
 
         old_values = c.user.get_api_data()
 
         try:
 
             UserModel().delete(
                 c.user,
                 handle_repos=handle_repos,
                 handle_repo_groups=handle_repo_groups,
                 handle_user_groups=handle_user_groups,
                 handle_pull_requests=handle_pull_requests,
                 handle_artifacts=handle_artifacts,
                 handle_new_owner=handle_user
             )
 
             audit_logger.store_web(
                 'user.delete', action_data={'old_data': old_values},
                 user=c.rhodecode_user)
 
             Session().commit()
             set_handle_flash_repos()
             set_handle_flash_repo_groups()
             set_handle_flash_user_groups()
             set_handle_flash_pull_requests()
             set_handle_flash_artifacts()
             username = h.escape(old_values['username'])
             h.flash(_('Successfully deleted user `{}`').format(username), category='success')
         except (UserOwnsReposException, UserOwnsRepoGroupsException,
                 UserOwnsUserGroupsException, UserOwnsPullRequestsException,
                 UserOwnsArtifactsException, DefaultUserException) as e:
 
             h.flash(safe_str(e), category='warning')
         except Exception:
             log.exception("Exception during deletion of user")
             h.flash(_('An error occurred during deletion of user'),
                     category='error')
         raise HTTPFound(h.route_path('users'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def user_edit(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'profile'
         c.extern_type = c.user.extern_type
         c.extern_name = c.user.extern_name
         c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
         c.edit_mode = self.request.GET.get('edit') == '1'
 
         defaults = c.user.get_dict()
         defaults.update({'language': c.user.user_data.get('language')})
 
         data = render(
             'rhodecode:templates/admin/users/user_edit.mako',
             self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def user_edit_advanced(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         c.detach_user = User.get_first_super_admin()
         detach_user_id = safe_int(self.request.GET.get('detach_user_id'))
         if detach_user_id:
             c.detach_user = User.get_or_404(detach_user_id)
 
         c.active = 'advanced'
         c.personal_repo_group = RepoGroup.get_user_personal_repo_group(user_id)
         c.personal_repo_group_name = RepoGroupModel()\
             .get_personal_group_name(c.user)
 
         c.user_to_review_rules = sorted(
             (x.user for x in c.user.user_review_rules),
             key=lambda u: u.username.lower())
 
         defaults = c.user.get_dict()
 
         # Interim workaround if the user participated on any pull requests as a
         # reviewer.
         has_review = len(c.user.reviewer_pull_requests)
         c.can_delete_user = not has_review
         c.can_delete_user_message = ''
         inactive_link = h.link_to(
             'inactive', h.route_path('user_edit', user_id=user_id, _anchor='active'))
         if has_review == 1:
             c.can_delete_user_message = h.literal(_(
                 'The user participates as reviewer in {} pull request and '
                 'cannot be deleted. \nYou can set the user to '
                 '"{}" instead of deleting it.').format(
                 has_review, inactive_link))
         elif has_review:
             c.can_delete_user_message = h.literal(_(
                 'The user participates as reviewer in {} pull requests and '
                 'cannot be deleted. \nYou can set the user to '
                 '"{}" instead of deleting it.').format(
                 has_review, inactive_link))
 
         data = render(
             'rhodecode:templates/admin/users/user_edit.mako',
             self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def user_edit_global_perms(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'global_perms'
 
         c.default_user = User.get_default_user()
         defaults = c.user.get_dict()
         defaults.update(c.default_user.get_default_perms(suffix='_inherited'))
         defaults.update(c.default_user.get_default_perms())
         defaults.update(c.user.get_default_perms())
 
         data = render(
             'rhodecode:templates/admin/users/user_edit.mako',
             self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def user_edit_global_perms_update(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         c.active = 'global_perms'
         try:
             # first stage that verifies the checkbox
             _form = UserIndividualPermissionsForm(self.request.translate)
             form_result = _form.to_python(dict(self.request.POST))
             inherit_perms = form_result['inherit_default_permissions']
             c.user.inherit_default_permissions = inherit_perms
             Session().add(c.user)
 
             if not inherit_perms:
                 # only update the individual ones if we un check the flag
                 _form = UserPermissionsForm(
                     self.request.translate,
                     [x[0] for x in c.repo_create_choices],
                     [x[0] for x in c.repo_create_on_write_choices],
                     [x[0] for x in c.repo_group_create_choices],
                     [x[0] for x in c.user_group_create_choices],
                     [x[0] for x in c.fork_choices],
                     [x[0] for x in c.inherit_default_permission_choices])()
 
                 form_result = _form.to_python(dict(self.request.POST))
                 form_result.update({'perm_user_id': c.user.user_id})
 
                 PermissionModel().update_user_permissions(form_result)
 
             # TODO(marcink): implement global permissions
             # audit_log.store_web('user.edit.permissions')
 
             Session().commit()
 
             h.flash(_('User global permissions updated successfully'),
                     category='success')
 
         except formencode.Invalid as errors:
             data = render(
                 'rhodecode:templates/admin/users/user_edit.mako',
                 self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.unpack_errors() or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
         except Exception:
             log.exception("Exception during permissions saving")
             h.flash(_('An error occurred during permissions saving'),
                     category='error')
 
         affected_user_ids = [user_id]
         PermissionModel().trigger_permission_flush(affected_user_ids)
         raise HTTPFound(h.route_path('user_edit_global_perms', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def user_enable_force_password_reset(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         try:
             c.user.update_userdata(force_password_change=True)
 
             msg = _('Force password change enabled for user')
             audit_logger.store_web('user.edit.password_reset.enabled',
                                    user=c.rhodecode_user)
 
             Session().commit()
             h.flash(msg, category='success')
         except Exception:
             log.exception("Exception during password reset for user")
             h.flash(_('An error occurred during password reset for user'),
                     category='error')
 
         raise HTTPFound(h.route_path('user_edit_advanced', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def user_disable_force_password_reset(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         try:
             c.user.update_userdata(force_password_change=False)
 
             msg = _('Force password change disabled for user')
             audit_logger.store_web(
                 'user.edit.password_reset.disabled',
                 user=c.rhodecode_user)
 
             Session().commit()
             h.flash(msg, category='success')
         except Exception:
             log.exception("Exception during password reset for user")
             h.flash(_('An error occurred during password reset for user'),
                     category='error')
 
         raise HTTPFound(h.route_path('user_edit_advanced', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def user_notice_dismiss(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
         user_notice_id = safe_int(self.request.POST.get('notice_id'))
         notice = UserNotice().query()\
             .filter(UserNotice.user_id == user_id)\
             .filter(UserNotice.user_notice_id == user_notice_id)\
             .scalar()
         read = False
         if notice:
             notice.notice_read = True
             Session().add(notice)
             Session().commit()
             read = True
 
         return {'notice': user_notice_id, 'read': read}
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def user_create_personal_repo_group(self):
         """
         Create personal repository group for this user
         """
         from rhodecode.model.repo_group import RepoGroupModel
 
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         personal_repo_group = RepoGroup.get_user_personal_repo_group(
             c.user.user_id)
         if personal_repo_group:
             raise HTTPFound(h.route_path('user_edit_advanced', user_id=user_id))
 
         personal_repo_group_name = RepoGroupModel().get_personal_group_name(c.user)
         named_personal_group = RepoGroup.get_by_group_name(
             personal_repo_group_name)
         try:
 
             if named_personal_group and named_personal_group.user_id == c.user.user_id:
                 # migrate the same named group, and mark it as personal
                 named_personal_group.personal = True
                 Session().add(named_personal_group)
                 Session().commit()
                 msg = _('Linked repository group `{}` as personal'.format(
                     personal_repo_group_name))
                 h.flash(msg, category='success')
             elif not named_personal_group:
                 RepoGroupModel().create_personal_repo_group(c.user)
 
                 msg = _('Created repository group `{}`'.format(
                     personal_repo_group_name))
                 h.flash(msg, category='success')
             else:
                 msg = _('Repository group `{}` is already taken'.format(
                     personal_repo_group_name))
                 h.flash(msg, category='warning')
         except Exception:
             log.exception("Exception during repository group creation")
             msg = _(
                 'An error occurred during repository group creation for user')
             h.flash(msg, category='error')
             Session().rollback()
 
         raise HTTPFound(h.route_path('user_edit_advanced', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def auth_tokens(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'auth_tokens'
 
         c.lifetime_values = AuthTokenModel.get_lifetime_values(translator=_)
         c.role_values = [
             (x, AuthTokenModel.cls._get_role_name(x))
             for x in AuthTokenModel.cls.ROLES]
         c.role_options = [(c.role_values, _("Role"))]
         c.user_auth_tokens = AuthTokenModel().get_auth_tokens(
             c.user.user_id, show_expired=True)
         c.role_vcs = AuthTokenModel.cls.ROLE_VCS
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def auth_tokens_view(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         auth_token_id = self.request.POST.get('auth_token_id')
 
         if auth_token_id:
             token = UserApiKeys.get_or_404(auth_token_id)
 
             return {
                 'auth_token': token.api_key
             }
 
     def maybe_attach_token_scope(self, token):
         # implemented in EE edition
         pass
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def auth_tokens_add(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         user_data = c.user.get_api_data()
         lifetime = safe_int(self.request.POST.get('lifetime'), -1)
         description = self.request.POST.get('description')
         role = self.request.POST.get('role')
 
         token = UserModel().add_auth_token(
             user=c.user.user_id,
             lifetime_minutes=lifetime, role=role, description=description,
             scope_callback=self.maybe_attach_token_scope)
         token_data = token.get_api_data()
 
         audit_logger.store_web(
             'user.edit.token.add', action_data={
                 'data': {'token': token_data, 'user': user_data}},
             user=self._rhodecode_user, )
         Session().commit()
 
         h.flash(_("Auth token successfully created"), category='success')
         return HTTPFound(h.route_path('edit_user_auth_tokens', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def auth_tokens_delete(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         user_data = c.user.get_api_data()
 
         del_auth_token = self.request.POST.get('del_auth_token')
 
         if del_auth_token:
             token = UserApiKeys.get_or_404(del_auth_token)
             token_data = token.get_api_data()
 
             AuthTokenModel().delete(del_auth_token, c.user.user_id)
             audit_logger.store_web(
                 'user.edit.token.delete', action_data={
                     'data': {'token': token_data, 'user': user_data}},
                 user=self._rhodecode_user,)
             Session().commit()
             h.flash(_("Auth token successfully deleted"), category='success')
 
         return HTTPFound(h.route_path('edit_user_auth_tokens', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def ssh_keys(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'ssh_keys'
         c.default_key = self.request.GET.get('default_key')
         c.user_ssh_keys = SshKeyModel().get_ssh_keys(c.user.user_id)
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def ssh_keys_generate_keypair(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         c.user = self.db_user
 
         c.active = 'ssh_keys_generate'
         comment = 'RhodeCode-SSH {}'.format(c.user.email or '')
         private_format = self.request.GET.get('private_format') \
                          or SshKeyModel.DEFAULT_PRIVATE_KEY_FORMAT
         c.private, c.public = SshKeyModel().generate_keypair(
             comment=comment, private_format=private_format)
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def ssh_keys_add(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         user_data = c.user.get_api_data()
         key_data = self.request.POST.get('key_data')
         description = self.request.POST.get('description')
 
         fingerprint = 'unknown'
         try:
             if not key_data:
                 raise ValueError('Please add a valid public key')
 
             key = SshKeyModel().parse_key(key_data.strip())
             fingerprint = key.hash_md5()
 
             ssh_key = SshKeyModel().create(
                 c.user.user_id, fingerprint, key.keydata, description)
             ssh_key_data = ssh_key.get_api_data()
 
             audit_logger.store_web(
                 'user.edit.ssh_key.add', action_data={
                     'data': {'ssh_key': ssh_key_data, 'user': user_data}},
                 user=self._rhodecode_user, )
             Session().commit()
 
             # Trigger an event on change of keys.
             trigger(SshKeyFileChangeEvent(), self.request.registry)
 
             h.flash(_("Ssh Key successfully created"), category='success')
 
         except IntegrityError:
             log.exception("Exception during ssh key saving")
             err = 'Such key with fingerprint `{}` already exists, ' \
                   'please use a different one'.format(fingerprint)
             h.flash(_('An error occurred during ssh key saving: {}').format(err),
                     category='error')
         except Exception as e:
             log.exception("Exception during ssh key saving")
             h.flash(_('An error occurred during ssh key saving: {}').format(e),
                     category='error')
 
         return HTTPFound(
             h.route_path('edit_user_ssh_keys', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def ssh_keys_delete(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         user_data = c.user.get_api_data()
 
         del_ssh_key = self.request.POST.get('del_ssh_key')
 
         if del_ssh_key:
             ssh_key = UserSshKeys.get_or_404(del_ssh_key)
             ssh_key_data = ssh_key.get_api_data()
 
             SshKeyModel().delete(del_ssh_key, c.user.user_id)
             audit_logger.store_web(
                 'user.edit.ssh_key.delete', action_data={
                     'data': {'ssh_key': ssh_key_data, 'user': user_data}},
                 user=self._rhodecode_user,)
             Session().commit()
             # Trigger an event on change of keys.
             trigger(SshKeyFileChangeEvent(), self.request.registry)
             h.flash(_("Ssh key successfully deleted"), category='success')
 
         return HTTPFound(h.route_path('edit_user_ssh_keys', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def emails(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'emails'
         c.user_email_map = UserEmailMap.query() \
             .filter(UserEmailMap.user == c.user).all()
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def emails_add(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         email = self.request.POST.get('new_email')
         user_data = c.user.get_api_data()
         try:
 
             form = UserExtraEmailForm(self.request.translate)()
             data = form.to_python({'email': email})
             email = data['email']
 
             UserModel().add_extra_email(c.user.user_id, email)
             audit_logger.store_web(
                 'user.edit.email.add',
                 action_data={'email': email, 'user': user_data},
                 user=self._rhodecode_user)
             Session().commit()
             h.flash(_("Added new email address `%s` for user account") % email,
                     category='success')
         except formencode.Invalid as error:
             msg = error.unpack_errors()['email']
             h.flash(h.escape(msg), category='error')
         except IntegrityError:
             log.warning("Email %s already exists", email)
             h.flash(_('Email `{}` is already registered for another user.').format(email),
                     category='error')
         except Exception:
             log.exception("Exception during email saving")
             h.flash(_('An error occurred during email saving'),
                     category='error')
         raise HTTPFound(h.route_path('edit_user_emails', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def emails_delete(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         email_id = self.request.POST.get('del_email_id')
         user_model = UserModel()
 
         email = UserEmailMap.query().get(email_id).email
         user_data = c.user.get_api_data()
         user_model.delete_extra_email(c.user.user_id, email_id)
         audit_logger.store_web(
             'user.edit.email.delete',
             action_data={'email': email, 'user': user_data},
             user=self._rhodecode_user)
         Session().commit()
         h.flash(_("Removed email address from user account"),
                 category='success')
         raise HTTPFound(h.route_path('edit_user_emails', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def ips(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'ips'
         c.user_ip_map = UserIpMap.query() \
             .filter(UserIpMap.user == c.user).all()
 
         c.inherit_default_ips = c.user.inherit_default_permissions
         c.default_user_ip_map = UserIpMap.query() \
             .filter(UserIpMap.user == User.get_default_user()).all()
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     # NOTE(marcink): this view is allowed for default users, as we can
     # edit their IP white list
     def ips_add(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         user_model = UserModel()
         desc = self.request.POST.get('description')
         try:
             ip_list = user_model.parse_ip_range(
                 self.request.POST.get('new_ip'))
         except Exception as e:
             ip_list = []
             log.exception("Exception during ip saving")
             h.flash(_('An error occurred during ip saving:%s' % (e,)),
                     category='error')
         added = []
         user_data = c.user.get_api_data()
         for ip in ip_list:
             try:
                 form = UserExtraIpForm(self.request.translate)()
                 data = form.to_python({'ip': ip})
                 ip = data['ip']
 
                 user_model.add_extra_ip(c.user.user_id, ip, desc)
                 audit_logger.store_web(
                     'user.edit.ip.add',
                     action_data={'ip': ip, 'user': user_data},
                     user=self._rhodecode_user)
                 Session().commit()
                 added.append(ip)
             except formencode.Invalid as error:
                 msg = error.unpack_errors()['ip']
                 h.flash(msg, category='error')
             except Exception:
                 log.exception("Exception during ip saving")
                 h.flash(_('An error occurred during ip saving'),
                         category='error')
         if added:
             h.flash(
                 _("Added ips %s to user whitelist") % (', '.join(ip_list), ),
                 category='success')
         if 'default_user' in self.request.POST:
             # case for editing global IP list we do it for 'DEFAULT' user
             raise HTTPFound(h.route_path('admin_permissions_ips'))
         raise HTTPFound(h.route_path('edit_user_ips', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     # NOTE(marcink): this view is allowed for default users, as we can
     # edit their IP white list
     def ips_delete(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         ip_id = self.request.POST.get('del_ip_id')
         user_model = UserModel()
         user_data = c.user.get_api_data()
         ip = UserIpMap.query().get(ip_id).ip_addr
         user_model.delete_extra_ip(c.user.user_id, ip_id)
         audit_logger.store_web(
             'user.edit.ip.delete', action_data={'ip': ip, 'user': user_data},
             user=self._rhodecode_user)
         Session().commit()
         h.flash(_("Removed ip address from user whitelist"), category='success')
 
         if 'default_user' in self.request.POST:
             # case for editing global IP list we do it for 'DEFAULT' user
             raise HTTPFound(h.route_path('admin_permissions_ips'))
         raise HTTPFound(h.route_path('edit_user_ips', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def groups_management(self):
         c = self.load_default_context()
         c.user = self.db_user
         c.data = c.user.group_member
 
         groups = [UserGroupModel.get_user_groups_as_dict(group.users_group)
                   for group in c.user.group_member]
         c.groups = ext_json.str_json(groups)
         c.active = 'groups'
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def groups_management_updates(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         user_groups = set(self.request.POST.getall('users_group_id'))
         user_groups_objects = []
 
         for ugid in user_groups:
             user_groups_objects.append(
                 UserGroupModel().get_group(safe_int(ugid)))
         user_group_model = UserGroupModel()
         added_to_groups, removed_from_groups = \
             user_group_model.change_groups(c.user, user_groups_objects)
 
         user_data = c.user.get_api_data()
         for user_group_id in added_to_groups:
             user_group = UserGroup.get(user_group_id)
             old_values = user_group.get_api_data()
             audit_logger.store_web(
                 'user_group.edit.member.add',
                 action_data={'user': user_data, 'old_data': old_values},
                 user=self._rhodecode_user)
 
         for user_group_id in removed_from_groups:
             user_group = UserGroup.get(user_group_id)
             old_values = user_group.get_api_data()
             audit_logger.store_web(
                 'user_group.edit.member.delete',
                 action_data={'user': user_data, 'old_data': old_values},
                 user=self._rhodecode_user)
 
         Session().commit()
         c.active = 'user_groups_management'
         h.flash(_("Groups successfully changed"), category='success')
 
         return HTTPFound(h.route_path(
             'edit_user_groups_management', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def user_audit_logs(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'audit'
 
         p = safe_int(self.request.GET.get('page', 1), 1)
 
         filter_term = self.request.GET.get('filter')
         user_log = UserModel().get_user_log(c.user, filter_term)
 
         def url_generator(page_num):
             query_params = {
                 'page': page_num
             }
             if filter_term:
                 query_params['filter'] = filter_term
             return self.request.current_route_path(_query=query_params)
 
         c.audit_logs = SqlPage(
             user_log, page=p, items_per_page=10, url_maker=url_generator)
         c.filter_term = filter_term
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def user_audit_logs_download(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         user_log = UserModel().get_user_log(c.user, filter_term=None)
 
         audit_log_data = {}
         for entry in user_log:
             audit_log_data[entry.user_log_id] = entry.get_dict()
 
         response = Response(ext_json.formatted_str_json(audit_log_data))
         response.content_disposition = f'attachment; filename=user_{c.user.user_id}_audit_logs.json'
         response.content_type = 'application/json'
 
         return response
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def user_perms_summary(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'perms_summary'
         c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def user_perms_summary_json(self):
         self.load_default_context()
         perm_user = self.db_user.AuthUser(ip_addr=self.request.remote_addr)
 
         return perm_user.permissions
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def user_caches(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'caches'
         c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
 
         cache_namespace_uid = f'cache_user_auth.{rc_cache.PERMISSIONS_CACHE_VER}.{self.db_user.user_id}'
         c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
         c.backend = c.region.backend
         c.user_keys = sorted(c.region.backend.list_keys(prefix=cache_namespace_uid))
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def user_caches_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'caches'
         c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
 
         cache_namespace_uid = f'cache_user_auth.{rc_cache.PERMISSIONS_CACHE_VER}.{self.db_user.user_id}'
         del_keys = rc_cache.clear_cache_namespace('cache_perms', cache_namespace_uid, method=rc_cache.CLEAR_DELETE)
 
         h.flash(_("Deleted {} cache keys").format(del_keys), category='success')
 
         return HTTPFound(h.route_path(
             'edit_user_caches', user_id=c.user.user_id))

rhodecode/apps/login/__init__.py

@@ -1,77 +1,101 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 from rhodecode.apps._base import ADMIN_PREFIX
 
 
 def includeme(config):
     from rhodecode.apps.login.views import LoginView
     
     config.add_route(
         name='login',
         pattern=ADMIN_PREFIX + '/login')
     config.add_view(
         LoginView,
         attr='login',
         route_name='login', request_method='GET',
         renderer='rhodecode:templates/login.mako')
     config.add_view(
         LoginView,
         attr='login_post',
         route_name='login', request_method='POST',
         renderer='rhodecode:templates/login.mako')
 
     config.add_route(
         name='logout',
         pattern=ADMIN_PREFIX + '/logout')
     config.add_view(
         LoginView,
         attr='logout',
         route_name='logout', request_method='POST')
 
     config.add_route(
         name='register',
         pattern=ADMIN_PREFIX + '/register')
     config.add_view(
         LoginView,
         attr='register',
         route_name='register', request_method='GET',
         renderer='rhodecode:templates/register.mako')
     config.add_view(
         LoginView,
         attr='register_post',
         route_name='register', request_method='POST',
         renderer='rhodecode:templates/register.mako')
 
     config.add_route(
         name='reset_password',
         pattern=ADMIN_PREFIX + '/password_reset')
     config.add_view(
         LoginView,
         attr='password_reset',
         route_name='reset_password', request_method=('GET', 'POST'),
         renderer='rhodecode:templates/password_reset.mako')
 
     config.add_route(
         name='reset_password_confirmation',
         pattern=ADMIN_PREFIX + '/password_reset_confirmation')
     config.add_view(
         LoginView,
         attr='password_reset_confirmation',
         route_name='reset_password_confirmation', request_method='GET')
+
+    config.add_route(
+        name='setup_2fa',
+        pattern=ADMIN_PREFIX + '/setup_2fa')
+    config.add_view(
+        LoginView,
+        attr='setup_2fa',
+        route_name='setup_2fa', request_method=['GET', 'POST'],
+        renderer='rhodecode:templates/configure_2fa.mako')
+
+    config.add_route(
+        name='check_2fa',
+        pattern=ADMIN_PREFIX + '/check_2fa')
+    config.add_view(
+        LoginView,
+        attr='verify_2fa',
+        route_name='check_2fa', request_method='GET',
+        renderer='rhodecode:templates/verify_2fa.mako')
+    config.add_view(
+        LoginView,
+        attr='verify_2fa',
+        route_name='check_2fa', request_method='POST',
+        renderer='rhodecode:templates/verify_2fa.mako')
+

rhodecode/apps/login/tests/test_login.py

@@ -1,581 +1,593 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import urllib.parse
 
 import mock
 import pytest
 
 
 from rhodecode.lib.auth import check_password
 from rhodecode.lib import helpers as h
 from rhodecode.model.auth_token import AuthTokenModel
 from rhodecode.model.db import User, Notification, UserApiKeys
 from rhodecode.model.meta import Session
 
 from rhodecode.tests import (
     assert_session_flash, HG_REPO, TEST_USER_ADMIN_LOGIN,
     no_newline_id_generator)
 from rhodecode.tests.fixture import Fixture
 from rhodecode.tests.routes import route_path
 
 fixture = Fixture()
 
 whitelist_view = ['RepoCommitsView:repo_commit_raw']
 
 
 @pytest.mark.usefixtures('app')
 class TestLoginController(object):
     destroy_users = set()
 
     @classmethod
     def teardown_class(cls):
         fixture.destroy_users(cls.destroy_users)
 
     def teardown_method(self, method):
         for n in Notification.query().all():
             Session().delete(n)
 
         Session().commit()
         assert Notification.query().all() == []
 
     def test_index(self):
         response = self.app.get(route_path('login'))
         assert response.status == '200 OK'
         # Test response...
 
     def test_login_admin_ok(self):
         response = self.app.post(route_path('login'),
                                  {'username': 'test_admin',
                                   'password': 'test12'}, status=302)
         response = response.follow()
         session = response.get_session_from_response()
         username = session['rhodecode_user'].get('username')
         assert username == 'test_admin'
         response.mustcontain('logout')
 
     def test_login_regular_ok(self):
         response = self.app.post(route_path('login'),
                                  {'username': 'test_regular',
                                   'password': 'test12'}, status=302)
 
         response = response.follow()
         session = response.get_session_from_response()
         username = session['rhodecode_user'].get('username')
         assert username == 'test_regular'
         response.mustcontain('logout')
 
+    def test_login_with_primary_email(self):
+        user_email = '[e-mail]'
+        response = self.app.post(route_path('login'),
+                                 {'username': user_email,
+                                  'password': 'test12'}, status=302)
+        response = response.follow()
+        session = response.get_session_from_response()
+        user = session['rhodecode_user']
+        assert user['username'] == user_email.split('@')[0]
+        assert user['is_authenticated']
+        response.mustcontain('logout')
+
     def test_login_regular_forbidden_when_super_admin_restriction(self):
         from rhodecode.authentication.plugins.auth_rhodecode import RhodeCodeAuthPlugin
         with fixture.auth_restriction(self.app._pyramid_registry,
                                       RhodeCodeAuthPlugin.AUTH_RESTRICTION_SUPER_ADMIN):
             response = self.app.post(route_path('login'),
                                      {'username': 'test_regular',
                                       'password': 'test12'})
 
             response.mustcontain('invalid user name')
             response.mustcontain('invalid password')
 
     def test_login_regular_forbidden_when_scope_restriction(self):
         from rhodecode.authentication.plugins.auth_rhodecode import RhodeCodeAuthPlugin
         with fixture.scope_restriction(self.app._pyramid_registry,
                                        RhodeCodeAuthPlugin.AUTH_RESTRICTION_SCOPE_VCS):
             response = self.app.post(route_path('login'),
                                      {'username': 'test_regular',
                                       'password': 'test12'})
 
             response.mustcontain('invalid user name')
             response.mustcontain('invalid password')
 
     def test_login_ok_came_from(self):
         test_came_from = '/_admin/users?branch=stable'
         _url = '{}?came_from={}'.format(route_path('login'), test_came_from)
         response = self.app.post(
             _url, {'username': 'test_admin', 'password': 'test12'}, status=302)
 
         assert 'branch=stable' in response.location
         response = response.follow()
 
         assert response.status == '200 OK'
         response.mustcontain('Users administration')
 
     def test_redirect_to_login_with_get_args(self):
         with fixture.anon_access(False):
             kwargs = {'branch': 'stable'}
             response = self.app.get(
                 h.route_path('repo_summary', repo_name=HG_REPO, _query=kwargs),
                 status=302)
 
             response_query = urllib.parse.parse_qsl(response.location)
             assert 'branch=stable' in response_query[0][1]
 
     def test_login_form_with_get_args(self):
         _url = '{}?came_from=/_admin/users,branch=stable'.format(route_path('login'))
         response = self.app.get(_url)
         assert 'branch%3Dstable' in response.form.action
 
     @pytest.mark.parametrize("url_came_from", [
         'data:text/html,<script>window.alert("xss")</script>',
         'mailto:[e-mail]',
         'file:///etc/passwd',
         'ftp://some.ftp.server',
         'http://other.domain',
     ], ids=no_newline_id_generator)
     def test_login_bad_came_froms(self, url_came_from):
         _url = '{}?came_from={}'.format(route_path('login'), url_came_from)
         response = self.app.post(
             _url, {'username': 'test_admin', 'password': 'test12'}, status=302)
         assert response.status == '302 Found'
         response = response.follow()
         assert response.status == '200 OK'
         assert response.request.path == '/'
 
     @pytest.mark.xfail(reason="newline params changed behaviour in python3")
     @pytest.mark.parametrize("url_came_from", [
         '/\r\nX-Forwarded-Host: \rhttp://example.org',
     ], ids=no_newline_id_generator)
     def test_login_bad_came_froms_404(self, url_came_from):
         _url = '{}?came_from={}'.format(route_path('login'), url_came_from)
         response = self.app.post(
             _url, {'username': 'test_admin', 'password': 'test12'}, status=302)
 
         response = response.follow()
         assert response.status == '404 Not Found'
 
     def test_login_short_password(self):
         response = self.app.post(route_path('login'),
                                  {'username': 'test_admin',
                                   'password': 'as'})
         assert response.status == '200 OK'
 
         response.mustcontain('Enter 3 characters or more')
 
     def test_login_wrong_non_ascii_password(self, user_regular):
         response = self.app.post(
             route_path('login'),
             {'username': user_regular.username,
              'password': 'invalid-non-asci\xe4'.encode('utf8')})
 
         response.mustcontain('invalid user name')
         response.mustcontain('invalid password')
 
     def test_login_with_non_ascii_password(self, user_util):
         password = u'valid-non-ascii\xe4'
         user = user_util.create_user(password=password)
         response = self.app.post(
             route_path('login'),
             {'username': user.username,
              'password': password})
         assert response.status_code == 302
 
     def test_login_wrong_username_password(self):
         response = self.app.post(route_path('login'),
                                  {'username': 'error',
                                   'password': 'test12'})
 
         response.mustcontain('invalid user name')
         response.mustcontain('invalid password')
 
     def test_login_admin_ok_password_migration(self, real_crypto_backend):
         from rhodecode.lib import auth
 
         # create new user, with sha256 password
         temp_user = 'test_admin_sha256'
         user = fixture.create_user(temp_user)
         user.password = auth._RhodeCodeCryptoSha256().hash_create(
             b'test123')
         Session().add(user)
         Session().commit()
         self.destroy_users.add(temp_user)
         response = self.app.post(route_path('login'),
                                  {'username': temp_user,
                                   'password': 'test123'}, status=302)
 
         response = response.follow()
         session = response.get_session_from_response()
         username = session['rhodecode_user'].get('username')
         assert username == temp_user
         response.mustcontain('logout')
 
         # new password should be bcrypted, after log-in and transfer
         user = User.get_by_username(temp_user)
         assert user.password.startswith('$')
 
     # REGISTRATIONS
     def test_register(self):
         response = self.app.get(route_path('register'))
         response.mustcontain('Create an Account')
 
     def test_register_err_same_username(self):
         uname = 'test_admin'
         response = self.app.post(
             route_path('register'),
             {
                 'username': uname,
                 'password': 'test12',
                 'password_confirmation': 'test12',
                 'email': '[e-mail]',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
 
         assertr = response.assert_response()
         msg = 'Username "%(username)s" already exists'
         msg = msg % {'username': uname}
         assertr.element_contains('#username+.error-message', msg)
 
     def test_register_err_same_email(self):
         response = self.app.post(
             route_path('register'),
             {
                 'username': 'test_admin_0',
                 'password': 'test12',
                 'password_confirmation': 'test12',
                 'email': '[e-mail]',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
 
         assertr = response.assert_response()
-        msg = u'This e-mail address is already taken'
+        msg = 'This e-mail address is already taken'
         assertr.element_contains('#email+.error-message', msg)
 
     def test_register_err_same_email_case_sensitive(self):
         response = self.app.post(
             route_path('register'),
             {
                 'username': 'test_admin_1',
                 'password': 'test12',
                 'password_confirmation': 'test12',
                 'email': '[e-mail]',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
         assertr = response.assert_response()
-        msg = u'This e-mail address is already taken'
+        msg = 'This e-mail address is already taken'
         assertr.element_contains('#email+.error-message', msg)
 
     def test_register_err_wrong_data(self):
         response = self.app.post(
             route_path('register'),
             {
                 'username': 'xs',
                 'password': 'test',
                 'password_confirmation': 'test',
                 'email': 'goodmailm',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
         assert response.status == '200 OK'
         response.mustcontain('An email address must contain a single @')
         response.mustcontain('Enter a value 6 characters long or more')
 
     def test_register_err_username(self):
         response = self.app.post(
             route_path('register'),
             {
                 'username': 'error user',
                 'password': 'test12',
                 'password_confirmation': 'test12',
                 'email': 'goodmailm',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
 
         response.mustcontain('An email address must contain a single @')
         response.mustcontain(
             'Username may only contain '
             'alphanumeric characters underscores, '
             'periods or dashes and must begin with '
             'alphanumeric character')
 
     def test_register_err_case_sensitive(self):
         usr = 'Test_Admin'
         response = self.app.post(
             route_path('register'),
             {
                 'username': usr,
                 'password': 'test12',
                 'password_confirmation': 'test12',
                 'email': 'goodmailm',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
 
         assertr = response.assert_response()
         msg = u'Username "%(username)s" already exists'
         msg = msg % {'username': usr}
         assertr.element_contains('#username+.error-message', msg)
 
     def test_register_special_chars(self):
         response = self.app.post(
             route_path('register'),
             {
                 'username': 'xxxaxn',
                 'password': 'ąćźżąśśśś',
                 'password_confirmation': 'ąćźżąśśśś',
                 'email': '[e-mail]',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
 
         msg = u'Invalid characters (non-ascii) in password'
         response.mustcontain(msg)
 
     def test_register_password_mismatch(self):
         response = self.app.post(
             route_path('register'),
             {
                 'username': 'xs',
                 'password': '123qwe',
                 'password_confirmation': 'qwe123',
                 'email': '[e-mail]',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
         msg = u'Passwords do not match'
         response.mustcontain(msg)
 
     def test_register_ok(self):
         username = 'test_regular4'
         password = 'qweqwe'
         email = '[e-mail]'
         name = 'testname'
         lastname = 'testlastname'
 
         # this initializes a session
         response = self.app.get(route_path('register'))
         response.mustcontain('Create an Account')
 
 
         response = self.app.post(
             route_path('register'),
             {
                 'username': username,
                 'password': password,
                 'password_confirmation': password,
                 'email': email,
                 'firstname': name,
                 'lastname': lastname,
                 'admin': True
             },
             status=302
         )  # This should be overridden
 
         assert_session_flash(
             response, 'You have successfully registered with RhodeCode. You can log-in now.')
 
         ret = Session().query(User).filter(
             User.username == 'test_regular4').one()
         assert ret.username == username
         assert check_password(password, ret.password)
         assert ret.email == email
         assert ret.name == name
         assert ret.lastname == lastname
         assert ret.auth_tokens is not None
         assert not ret.admin
 
     def test_forgot_password_wrong_mail(self):
         bad_email = '[e-mail]'
         # this initializes a session
         self.app.get(route_path('reset_password'))
 
         response = self.app.post(
             route_path('reset_password'), {'email': bad_email, }
         )
         assert_session_flash(response,
             'If such email exists, a password reset link was sent to it.')
 
     def test_forgot_password(self, user_util):
         # this initializes a session
         self.app.get(route_path('reset_password'))
 
         user = user_util.create_user()
         user_id = user.user_id
         email = user.email
 
         response = self.app.post(route_path('reset_password'), {'email': email, })
 
         assert_session_flash(response,
             'If such email exists, a password reset link was sent to it.')
 
         # BAD KEY
-        confirm_url = '{}?key={}'.format(route_path('reset_password_confirmation'), 'badkey')
+        confirm_url = route_path('reset_password_confirmation', params={'key': 'badkey'})
         response = self.app.get(confirm_url, status=302)
         assert response.location.endswith(route_path('reset_password'))
         assert_session_flash(response, 'Given reset token is invalid')
 
         response.follow()  # cleanup flash
 
         # GOOD KEY
         key = UserApiKeys.query()\
             .filter(UserApiKeys.user_id == user_id)\
             .filter(UserApiKeys.role == UserApiKeys.ROLE_PASSWORD_RESET)\
             .first()
 
         assert key
 
         confirm_url = '{}?key={}'.format(route_path('reset_password_confirmation'), key.api_key)
         response = self.app.get(confirm_url)
         assert response.status == '302 Found'
         assert response.location.endswith(route_path('login'))
 
         assert_session_flash(
             response,
             'Your password reset was successful, '
             'a new password has been sent to your email')
 
         response.follow()
 
     def _get_api_whitelist(self, values=None):
         config = {'api_access_controllers_whitelist': values or []}
         return config
 
     @pytest.mark.parametrize("test_name, auth_token", [
         ('none', None),
         ('empty_string', ''),
         ('fake_number', '123456'),
         ('proper_auth_token', None)
     ])
     def test_access_not_whitelisted_page_via_auth_token(
             self, test_name, auth_token, user_admin):
 
         whitelist = self._get_api_whitelist([])
         with mock.patch.dict('rhodecode.CONFIG', whitelist):
             assert [] == whitelist['api_access_controllers_whitelist']
             if test_name == 'proper_auth_token':
                 # use builtin if api_key is None
                 auth_token = user_admin.api_key
 
             with fixture.anon_access(False):
                 # webtest uses linter to check if response is bytes,
                 # and we use memoryview here as a wrapper, quick turn-off
                 self.app.lint = False
 
                 self.app.get(
                     route_path('repo_commit_raw',
                                repo_name=HG_REPO, commit_id='tip',
                                params=dict(api_key=auth_token)),
                     status=302)
 
     @pytest.mark.parametrize("test_name, auth_token, code", [
         ('none', None, 302),
         ('empty_string', '', 302),
         ('fake_number', '123456', 302),
         ('proper_auth_token', None, 200)
     ])
     def test_access_whitelisted_page_via_auth_token(
             self, test_name, auth_token, code, user_admin):
 
         whitelist = self._get_api_whitelist(whitelist_view)
 
         with mock.patch.dict('rhodecode.CONFIG', whitelist):
             assert whitelist_view == whitelist['api_access_controllers_whitelist']
 
             if test_name == 'proper_auth_token':
                 auth_token = user_admin.api_key
                 assert auth_token
 
             with fixture.anon_access(False):
                 # webtest uses linter to check if response is bytes,
                 # and we use memoryview here as a wrapper, quick turn-off
                 self.app.lint = False
                 self.app.get(
                     route_path('repo_commit_raw',
                                repo_name=HG_REPO, commit_id='tip',
                                params=dict(api_key=auth_token)),
                     status=code)
 
     @pytest.mark.parametrize("test_name, auth_token, code", [
         ('proper_auth_token', None, 200),
         ('wrong_auth_token', '123456', 302),
     ])
     def test_access_whitelisted_page_via_auth_token_bound_to_token(
             self, test_name, auth_token, code, user_admin):
 
         expected_token = auth_token
         if test_name == 'proper_auth_token':
             auth_token = user_admin.api_key
             expected_token = auth_token
             assert auth_token
 
         whitelist = self._get_api_whitelist([
             'RepoCommitsView:repo_commit_raw@{}'.format(expected_token)])
 
         with mock.patch.dict('rhodecode.CONFIG', whitelist):
 
             with fixture.anon_access(False):
                 # webtest uses linter to check if response is bytes,
                 # and we use memoryview here as a wrapper, quick turn-off
                 self.app.lint = False
 
                 self.app.get(
                     route_path('repo_commit_raw',
                                repo_name=HG_REPO, commit_id='tip',
                                params=dict(api_key=auth_token)),
                     status=code)
 
     def test_access_page_via_extra_auth_token(self):
         whitelist = self._get_api_whitelist(whitelist_view)
         with mock.patch.dict('rhodecode.CONFIG', whitelist):
             assert whitelist_view == \
                 whitelist['api_access_controllers_whitelist']
 
             new_auth_token = AuthTokenModel().create(
                 TEST_USER_ADMIN_LOGIN, 'test')
             Session().commit()
             with fixture.anon_access(False):
                 # webtest uses linter to check if response is bytes,
                 # and we use memoryview here as a wrapper, quick turn-off
                 self.app.lint = False
                 self.app.get(
                     route_path('repo_commit_raw',
                                repo_name=HG_REPO, commit_id='tip',
                                params=dict(api_key=new_auth_token.api_key)),
                     status=200)
 
     def test_access_page_via_expired_auth_token(self):
         whitelist = self._get_api_whitelist(whitelist_view)
         with mock.patch.dict('rhodecode.CONFIG', whitelist):
             assert whitelist_view == \
                 whitelist['api_access_controllers_whitelist']
 
             new_auth_token = AuthTokenModel().create(
                 TEST_USER_ADMIN_LOGIN, 'test')
             Session().commit()
             # patch the api key and make it expired
             new_auth_token.expires = 0
             Session().add(new_auth_token)
             Session().commit()
             with fixture.anon_access(False):
                 # webtest uses linter to check if response is bytes,
                 # and we use memoryview here as a wrapper, quick turn-off
                 self.app.lint = False
                 self.app.get(
                     route_path('repo_commit_raw',
                                repo_name=HG_REPO, commit_id='tip',
                                params=dict(api_key=new_auth_token.api_key)),
                     status=302)