diff --git a/.bumpversion.cfg b/.bumpversion.cfg --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,6 +1,5 @@ [bumpversion] -current_version = 4.23.2 +current_version = 4.24.0 message = release: Bump version {current_version} to {new_version} [bumpversion:file:rhodecode/VERSION] - diff --git a/.release.cfg b/.release.cfg --- a/.release.cfg +++ b/.release.cfg @@ -5,25 +5,20 @@ done = false done = true [task:rc_tools_pinned] -done = true [task:fixes_on_stable] -done = true [task:pip2nix_generated] -done = true [task:changelog_updated] -done = true [task:generate_api_docs] -done = true + +[task:updated_translation] [release] -state = prepared -version = 4.23.2 - -[task:updated_translation] +state = in_progress +version = 4.24.0 [task:generate_js_routes] diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -1,69 +1,98 @@ +.DEFAULT_GOAL := help -.PHONY: clean docs docs-clean docs-cleanup test test-clean test-only test-only-postgres test-only-mysql web-build generate-pkgs pip-packages +# set by: PATH_TO_OUTDATED_PACKAGES=/some/path/outdated_packages.py +OUTDATED_PACKAGES = ${PATH_TO_OUTDATED_PACKAGES} NODE_PATH=./node_modules WEBPACK=./node_binaries/webpack GRUNT=./node_binaries/grunt -# set by: PATH_TO_OUTDATED_PACKAGES=/some/path/outdated_packages.py -OUTDATED_PACKAGES = ${PATH_TO_OUTDATED_PACKAGES} -clean: +.PHONY: clean +clean: ## full clean make test-clean find . -type f \( -iname '*.c' -o -iname '*.pyc' -o -iname '*.so' -o -iname '*.orig' \) -exec rm '{}' ';' -test: + +.PHONY: test +test: ## run test-clean and tests make test-clean make test-only -test-clean: + +.PHONY:test-clean +test-clean: ## run test-clean and tests rm -rf coverage.xml htmlcov junit.xml pylint.log result find . -type d -name "__pycache__" -prune -exec rm -rf '{}' ';' find . -type f \( -iname '.coverage.*' \) -exec rm '{}' ';' -test-only: + +.PHONY: test-only +test-only: ## run tests PYTHONHASHSEED=random \ - py.test -x -vv -r xw -p no:sugar --cov=rhodecode \ - --cov-report=term-missing --cov-report=html \ + py.test -x -vv -r xw -p no:sugar \ + --cov=rhodecode --cov-report=term-missing --cov-report=html \ rhodecode -test-only-mysql: + +.PHONY: test-only-mysql +test-only-mysql: ## run tests against mysql PYTHONHASHSEED=random \ - py.test -x -vv -r xw -p no:sugar --cov=rhodecode \ - --cov-report=term-missing --cov-report=html \ + py.test -x -vv -r xw -p no:sugar \ + --cov=rhodecode --cov-report=term-missing --cov-report=html \ --ini-config-override='{"app:main": {"sqlalchemy.db1.url": "mysql://root:qweqwe@localhost/rhodecode_test?charset=utf8"}}' \ rhodecode -test-only-postgres: + +.PHONY: test-only-postgres +test-only-postgres: ## run tests against postgres PYTHONHASHSEED=random \ - py.test -x -vv -r xw -p no:sugar --cov=rhodecode \ - --cov-report=term-missing --cov-report=html \ + py.test -x -vv -r xw -p no:sugar \ + --cov=rhodecode --cov-report=term-missing --cov-report=html \ --ini-config-override='{"app:main": {"sqlalchemy.db1.url": "postgresql://postgres:qweqwe@localhost/rhodecode_test"}}' \ rhodecode - -docs: +.PHONY: docs +docs: ## build docs (cd docs; nix-build default.nix -o result; make clean html) -docs-clean: + +.PHONY: docs-clean +docs-clean: ## Cleanup docs (cd docs; make clean) -docs-cleanup: + +.PHONY: docs-cleanup +docs-cleanup: ## Cleanup docs (cd docs; make cleanup) -web-build: + +.PHONY: web-build +web-build: ## Build static/js NODE_PATH=$(NODE_PATH) $(GRUNT) -generate-pkgs: + +.PHONY: generate-pkgs +generate-pkgs: ## generate new python packages nix-shell pkgs/shell-generate.nix --command "pip2nix generate --licenses" -pip-packages: + +.PHONY: pip-packages +pip-packages: ## show outdated packages python ${OUTDATED_PACKAGES} -generate-js-pkgs: + +.PHONY: generate-js-pkgs +generate-js-pkgs: ## generate js packages rm -rf node_modules && \ nix-shell pkgs/shell-generate.nix --command "node2nix --input package.json -o pkgs/node-packages.nix -e pkgs/node-env.nix -c pkgs/node-default.nix -d --flatten --nodejs-8" && \ sed -i -e 's/http:\/\//https:\/\//g' pkgs/node-packages.nix -generate-license-meta: + +.PHONY: generate-license-meta +generate-license-meta: ## Generate license metadata nix-build pkgs/license-generate.nix -o result-license && \ - cat result-license/licenses.json | python -m json.tool > rhodecode/config/licenses.json \ No newline at end of file + cat result-license/licenses.json | python -m json.tool > rhodecode/config/licenses.json + +.PHONY: help +help: + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-24s\033[0m %s\n", $$1, $$2}' diff --git a/docs/release-notes/release-notes-4.24.0.rst b/docs/release-notes/release-notes-4.24.0.rst new file mode 100644 --- /dev/null +++ b/docs/release-notes/release-notes-4.24.0.rst @@ -0,0 +1,54 @@ +|RCE| 4.23.0 |RNS| +------------------ + +Release Date +^^^^^^^^^^^^ + +- 2021-01-10 + + +New Features +^^^^^^^^^^^^ + +- Artifacts: expose additional headers, and content-disposition for downloads from artifacts exposing the real name of the file. +- Token access: allow token in headers not only in GET/URL. +- File-store: added a stream upload endpoint, it allows to upload GBs of data into artifact store efficiently. + Can be used for backups etc. +- Pull requests: expose commit versions in the pull-request commit list. + +General +^^^^^^^ + +- Deps: bumped redis to 3.5.3 +- rcextensions: improve examples +- Setup: added optional parameters to apply a default license, or skip re-creation of database at install. +- Docs: update headers for NGINX +- Beaker cache: remove no longer used beaker cache init + + +Security +^^^^^^^^ + + + +Performance +^^^^^^^^^^^ + +- Core: speed up cache loading on application startup. +- Core: allow loading all auth plugins in once place for CE/EE code. +- Application: not use config.scan(), and replace all @add_view decorator into a explicit add_view call for faster app start. + + +Fixes +^^^^^ + +- Svn: don't print exceptions in case of safe calls +- Vcsserver: use safer maxfd reporting, some linux systems get a problem with this +- Hooks-daemon: fixed problem with lost hooks value from .ini file. +- Exceptions: fixed truncated exception text + + +Upgrade notes +^^^^^^^^^^^^^ + +- Scheduled release 4.24.0 diff --git a/docs/release-notes/release-notes.rst b/docs/release-notes/release-notes.rst --- a/docs/release-notes/release-notes.rst +++ b/docs/release-notes/release-notes.rst @@ -9,6 +9,7 @@ Release Notes .. toctree:: :maxdepth: 1 + release-notes-4.24.0.rst release-notes-4.23.2.rst release-notes-4.23.1.rst release-notes-4.23.0.rst diff --git a/pkgs/python-packages.nix b/pkgs/python-packages.nix --- a/pkgs/python-packages.nix +++ b/pkgs/python-packages.nix @@ -1828,11 +1828,11 @@ self: super: { }; }; "redis" = super.buildPythonPackage { - name = "redis-3.4.1"; + name = "redis-3.5.3"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/ef/2e/2c0f59891db7db087a7eeaa79bc7c7f2c039e71a2b5b0a41391e9d462926/redis-3.4.1.tar.gz"; - sha256 = "07yaj0j9fs7xdkg5bg926fa990khyigjbp31si8ai20vj8sv7kqd"; + url = "https://files.pythonhosted.org/packages/b3/17/1e567ff78c83854e16b98694411fe6e08c3426af866ad11397cddceb80d3/redis-3.5.3.tar.gz"; + sha256 = "0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -1883,7 +1883,7 @@ self: super: { }; }; "rhodecode-enterprise-ce" = super.buildPythonPackage { - name = "rhodecode-enterprise-ce-4.23.2"; + name = "rhodecode-enterprise-ce-4.24.0"; buildInputs = [ self."pytest" self."py" diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -57,7 +57,7 @@ tzlocal==1.5.1 pyzmq==14.6.0 py-gfm==0.1.4 regex==2020.9.27 -redis==3.4.1 +redis==3.5.3 repoze.lru==0.7 requests==2.22.0 routes==2.4.1 diff --git a/rhodecode/VERSION b/rhodecode/VERSION --- a/rhodecode/VERSION +++ b/rhodecode/VERSION @@ -1,1 +1,1 @@ -4.23.2 \ No newline at end of file +4.24.0 \ No newline at end of file diff --git a/rhodecode/apps/my_account/tests/test_my_account_auth_tokens.py b/rhodecode/apps/my_account/tests/test_my_account_auth_tokens.py --- a/rhodecode/apps/my_account/tests/test_my_account_auth_tokens.py +++ b/rhodecode/apps/my_account/tests/test_my_account_auth_tokens.py @@ -23,23 +23,23 @@ import pytest from rhodecode.apps._base import ADMIN_PREFIX from rhodecode.model.db import User from rhodecode.tests import ( - TestController, TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS, - TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS, assert_session_flash) + TestController, route_path_generator, assert_session_flash) from rhodecode.tests.fixture import Fixture from rhodecode.tests.utils import AssertResponse fixture = Fixture() -def route_path(name, **kwargs): - return { +def route_path(name, params=None, **kwargs): + url_defs = { 'my_account_auth_tokens': ADMIN_PREFIX + '/my_account/auth_tokens', 'my_account_auth_tokens_add': ADMIN_PREFIX + '/my_account/auth_tokens/new', 'my_account_auth_tokens_delete': ADMIN_PREFIX + '/my_account/auth_tokens/delete', - }[name].format(**kwargs) + } + return route_path_generator(url_defs, name=name, params=params, **kwargs) class TestMyAccountAuthTokens(TestController): diff --git a/rhodecode/apps/repository/views/repo_pull_requests.py b/rhodecode/apps/repository/views/repo_pull_requests.py --- a/rhodecode/apps/repository/views/repo_pull_requests.py +++ b/rhodecode/apps/repository/views/repo_pull_requests.py @@ -357,6 +357,9 @@ class RepoPullRequestsView(RepoAppView, pull_request_id=pull_request_id)) versions = pull_request_display_obj.versions() + + c.commit_versions = PullRequestModel().pr_commits_versions(versions) + # used to store per-commit range diffs c.changes = collections.OrderedDict() diff --git a/rhodecode/apps/search/tests/test_search.py b/rhodecode/apps/search/tests/test_search.py --- a/rhodecode/apps/search/tests/test_search.py +++ b/rhodecode/apps/search/tests/test_search.py @@ -25,20 +25,20 @@ import pytest from whoosh import query from rhodecode.tests import ( - TestController, HG_REPO, + TestController, route_path_generator, HG_REPO, TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS) from rhodecode.tests.utils import AssertResponse -def route_path(name, **kwargs): +def route_path(name, params=None, **kwargs): from rhodecode.apps._base import ADMIN_PREFIX - return { + url_defs = { 'search': ADMIN_PREFIX + '/search', 'search_repo': '/{repo_name}/search', - - }[name].format(**kwargs) + } + return route_path_generator(url_defs, name=name, params=params, **kwargs) class TestSearchController(TestController): diff --git a/rhodecode/config/middleware.py b/rhodecode/config/middleware.py --- a/rhodecode/config/middleware.py +++ b/rhodecode/config/middleware.py @@ -47,7 +47,6 @@ from rhodecode.lib.vcs import VCSCommuni from rhodecode.lib.exceptions import VCSServerUnavailable from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled from rhodecode.lib.middleware.https_fixup import HttpsFixup -from rhodecode.lib.celerylib.loader import configure_celery from rhodecode.lib.plugins.utils import register_rhodecode_plugin from rhodecode.lib.utils2 import aslist as rhodecode_aslist, AttributeDict from rhodecode.lib.exc_tracking import store_exception @@ -237,6 +236,7 @@ def includeme_first(config): def includeme(config, auth_resources=None): + from rhodecode.lib.celerylib.loader import configure_celery log.debug('Initializing main includeme from %s', os.path.basename(__file__)) settings = config.registry.settings config.set_request_factory(Request) diff --git a/rhodecode/lib/db_manage.py b/rhodecode/lib/db_manage.py --- a/rhodecode/lib/db_manage.py +++ b/rhodecode/lib/db_manage.py @@ -66,11 +66,23 @@ class DbManage(object): self.root = root self.dburi = dbconf self.log_sql = log_sql - self.db_exists = False self.cli_args = cli_args or {} self.init_db(SESSION=SESSION) self.ask_ok = self.get_ask_ok_func(self.cli_args.get('force_ask')) + def db_exists(self): + if not self.sa: + self.init_db() + try: + self.sa.query(RhodeCodeUi)\ + .filter(RhodeCodeUi.ui_key == '/')\ + .scalar() + return True + except Exception: + return False + finally: + self.sa.rollback() + def get_ask_ok_func(self, param): if param not in [None]: # return a function lambda that has a default set to param diff --git a/rhodecode/lib/hooks_daemon.py b/rhodecode/lib/hooks_daemon.py --- a/rhodecode/lib/hooks_daemon.py +++ b/rhodecode/lib/hooks_daemon.py @@ -120,7 +120,7 @@ class ThreadedHookCallbackDaemon(object) _done = False def __init__(self, txn_id=None, host=None, port=None): - self._prepare(txn_id=txn_id, host=None, port=port) + self._prepare(txn_id=txn_id, host=host, port=port) def __enter__(self): log.debug('Running `%s` callback daemon', self.__class__.__name__) diff --git a/rhodecode/lib/rc_commands/setup_rc.py b/rhodecode/lib/rc_commands/setup_rc.py --- a/rhodecode/lib/rc_commands/setup_rc.py +++ b/rhodecode/lib/rc_commands/setup_rc.py @@ -60,14 +60,24 @@ log = logging.getLogger(__name__) default=None, help='Enable public access on this installation. ' 'Default is public access enabled.') +@click.option( + '--skip-existing-db', + default=False, + is_flag=True, + help='Do not destroy and re-initialize the database if it already exist.') +@click.option( + '--apply-license-key', + default=False, + is_flag=True, + help='Get the license key from a license file or ENV and apply during DB creation.') def main(ini_path, force_yes, user, email, password, api_key, repos, - public_access): + public_access, skip_existing_db, apply_license_key): return command(ini_path, force_yes, user, email, password, api_key, - repos, public_access) + repos, public_access, skip_existing_db, apply_license_key) def command(ini_path, force_yes, user, email, password, api_key, repos, - public_access): + public_access, skip_existing_db, apply_license_key): # mapping of old parameters to new CLI from click options = dict( username=user, @@ -85,6 +95,9 @@ def command(ini_path, force_yes, user, e db_uri = config['sqlalchemy.db1.url'] dbmanage = DbManage(log_sql=True, dbconf=db_uri, root='.', tests=False, cli_args=options) + if skip_existing_db and dbmanage.db_exists(): + return + dbmanage.create_tables(override=True) dbmanage.set_db_version() opts = dbmanage.config_prompt(None) @@ -93,6 +106,13 @@ def command(ini_path, force_yes, user, e dbmanage.create_admin_and_prompt() dbmanage.create_permissions() dbmanage.populate_default_permissions() + if apply_license_key: + try: + from rc_license.models import apply_trial_license_if_missing + apply_trial_license_if_missing(force=True) + except ImportError: + pass + Session().commit() with bootstrap(ini_path, env={'RC_CMD_SETUP_RC': '1'}) as env: diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py --- a/rhodecode/model/db.py +++ b/rhodecode/model/db.py @@ -3821,8 +3821,12 @@ class ChangesetComment(Base, BaseModel): return q.all() @classmethod - def get_index_from_version(cls, pr_version, versions): - num_versions = [x.pull_request_version_id for x in versions] + def get_index_from_version(cls, pr_version, versions=None, num_versions=None): + + if versions is not None: + num_versions = [x.pull_request_version_id for x in versions] + + num_versions = num_versions or [] try: return num_versions.index(pr_version) + 1 except (IndexError, ValueError): diff --git a/rhodecode/model/pull_request.py b/rhodecode/model/pull_request.py --- a/rhodecode/model/pull_request.py +++ b/rhodecode/model/pull_request.py @@ -610,6 +610,20 @@ class PullRequestModel(BaseModel): return _org_pull_request_obj, pull_request_obj, \ pull_request_display_obj, at_version + def pr_commits_versions(self, versions): + """ + Maps the pull-request commits into all known PR versions. This way we can obtain + each pr version the commit was introduced in. + """ + commit_versions = collections.defaultdict(list) + num_versions = [x.pull_request_version_id for x in versions] + for ver in versions: + for commit_id in ver.revisions: + ver_idx = ChangesetComment.get_index_from_version( + ver.pull_request_version_id, num_versions=num_versions) + commit_versions[commit_id].append(ver_idx) + return commit_versions + def create(self, created_by, source_repo, source_ref, target_repo, target_ref, revisions, reviewers, observers, title, description=None, common_ancestor_id=None, diff --git a/rhodecode/templates/compare/compare_commits.mako b/rhodecode/templates/compare/compare_commits.mako --- a/rhodecode/templates/compare/compare_commits.mako +++ b/rhodecode/templates/compare/compare_commits.mako @@ -12,6 +12,9 @@ + % if hasattr(c, 'commit_versions'): + + % endif @@ -30,6 +33,11 @@ class="compare_select" style="${'display: none' if c.collapse_all_commits else ''}" > + % if hasattr(c, 'commit_versions'): + + % endif diff --git a/rhodecode/tests/__init__.py b/rhodecode/tests/__init__.py --- a/rhodecode/tests/__init__.py +++ b/rhodecode/tests/__init__.py @@ -33,15 +33,15 @@ import pytest from rhodecode.model.db import User from rhodecode.lib import auth from rhodecode.lib import helpers as h -from rhodecode.lib.helpers import flash, link_to +from rhodecode.lib.helpers import flash from rhodecode.lib.utils2 import safe_str log = logging.getLogger(__name__) __all__ = [ - 'get_new_dir', 'TestController', - 'link_to', 'clear_cache_regions', + 'get_new_dir', 'TestController', 'route_path_generator', + 'clear_cache_regions', 'assert_session_flash', 'login_user', 'no_newline_id_generator', 'TESTS_TMP_PATH', 'HG_REPO', 'GIT_REPO', 'SVN_REPO', 'NEW_HG_REPO', 'NEW_GIT_REPO', @@ -243,3 +243,13 @@ def no_newline_id_generator(test_name): .replace(' ', '_S') return test_name or 'test-with-empty-name' + + +def route_path_generator(url_defs, name, params=None, **kwargs): + import urllib + + base_url = url_defs[name].format(**kwargs) + + if params: + base_url = '{}?{}'.format(base_url, urllib.urlencode(params)) + return base_url
ver${_('Time')} ${_('Author')} ${_('Commit')} + ${('v{}'.format(c.commit_versions[commit.raw_id][0]) if c.commit_versions[commit.raw_id] else 'latest')} + ${h.age_component(commit.date)}