release: Merge default into stable for release preparation

docs/release-notes/release-notes-4.24.0.rst

@@ -0,0 +1,54 b''
+|RCE| 4.23.0 |RNS|
+------------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2021-01-10
+
+
+New Features
+^^^^^^^^^^^^
+
+- Artifacts: expose additional headers, and content-disposition for downloads from artifacts exposing the real name of the file.
+- Token access: allow token in headers not only in GET/URL.
+- File-store: added a stream upload endpoint, it allows to upload GBs of data into artifact store efficiently.
+  Can be used for backups etc.
+- Pull requests: expose commit versions in the pull-request commit list.
+
+General
+^^^^^^^
+
+- Deps: bumped redis to 3.5.3
+- rcextensions: improve examples
+- Setup: added optional parameters to apply a default license, or skip re-creation of database at install.
+- Docs: update headers for NGINX
+- Beaker cache: remove no longer used beaker cache init
+
+
+Security
+^^^^^^^^
+
+
+
+Performance
+^^^^^^^^^^^
+
+- Core: speed up cache loading on application startup.
+- Core: allow loading all auth plugins in once place for CE/EE code.
+- Application: not use config.scan(), and replace all @add_view decorator into a explicit add_view call for faster app start.
+
+
+Fixes
+^^^^^
+
+- Svn: don't print exceptions in case of safe calls
+- Vcsserver: use safer maxfd reporting, some linux systems get a problem with this
+- Hooks-daemon: fixed problem with lost hooks value from .ini file.
+- Exceptions: fixed truncated exception text
+
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- Scheduled release 4.24.0

.bumpversion.cfg

@@ -1,6 +1,5 b''
 [bumpversion]
-current_version = 4.23.2
+current_version = 4.24.0
 message = release: Bump version {current_version} to {new_version}
 
 [bumpversion:file:rhodecode/VERSION]
-

.release.cfg

@@ -5,25 +5,20 b' done = false'
 done = true
 
 [task:rc_tools_pinned]
-done = true
 
 [task:fixes_on_stable]
-done = true
 
 [task:pip2nix_generated]
-done = true
 
 [task:changelog_updated]
-done = true
 
 [task:generate_api_docs]
-done = true
+
+[task:updated_translation]
 
 [release]
-state = prepared
-version = 4.23.2
-
-[task:updated_translation]
+state = in_progress
+version = 4.24.0
 
 [task:generate_js_routes]
 

Makefile

@@ -1,69 +1,98 b''
+.DEFAULT_GOAL := help
 
-.PHONY: clean docs docs-clean docs-cleanup test test-clean test-only test-only-postgres test-only-mysql web-build generate-pkgs pip-packages
+# set by: PATH_TO_OUTDATED_PACKAGES=/some/path/outdated_packages.py
+OUTDATED_PACKAGES = ${PATH_TO_OUTDATED_PACKAGES}
 
 NODE_PATH=./node_modules
 WEBPACK=./node_binaries/webpack
 GRUNT=./node_binaries/grunt
-# set by: PATH_TO_OUTDATED_PACKAGES=/some/path/outdated_packages.py
-OUTDATED_PACKAGES = ${PATH_TO_OUTDATED_PACKAGES}
 
-clean:
+.PHONY: clean
+clean: ## full clean
 	make test-clean
 	find . -type f \( -iname '*.c' -o -iname '*.pyc' -o -iname '*.so' -o -iname '*.orig' \) -exec rm '{}' ';'
 
-test:
+
+.PHONY: test
+test: ## run test-clean and tests
 	make test-clean
 	make test-only
 
-test-clean:
+
+.PHONY:test-clean
+test-clean:  ## run test-clean and tests
 	rm -rf coverage.xml htmlcov junit.xml pylint.log result
 	find . -type d -name "__pycache__" -prune -exec rm -rf '{}' ';'
 	find . -type f \( -iname '.coverage.*' \) -exec rm '{}' ';'
 
-test-only:
+
+.PHONY: test-only
+test-only: ## run tests
 	PYTHONHASHSEED=random \
-	py.test -x -vv -r xw -p no:sugar --cov=rhodecode \
-    --cov-report=term-missing --cov-report=html \
+	py.test -x -vv -r xw -p no:sugar \
+	--cov=rhodecode --cov-report=term-missing --cov-report=html \
     rhodecode
 
-test-only-mysql:
+
+.PHONY: test-only-mysql
+test-only-mysql: ## run tests against mysql
 	PYTHONHASHSEED=random \
-	py.test -x -vv -r xw -p no:sugar --cov=rhodecode \
-    --cov-report=term-missing --cov-report=html \
+	py.test -x -vv -r xw -p no:sugar \
+	--cov=rhodecode --cov-report=term-missing --cov-report=html \
     --ini-config-override='{"app:main": {"sqlalchemy.db1.url": "mysql://root:qweqwe@localhost/rhodecode_test?charset=utf8"}}' \
     rhodecode
 
-test-only-postgres:
+
+.PHONY: test-only-postgres
+test-only-postgres: ## run tests against postgres
 	PYTHONHASHSEED=random \
-	py.test -x -vv -r xw -p no:sugar --cov=rhodecode \
-    --cov-report=term-missing --cov-report=html \
+	py.test -x -vv -r xw -p no:sugar \
+	--cov=rhodecode --cov-report=term-missing --cov-report=html \
     --ini-config-override='{"app:main": {"sqlalchemy.db1.url": "postgresql://postgres:qweqwe@localhost/rhodecode_test"}}' \
     rhodecode
 
-
-docs:
+.PHONY: docs
+docs: ## build docs
 	(cd docs; nix-build default.nix -o result; make clean html)
 
-docs-clean:
+
+.PHONY: docs-clean
+docs-clean: ## Cleanup docs
 	(cd docs; make clean)
 
-docs-cleanup:
+
+.PHONY: docs-cleanup
+docs-cleanup: ## Cleanup docs
 	(cd docs; make cleanup)
 
-web-build:
+
+.PHONY: web-build
+web-build: ## Build static/js
 	NODE_PATH=$(NODE_PATH) $(GRUNT)
 
-generate-pkgs:
+
+.PHONY: generate-pkgs
+generate-pkgs: ## generate new python packages
 	nix-shell pkgs/shell-generate.nix --command "pip2nix generate --licenses"
 
-pip-packages:
+
+.PHONY: pip-packages
+pip-packages: ## show outdated packages
 	python ${OUTDATED_PACKAGES}
 
-generate-js-pkgs:
+
+.PHONY: generate-js-pkgs
+generate-js-pkgs: ## generate js packages
 	rm -rf node_modules && \
 	nix-shell pkgs/shell-generate.nix --command "node2nix --input package.json -o pkgs/node-packages.nix -e pkgs/node-env.nix -c pkgs/node-default.nix -d --flatten --nodejs-8" && \
 	sed -i -e 's/http:\/\//https:\/\//g' pkgs/node-packages.nix
 
-generate-license-meta:
+
+.PHONY: generate-license-meta
+generate-license-meta: ## Generate license metadata
 	nix-build pkgs/license-generate.nix -o result-license && \
-	cat result-license/licenses.json | python -m json.tool > rhodecode/config/licenses.json
  No newline at end of file
+	cat result-license/licenses.json | python -m json.tool > rhodecode/config/licenses.json
+
+.PHONY: help
+help:
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-24s\033[0m %s\n", $$1, $$2}'

docs/release-notes/release-notes.rst

@@ -9,6 +9,7 b' Release Notes'
 .. toctree::
    :maxdepth: 1
 
+   release-notes-4.24.0.rst
    release-notes-4.23.2.rst
    release-notes-4.23.1.rst
    release-notes-4.23.0.rst

pkgs/python-packages.nix

@@ -1828,11 +1828,11 b' self: super: {'
     };
   };
   "redis" = super.buildPythonPackage {
-    name = "redis-3.4.1";
+    name = "redis-3.5.3";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/ef/2e/2c0f59891db7db087a7eeaa79bc7c7f2c039e71a2b5b0a41391e9d462926/redis-3.4.1.tar.gz";
-      sha256 = "07yaj0j9fs7xdkg5bg926fa990khyigjbp31si8ai20vj8sv7kqd";
+      url = "https://files.pythonhosted.org/packages/b3/17/1e567ff78c83854e16b98694411fe6e08c3426af866ad11397cddceb80d3/redis-3.5.3.tar.gz";
+      sha256 = "0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
@@ -1883,7 +1883,7 b' self: super: {'
     };
   };
   "rhodecode-enterprise-ce" = super.buildPythonPackage {
-    name = "rhodecode-enterprise-ce-4.23.2";
+    name = "rhodecode-enterprise-ce-4.24.0";
     buildInputs = [
       self."pytest"
       self."py"

requirements.txt

@@ -57,7 +57,7 b' tzlocal==1.5.1'
 pyzmq==14.6.0
 py-gfm==0.1.4
 regex==2020.9.27
-redis==3.4.1
+redis==3.5.3
 repoze.lru==0.7
 requests==2.22.0
 routes==2.4.1

rhodecode/VERSION

@@ -1,1 +1,1 b''
-4.23.2
  No newline at end of file
+4.24.0
  No newline at end of file

rhodecode/apps/my_account/tests/test_my_account_auth_tokens.py

@@ -23,23 +23,23 b' import pytest'
 from rhodecode.apps._base import ADMIN_PREFIX
 from rhodecode.model.db import User
 from rhodecode.tests import (
-    TestController, TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS,
-    TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS, assert_session_flash)
+    TestController, route_path_generator, assert_session_flash)
 from rhodecode.tests.fixture import Fixture
 from rhodecode.tests.utils import AssertResponse
 
 fixture = Fixture()
 
 
-def route_path(name, **kwargs):
-    return {
+def route_path(name, params=None, **kwargs):
+    url_defs = {
         'my_account_auth_tokens':
             ADMIN_PREFIX + '/my_account/auth_tokens',
         'my_account_auth_tokens_add':
             ADMIN_PREFIX + '/my_account/auth_tokens/new',
         'my_account_auth_tokens_delete':
             ADMIN_PREFIX + '/my_account/auth_tokens/delete',
-    }[name].format(**kwargs)
+    }
+    return route_path_generator(url_defs, name=name, params=params, **kwargs)
 
 
 class TestMyAccountAuthTokens(TestController):

rhodecode/apps/repository/views/repo_pull_requests.py

@@ -357,6 +357,9 b' class RepoPullRequestsView(RepoAppView, '
                 pull_request_id=pull_request_id))
 
         versions = pull_request_display_obj.versions()
+
+        c.commit_versions = PullRequestModel().pr_commits_versions(versions)
+
         # used to store per-commit range diffs
         c.changes = collections.OrderedDict()
 

rhodecode/apps/search/tests/test_search.py

@@ -25,20 +25,20 b' import pytest'
 from whoosh import query
 
 from rhodecode.tests import (
-    TestController, HG_REPO,
+    TestController, route_path_generator, HG_REPO,
     TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS)
 from rhodecode.tests.utils import AssertResponse
 
 
-def route_path(name, **kwargs):
+def route_path(name, params=None, **kwargs):
     from rhodecode.apps._base import ADMIN_PREFIX
-    return {
+    url_defs = {
         'search':
             ADMIN_PREFIX + '/search',
         'search_repo':
             '/{repo_name}/search',
-
-    }[name].format(**kwargs)
+    }
+    return route_path_generator(url_defs, name=name, params=params, **kwargs)
 
 
 class TestSearchController(TestController):

rhodecode/config/middleware.py

@@ -47,7 +47,6 b' from rhodecode.lib.vcs import VCSCommuni'
 from rhodecode.lib.exceptions import VCSServerUnavailable
 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
 from rhodecode.lib.middleware.https_fixup import HttpsFixup
-from rhodecode.lib.celerylib.loader import configure_celery
 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
 from rhodecode.lib.utils2 import aslist as rhodecode_aslist, AttributeDict
 from rhodecode.lib.exc_tracking import store_exception
@@ -237,6 +236,7 b' def includeme_first(config):'
 
 
 def includeme(config, auth_resources=None):
+    from rhodecode.lib.celerylib.loader import configure_celery
     log.debug('Initializing main includeme from %s', os.path.basename(__file__))
     settings = config.registry.settings
     config.set_request_factory(Request)

rhodecode/lib/db_manage.py

@@ -66,11 +66,23 b' class DbManage(object):'
         self.root = root
         self.dburi = dbconf
         self.log_sql = log_sql
-        self.db_exists = False
         self.cli_args = cli_args or {}
         self.init_db(SESSION=SESSION)
         self.ask_ok = self.get_ask_ok_func(self.cli_args.get('force_ask'))
 
+    def db_exists(self):
+        if not self.sa:
+            self.init_db()
+        try:
+            self.sa.query(RhodeCodeUi)\
+                .filter(RhodeCodeUi.ui_key == '/')\
+                .scalar()
+            return True
+        except Exception:
+            return False
+        finally:
+            self.sa.rollback()
+
     def get_ask_ok_func(self, param):
         if param not in [None]:
             # return a function lambda that has a default set to param

rhodecode/lib/hooks_daemon.py

@@ -120,7 +120,7 b' class ThreadedHookCallbackDaemon(object)'
     _done = False
 
     def __init__(self, txn_id=None, host=None, port=None):
-        self._prepare(txn_id=txn_id, host=None, port=port)
+        self._prepare(txn_id=txn_id, host=host, port=port)
 
     def __enter__(self):
         log.debug('Running `%s` callback daemon', self.__class__.__name__)

rhodecode/lib/rc_commands/setup_rc.py

@@ -60,14 +60,24 b' log = logging.getLogger(__name__)'
     default=None,
     help='Enable public access on this installation. '
          'Default is public access enabled.')
[e-mail](
+    '--skip-existing-db',
+    default=False,
+    is_flag=True,
+    help='Do not destroy and re-initialize the database if it already exist.')
[e-mail](
+    '--apply-license-key',
+    default=False,
+    is_flag=True,
+    help='Get the license key from a license file or ENV and apply during DB creation.')
 def main(ini_path, force_yes, user, email, password, api_key, repos,
-         public_access):
+         public_access, skip_existing_db, apply_license_key):
     return command(ini_path, force_yes, user, email, password, api_key,
-                   repos, public_access)
+                   repos, public_access, skip_existing_db, apply_license_key)
 
 
 def command(ini_path, force_yes, user, email, password, api_key, repos,
-            public_access):
+            public_access, skip_existing_db, apply_license_key):
     # mapping of old parameters to new CLI from click
     options = dict(
         username=user,
@@ -85,6 +95,9 b' def command(ini_path, force_yes, user, e'
     db_uri = config['sqlalchemy.db1.url']
     dbmanage = DbManage(log_sql=True, dbconf=db_uri, root='.',
                         tests=False, cli_args=options)
+    if skip_existing_db and dbmanage.db_exists():
+        return
+
     dbmanage.create_tables(override=True)
     dbmanage.set_db_version()
     opts = dbmanage.config_prompt(None)
@@ -93,6 +106,13 b' def command(ini_path, force_yes, user, e'
     dbmanage.create_admin_and_prompt()
     dbmanage.create_permissions()
     dbmanage.populate_default_permissions()
+    if apply_license_key:
+        try:
+            from rc_license.models import apply_trial_license_if_missing
+            apply_trial_license_if_missing(force=True)
+        except ImportError:
+            pass
+
     Session().commit()
 
     with bootstrap(ini_path, env={'RC_CMD_SETUP_RC': '1'}) as env:

rhodecode/model/db.py

@@ -3821,8 +3821,12 b' class ChangesetComment(Base, BaseModel):'
         return q.all()
 
     @classmethod
-    def get_index_from_version(cls, pr_version, versions):
-        num_versions = [x.pull_request_version_id for x in versions]
+    def get_index_from_version(cls, pr_version, versions=None, num_versions=None):
+
+        if versions is not None:
+            num_versions = [x.pull_request_version_id for x in versions]
+
+        num_versions = num_versions or []
         try:
             return num_versions.index(pr_version) + 1
         except (IndexError, ValueError):

rhodecode/model/pull_request.py

@@ -610,6 +610,20 b' class PullRequestModel(BaseModel):'
         return _org_pull_request_obj, pull_request_obj, \
                pull_request_display_obj, at_version
 
+    def pr_commits_versions(self, versions):
+        """
+        Maps the pull-request commits into all known PR versions. This way we can obtain
+        each pr version the commit was introduced in.
+        """
+        commit_versions = collections.defaultdict(list)
+        num_versions = [x.pull_request_version_id for x in versions]
+        for ver in versions:
+            for commit_id in ver.revisions:
+                ver_idx = ChangesetComment.get_index_from_version(
+                    ver.pull_request_version_id, num_versions=num_versions)
+                commit_versions[commit_id].append(ver_idx)
+        return commit_versions
+
     def create(self, created_by, source_repo, source_ref, target_repo,
                target_ref, revisions, reviewers, observers, title, description=None,
                common_ancestor_id=None,

rhodecode/templates/compare/compare_commits.mako

@@ -12,6 +12,9 b''
     <input type="hidden" name="__start__" value="revisions:sequence">
     <table class="rctable compare_view_commits">
         <tr>
+            % if hasattr(c, 'commit_versions'):
+                <th>ver</th>
+            % endif
             <th>${_('Time')}</th>
             <th>${_('Author')}</th>
             <th>${_('Commit')}</th>
@@ -30,6 +33,11 b''
             class="compare_select"
             style="${'display: none' if c.collapse_all_commits else ''}"
         >
+            % if hasattr(c, 'commit_versions'):
+                <td class="tooltip" title="${_('Pull request version this commit was introduced')}">
+                    <code>${('v{}'.format(c.commit_versions[commit.raw_id][0]) if c.commit_versions[commit.raw_id] else 'latest')}</code>
+                </td>
+            % endif
             <td class="td-time">
                 ${h.age_component(commit.date)}
             </td>

rhodecode/tests/__init__.py

@@ -33,15 +33,15 b' import pytest'
 from rhodecode.model.db import User
 from rhodecode.lib import auth
 from rhodecode.lib import helpers as h
-from rhodecode.lib.helpers import flash, link_to
+from rhodecode.lib.helpers import flash
 from rhodecode.lib.utils2 import safe_str
 
 
 log = logging.getLogger(__name__)
 
 __all__ = [
-    'get_new_dir', 'TestController',
-    'link_to', 'clear_cache_regions',
+    'get_new_dir', 'TestController', 'route_path_generator',
+    'clear_cache_regions',
     'assert_session_flash', 'login_user', 'no_newline_id_generator',
     'TESTS_TMP_PATH', 'HG_REPO', 'GIT_REPO', 'SVN_REPO',
     'NEW_HG_REPO', 'NEW_GIT_REPO',
@@ -243,3 +243,13 b' def no_newline_id_generator(test_name):'
         .replace(' ', '_S')
 
     return test_name or 'test-with-empty-name'
+
+
+def route_path_generator(url_defs, name, params=None, **kwargs):
+    import urllib
+
+    base_url = url_defs[name].format(**kwargs)
+
+    if params:
+        base_url = '{}?{}'.format(base_url, urllib.urlencode(params))
+    return base_url