# HG changeset patch # User RhodeCode Admin # Date 2023-03-06 11:58:49 # Node ID 98004a1b025952768caebaa01ba7940ab86b4567 # Parent cae6657c8c238ab2747970942933f59d19c93e77 # Parent 6fb7c99d387360005b1351b066ac553f4332c780 merge with default diff --git a/MANIFEST.in b/MANIFEST.in --- a/MANIFEST.in +++ b/MANIFEST.in @@ -48,5 +48,6 @@ recursive-include rhodecode/public/js * recursive-include rhodecode/templates * # skip any tests files +recursive-exclude result * recursive-exclude rhodecode/tests * diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -1,4 +1,3 @@ -.DEFAULT_GOAL := help # set by: PATH_TO_OUTDATED_PACKAGES=/some/path/outdated_packages.py OUTDATED_PACKAGES = ${PATH_TO_OUTDATED_PACKAGES} @@ -8,91 +7,129 @@ WEBPACK=./node_binaries/webpack GRUNT=./node_binaries/grunt .PHONY: clean -clean: ## full clean +## Cleanup compiled and cache py files +clean: make test-clean find . -type f \( -iname '*.c' -o -iname '*.pyc' -o -iname '*.so' -o -iname '*.orig' \) -exec rm '{}' ';' .PHONY: test -test: ## run test-clean and tests +## run test-clean and tests +test: make test-clean make test-only .PHONY:test-clean -test-clean: ## run test-clean and tests +## run test-clean and tests +test-clean: rm -rf coverage.xml htmlcov junit.xml pylint.log result find . -type d -name "__pycache__" -prune -exec rm -rf '{}' ';' find . -type f \( -iname '.coverage.*' \) -exec rm '{}' ';' .PHONY: test-only -test-only: ## run tests +## Run tests only without cleanup +test-only: PYTHONHASHSEED=random \ py.test -x -vv -r xw -p no:sugar \ - --cov=rhodecode --cov-report=term-missing --cov-report=html \ - rhodecode - + --cov-report=term-missing --cov-report=html \ + --cov=rhodecode rhodecode .PHONY: test-only-mysql -test-only-mysql: ## run tests against mysql +## run tests against mysql +test-only-mysql: PYTHONHASHSEED=random \ py.test -x -vv -r xw -p no:sugar \ - --cov=rhodecode --cov-report=term-missing --cov-report=html \ + --cov-report=term-missing --cov-report=html \ --ini-config-override='{"app:main": {"sqlalchemy.db1.url": "mysql://root:qweqwe@localhost/rhodecode_test?charset=utf8"}}' \ - rhodecode + --cov=rhodecode rhodecode .PHONY: test-only-postgres -test-only-postgres: ## run tests against postgres +## run tests against postgres +test-only-postgres: PYTHONHASHSEED=random \ py.test -x -vv -r xw -p no:sugar \ - --cov=rhodecode --cov-report=term-missing --cov-report=html \ + --cov-report=term-missing --cov-report=html \ --ini-config-override='{"app:main": {"sqlalchemy.db1.url": "postgresql://postgres:qweqwe@localhost/rhodecode_test"}}' \ - rhodecode + --cov=rhodecode rhodecode .PHONY: docs -docs: ## build docs +## build docs +docs: (cd docs; nix-build default.nix -o result; make clean html) .PHONY: docs-clean -docs-clean: ## Cleanup docs +## Cleanup docs +docs-clean: (cd docs; make clean) .PHONY: docs-cleanup -docs-cleanup: ## Cleanup docs +## Cleanup docs +docs-cleanup: (cd docs; make cleanup) .PHONY: web-build -web-build: ## Build static/js +## Build JS packages static/js +web-build: NODE_PATH=$(NODE_PATH) $(GRUNT) -.PHONY: generate-pkgs -generate-pkgs: ## generate new python packages - nix-shell pkgs/shell-generate.nix --command "pip2nix generate --licenses" - - .PHONY: pip-packages -pip-packages: ## show outdated packages +## show outdated packages +pip-packages: python ${OUTDATED_PACKAGES} -.PHONY: generate-js-pkgs -generate-js-pkgs: ## generate js packages - rm -rf node_modules && \ - nix-shell pkgs/shell-generate.nix --command "node2nix --input package.json -o pkgs/node-packages.nix -e pkgs/node-env.nix -c pkgs/node-default.nix -d --flatten --nodejs-8" && \ - sed -i -e 's/http:\/\//https:\/\//g' pkgs/node-packages.nix +.PHONY: sdist +## Build sdist +sdist: + python setup.py sdist -.PHONY: generate-license-meta -generate-license-meta: ## Generate license metadata - nix-build pkgs/license-generate.nix -o result-license && \ - cat result-license/licenses.json | python -m json.tool > rhodecode/config/licenses.json +# Default command on calling make +.DEFAULT_GOAL := show-help -.PHONY: help -help: - @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-24s\033[0m %s\n", $$1, $$2}' +.PHONY: show-help +show-help: + @echo "$$(tput bold)Available rules:$$(tput sgr0)" + @echo + @sed -n -e "/^## / { \ + h; \ + s/.*//; \ + :doc" \ + -e "H; \ + n; \ + s/^## //; \ + t doc" \ + -e "s/:.*//; \ + G; \ + s/\\n## /---/; \ + s/\\n/ /g; \ + p; \ + }" ${MAKEFILE_LIST} \ + | LC_ALL='C' sort --ignore-case \ + | awk -F '---' \ + -v ncol=$$(tput cols) \ + -v indent=19 \ + -v col_on="$$(tput setaf 6)" \ + -v col_off="$$(tput sgr0)" \ + '{ \ + printf "%s%*s%s ", col_on, -indent, $$1, col_off; \ + n = split($$2, words, " "); \ + line_length = ncol - indent; \ + for (i = 1; i <= n; i++) { \ + line_length -= length(words[i]) + 1; \ + if (line_length <= 0) { \ + line_length = ncol - indent - length(words[i]) - 1; \ + printf "\n%*s ", -indent, " "; \ + } \ + printf "%s ", words[i]; \ + } \ + printf "\n"; \ + }' + diff --git a/default.nix b/default.nix --- a/default.nix +++ b/default.nix @@ -47,20 +47,22 @@ let # Evaluates to the last segment of a file system path. basename = path: with pkgs.lib; last (splitString "/" path); + startsWith = prefix: full: let + actualPrefix = builtins.substring 0 (builtins.stringLength prefix) full; + in actualPrefix == prefix; # source code filter used as arugment to builtins.filterSource. src-filter = path: type: with pkgs.lib; let ext = last (splitString "." path); + parts = last (splitString "/" path); in !builtins.elem (basename path) [ ".git" ".hg" "__pycache__" ".eggs" ".idea" ".dev" "node_modules" "node_binaries" "build" "data" "result" "tmp"] && !builtins.elem ext ["egg-info" "pyc"] && - # TODO: johbo: This check is wrong, since "path" contains an absolute path, - # it would still be good to restore it since we want to ignore "result-*". - !hasPrefix "result" path; + !startsWith "result" (basename path); sources = let @@ -69,6 +71,7 @@ let all isString attrValues; + sourcesConfig = pkgs.config.rc.sources or {}; in # Ensure that sources are configured as strings. Using a path @@ -76,8 +79,8 @@ let assert all isString (attrValues sourcesConfig); sourcesConfig; + rhodecode-enterprise-ce-src = builtins.filterSource src-filter ./.; version = builtins.readFile "${rhodecode-enterprise-ce-src}/rhodecode/VERSION"; - rhodecode-enterprise-ce-src = builtins.filterSource src-filter ./.; nodeEnv = import ./pkgs/node-default.nix { inherit @@ -122,6 +125,12 @@ let ''; releaseName = "RhodeCodeEnterpriseCE-${version}"; + pythonWithEnv = + self.python.buildEnv.override { + extraLibs = [ ] ++ self.rhodecode-enterprise-ce.propagatedBuildInputs; + ignoreCollisions = true; + #--set PYTHONHASHSEED random TODO + }; in super.rhodecode-enterprise-ce.override (attrs: { inherit doCheck @@ -142,6 +151,7 @@ let pythonCommunityOverrides; pythonPackages = self; + rc_pkgs = pkgs; }; buildInputs = @@ -153,16 +163,23 @@ let propagatedBuildInputs = attrs.propagatedBuildInputs or [] ++ [ - ]; + ]; + + preBuild = '' + export NIX_PATH=nixpkgs=${pkgs.path} + export SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt - LC_ALL = "en_US.UTF-8"; - LOCALE_ARCHIVE = - if pkgs.stdenv.isLinux - then "${pkgs.glibcLocales}/lib/locale/locale-archive" - else ""; + echo "[BEGIN]: Building frontend assets" + ${linkNodePackages} + make web-build + rm -fr node_modules + rm -fr node_binaries + echo "[DONE ]: Building frontend assets" + ''; # Add bin directory to path so that tests can find 'rhodecode'. preCheck = '' + echo "Expanding PATH with $out/bin directory" export PATH="$out/bin:$PATH" ''; @@ -178,15 +195,6 @@ let rm -rf $out/lib/${self.python.libPrefix}/site-packages/rhodecode/tests ''; - preBuild = '' - echo "[BEGIN]: Building frontend assets" - ${linkNodePackages} - make web-build - rm -fr node_modules - rm -fr node_binaries - echo "[DONE ]: Building frontend assets" - ''; - postInstall = '' # check required files STATIC_CHECK="/robots.txt /502.html @@ -210,31 +218,34 @@ let cp configs/production.ini $out/etc echo "[DONE ]: saved enterprise-ce production.ini into $out/etc" + echo "saving env in $out/etc/env_vars.txt" + touch $out/etc/env_vars.txt + echo "# RhodeCode build env vars" >> $out/etc/env_vars.txt + echo "LOCALE_ARCHIVE=\"${pkgs.glibcLocales}/lib/locale/locale-archive\"" >> $out/etc/env_vars.txt + echo "LC_ALL=\"en_US.UTF-8\"" >> $out/etc/env_vars.txt + cp -Rf rhodecode/config/rcextensions $out/etc/rcextensions.tmpl echo "[DONE ]: saved enterprise-ce rcextensions into $out/etc/rcextensions.tmpl" # python based programs need to be wrapped mkdir -p $out/bin + # expose python + ln -s ${pythonWithEnv}/bin/python $out/bin/ + # required binaries from dependencies - ln -s ${self.supervisor}/bin/supervisorctl $out/bin/ - ln -s ${self.supervisor}/bin/supervisord $out/bin/ - ln -s ${self.pastescript}/bin/paster $out/bin/ - ln -s ${self.channelstream}/bin/channelstream $out/bin/ - ln -s ${self.celery}/bin/celery $out/bin/ - ln -s ${self.gunicorn}/bin/gunicorn $out/bin/ - ln -s ${self.pyramid}/bin/prequest $out/bin/ - ln -s ${self.pyramid}/bin/pserve $out/bin/ + ln -s ${pythonWithEnv}/bin/supervisorctl $out/bin/ + ln -s ${pythonWithEnv}/bin/supervisord $out/bin/ + ln -s ${pythonWithEnv}/bin/paster $out/bin/ + ln -s ${pythonWithEnv}/bin/channelstream $out/bin/ + ln -s ${pythonWithEnv}/bin/celery $out/bin/ + ln -s ${pythonWithEnv}/bin/gunicorn $out/bin/ + ln -s ${pythonWithEnv}/bin/prequest $out/bin/ + ln -s ${pythonWithEnv}/bin/pserve $out/bin/ echo "[DONE ]: created symlinks into $out/bin" - DEPS="$out/bin/supervisorctl \ - $out/bin/supervisord \ - $out/bin/paster \ - $out/bin/channelstream \ - $out/bin/celery \ - $out/bin/gunicorn \ - $out/bin/prequest \ - $out/bin/pserve" + DEPS=" + " # wrap only dependency scripts, they require to have full PYTHONPATH set # to be able to import all packages @@ -251,17 +262,17 @@ let # rhodecode-tools don't need wrapping ln -s ${self.rhodecode-tools}/bin/rhodecode-* $out/bin/ - # expose sources of CE - ln -s $out $out/etc/rhodecode_enterprise_ce_source - # expose static files folder cp -Rf $out/lib/${self.python.libPrefix}/site-packages/rhodecode/public/ $out/etc/static chmod 755 -R $out/etc/static + # expose sources of rhodecode-enterprise-ce + ln -s $out $out/etc/rhodecode_enterprise_ce_source ''; + }); + }; - }; basePythonPackages = with builtins; if isAttrs pythonPackages then @@ -270,13 +281,8 @@ let getAttr pythonPackages pkgs; pythonGeneratedPackages = import ./pkgs/python-packages.nix { - inherit - pkgs; - inherit - (pkgs) - fetchurl - fetchgit - fetchhg; + inherit pkgs; + inherit (pkgs) fetchurl fetchgit fetchhg; }; pythonCommunityOverrides = import ./pkgs/python-packages-overrides.nix { diff --git a/package.json b/package.json --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rhodecode-enterprise", - "version": "2.0.0", + "version": "4.19.0", "private": true, "description" : "RhodeCode JS packaged", "license": "SEE LICENSE IN LICENSE.txt", diff --git a/pkgs/nix-common/pip2nix.nix b/pkgs/nix-common/pip2nix.nix --- a/pkgs/nix-common/pip2nix.nix +++ b/pkgs/nix-common/pip2nix.nix @@ -14,4 +14,12 @@ rec { pythonPackages; }; + pip-tools = pythonPackages.pip-tools; + + setuptools = pythonPackages.setuptools; + + wheel = pythonPackages.wheel; + + pip = pythonPackages.pip; + } diff --git a/pkgs/node-default.nix b/pkgs/node-default.nix --- a/pkgs/node-default.nix +++ b/pkgs/node-default.nix @@ -1,8 +1,8 @@ -# This file has been generated by node2nix 1.6.0. Do not edit! +# This file has been generated by node2nix 1.8.0. Do not edit! {pkgs ? import { inherit system; - }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-8_x"}: + }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-12_x"}: let nodeEnv = import ./node-env.nix { diff --git a/pkgs/node-env.nix b/pkgs/node-env.nix --- a/pkgs/node-env.nix +++ b/pkgs/node-env.nix @@ -11,7 +11,7 @@ let cat > $out/bin/tar <` elements. use: [ - {loader: 'babel-loader', - options: babelRCOptions}, + {loader: 'babel-loader', options: babelRCOptions}, {loader: 'polymer-webpack-loader', options: { processStyleLinks: true,