packaging: switched to nix 20.03 packages and latest pip2nix code

- dependencies changed:
  atomicwrites==1.4.0
  ipython==5.10.0
  pytest==4.6.9
  py==1.8.1
  pytest-cov==2.8.1
  pytest-sugar==0.9.3
  pytest-runner==5.2.0
  rhodecode-tools==2.0.0
  ipython==5.10.0
  mysqlclient==1.4.6 (change from mysql-python, which is deprecated)
  mako==1.1.2
  lxml==4.5.0
  click==7.1.2

MANIFEST.in

@@ -1,52 +1,53 b''
 # top level files
 
 include MANIFEST.in
 include README.rst
 include CHANGES.rst
 include LICENSE.txt
 
 include rhodecode/VERSION
 
 # docs
 recursive-include docs *
 
 # all config files
 recursive-include configs *
 
 # translations
 recursive-include rhodecode/i18n *
 
 # non-python core stuff
 recursive-include rhodecode *.cfg
 recursive-include rhodecode *.json
 recursive-include rhodecode *.ini_tmpl
 recursive-include rhodecode *.sh
 recursive-include rhodecode *.mako
 
 # 502 page
 include rhodecode/public/502.html
 
 # robots
 include rhodecode/public/robots.txt
 
 # images, css
 include rhodecode/public/css/*.css
 include rhodecode/public/images/*.*
 include rhodecode/public/images/ee_features/*.*
 
 # sound files
 include rhodecode/public/sounds/*.mp3
 include rhodecode/public/sounds/*.wav
 
 # fonts
 recursive-include rhodecode/public/fonts/RCIcons *
 
 # js
 recursive-include rhodecode/public/js *
 
 # templates
 recursive-include rhodecode/templates *
 
 # skip any tests files
+recursive-exclude result *
 recursive-exclude rhodecode/tests *
 

Makefile

@@ -1,69 +1,69 b''
 
 .PHONY: clean docs docs-clean docs-cleanup test test-clean test-only test-only-postgres test-only-mysql web-build generate-pkgs pip-packages
 
 NODE_PATH=./node_modules
 WEBPACK=./node_binaries/webpack
 GRUNT=./node_binaries/grunt
 # set by: PATH_TO_OUTDATED_PACKAGES=/some/path/outdated_packages.py
 OUTDATED_PACKAGES = ${PATH_TO_OUTDATED_PACKAGES}
 
 clean:
 	make test-clean
 	find . -type f \( -iname '*.c' -o -iname '*.pyc' -o -iname '*.so' -o -iname '*.orig' \) -exec rm '{}' ';'
 
 test:
 	make test-clean
 	make test-only
 
 test-clean:
 	rm -rf coverage.xml htmlcov junit.xml pylint.log result
 	find . -type d -name "__pycache__" -prune -exec rm -rf '{}' ';'
 	find . -type f \( -iname '.coverage.*' \) -exec rm '{}' ';'
 
 test-only:
 	PYTHONHASHSEED=random \
 	py.test -x -vv -r xw -p no:sugar --cov=rhodecode \
     --cov-report=term-missing --cov-report=html \
     rhodecode
 
 test-only-mysql:
 	PYTHONHASHSEED=random \
 	py.test -x -vv -r xw -p no:sugar --cov=rhodecode \
     --cov-report=term-missing --cov-report=html \
     --ini-config-override='{"app:main": {"sqlalchemy.db1.url": "mysql://root:qweqwe@localhost/rhodecode_test?charset=utf8"}}' \
     rhodecode
 
 test-only-postgres:
 	PYTHONHASHSEED=random \
 	py.test -x -vv -r xw -p no:sugar --cov=rhodecode \
     --cov-report=term-missing --cov-report=html \
     --ini-config-override='{"app:main": {"sqlalchemy.db1.url": "postgresql://postgres:qweqwe@localhost/rhodecode_test"}}' \
     rhodecode
 
 
 docs:
 	(cd docs; nix-build default.nix -o result; make clean html)
 
 docs-clean:
 	(cd docs; make clean)
 
 docs-cleanup:
 	(cd docs; make cleanup)
 
 web-build:
 	NODE_PATH=$(NODE_PATH) $(GRUNT)
 
 generate-pkgs:
-	nix-shell pkgs/shell-generate.nix --command "pip2nix generate --licenses"
+	nix-shell pkgs/shell-generate.nix --command "pip2nix generate --licenses --no-binary :all:"
 
 pip-packages:
 	python ${OUTDATED_PACKAGES}
 
 generate-js-pkgs:
 	rm -rf node_modules && \
 	nix-shell pkgs/shell-generate.nix --command "node2nix --input package.json -o pkgs/node-packages.nix -e pkgs/node-env.nix -c pkgs/node-default.nix -d --flatten --nodejs-8" && \
 	sed -i -e 's/http:\/\//https:\/\//g' pkgs/node-packages.nix
 
 generate-license-meta:
 	nix-build pkgs/license-generate.nix -o result-license && \
 	cat result-license/licenses.json | python -m json.tool > rhodecode/config/licenses.json
  No newline at end of file

default.nix

@@ -1,296 +1,292 b''
 # Nix environment for the community edition
 #
 # This shall be as lean as possible, just producing the enterprise-ce
 # derivation. For advanced tweaks to pimp up the development environment we use
 # "shell.nix" so that it does not have to clutter this file.
 #
 # Configuration, set values in "~/.nixpkgs/config.nix".
 # example
 #  {
 #    # Thoughts on how to configure the dev environment
 #    rc = {
 #      codeInternalUrl = "https://usr:[e-mail]/internal";
 #      sources = {
 #        rhodecode-vcsserver = "/home/user/work/rhodecode-vcsserver";
 #        rhodecode-enterprise-ce = "/home/user/work/rhodecode-enterprise-ce";
 #        rhodecode-enterprise-ee = "/home/user/work/rhodecode-enterprise-ee";
 #      };
 #    };
 #  }
 
 args@
 { system ? builtins.currentSystem
 , pythonPackages ? "python27Packages"
 , pythonExternalOverrides ? self: super: {}
 , doCheck ? false
 , ...
 }:
 
 let
   pkgs_ = args.pkgs or (import <nixpkgs> { inherit system; });
 in
 
 let
   pkgs = import <nixpkgs> {
     overlays = [
       (import ./pkgs/overlays.nix)
     ];
     inherit
       (pkgs_)
       system;
   };
 
   # Works with the new python-packages, still can fallback to the old
   # variant.
   basePythonPackagesUnfix = basePythonPackages.__unfix__ or (
     self: basePythonPackages.override (a: { inherit self; }));
 
   # Evaluates to the last segment of a file system path.
   basename = path: with pkgs.lib; last (splitString "/" path);
+  startsWith = prefix: full: let
+    actualPrefix = builtins.substring 0 (builtins.stringLength prefix) full;
+  in actualPrefix == prefix;
 
   # source code filter used as arugment to builtins.filterSource.
   src-filter = path: type: with pkgs.lib;
     let
       ext = last (splitString "." path);
+      parts = last (splitString "/" path);
     in
       !builtins.elem (basename path) [
         ".git" ".hg" "__pycache__" ".eggs" ".idea" ".dev"
         "node_modules" "node_binaries"
         "build" "data" "result" "tmp"] &&
       !builtins.elem ext ["egg-info" "pyc"] &&
-      # TODO: johbo: This check is wrong, since "path" contains an absolute path,
-      # it would still be good to restore it since we want to ignore "result-*".
-      !hasPrefix "result" path;
+      !startsWith "result" (basename path);
 
   sources =
     let
       inherit
         (pkgs.lib)
         all
         isString
         attrValues;
+
       sourcesConfig = pkgs.config.rc.sources or {};
     in
       # Ensure that sources are configured as strings. Using a path
       # would result in a copy into the nix store.
       assert all isString (attrValues sourcesConfig);
       sourcesConfig;
 
   version = builtins.readFile "${rhodecode-enterprise-ce-src}/rhodecode/VERSION";
   rhodecode-enterprise-ce-src = builtins.filterSource src-filter ./.;
 
   nodeEnv = import ./pkgs/node-default.nix {
     inherit
       pkgs
       system;
   };
   nodeDependencies = nodeEnv.shell.nodeDependencies;
 
   rhodecode-testdata-src = sources.rhodecode-testdata or (
     pkgs.fetchhg {
       url = "https://code.rhodecode.com/upstream/rc_testdata";
       rev = "v0.10.0";
       sha256 = "0zn9swwvx4vgw4qn8q3ri26vvzgrxn15x6xnjrysi1bwmz01qjl0";
   });
 
   rhodecode-testdata = import "${rhodecode-testdata-src}/default.nix" {
   inherit
     doCheck
     pkgs
     pythonPackages;
   };
 
   pythonLocalOverrides = self: super: {
     rhodecode-enterprise-ce =
       let
         linkNodePackages = ''
           export RHODECODE_CE_PATH=${rhodecode-enterprise-ce-src}
 
           echo "[BEGIN]: Link node packages and binaries"
           # johbo: Linking individual packages allows us to run "npm install"
           # inside of a shell to try things out. Re-entering the shell will
           # restore a clean environment.
           rm -fr node_modules
           mkdir node_modules
           ln -s ${nodeDependencies}/lib/node_modules/* node_modules/
           export NODE_PATH=./node_modules
 
           rm -fr node_binaries
           mkdir node_binaries
           ln -s ${nodeDependencies}/bin/* node_binaries/
           echo "[DONE ]: Link node packages and binaries"
         '';
 
         releaseName = "RhodeCodeEnterpriseCE-${version}";
       in super.rhodecode-enterprise-ce.override (attrs: {
       inherit
         doCheck
         version;
 
       name = "rhodecode-enterprise-ce-${version}";
       releaseName = releaseName;
       src = rhodecode-enterprise-ce-src;
       dontStrip = true; # prevent strip, we don't need it.
 
       # expose following attributed outside
       passthru = {
         inherit
           rhodecode-testdata
           linkNodePackages
           myPythonPackagesUnfix
           pythonLocalOverrides
           pythonCommunityOverrides;
 
         pythonPackages = self;
       };
 
       buildInputs =
         attrs.buildInputs or [] ++ [
           rhodecode-testdata
         ];
 
       #NOTE: option to inject additional propagatedBuildInputs
       propagatedBuildInputs =
         attrs.propagatedBuildInputs or [] ++ [
 
-        ];
+      ];
+
+      preBuild = ''
+        export NIX_PATH=nixpkgs=${pkgs.path}
+        export SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt
 
-      LC_ALL = "en_US.UTF-8";
-      LOCALE_ARCHIVE =
-        if pkgs.stdenv.isLinux
-        then "${pkgs.glibcLocales}/lib/locale/locale-archive"
-        else "";
+        echo "[BEGIN]: Building frontend assets"
+        ${linkNodePackages}
+        make web-build
+        rm -fr node_modules
+        rm -fr node_binaries
+        echo "[DONE ]: Building frontend assets"
+      '';
 
       # Add bin directory to path so that tests can find 'rhodecode'.
       preCheck = ''
+        echo "Expanding PATH with $out/bin directory"
         export PATH="$out/bin:$PATH"
       '';
 
       # custom check phase for testing
       checkPhase = ''
         runHook preCheck
         PYTHONHASHSEED=random py.test -vv -p no:sugar -r xw --cov-config=.coveragerc --cov=rhodecode --cov-report=term-missing rhodecode
         runHook postCheck
       '';
 
       postCheck = ''
         echo "Cleanup of rhodecode/tests"
         rm -rf $out/lib/${self.python.libPrefix}/site-packages/rhodecode/tests
       '';
 
-      preBuild = ''
-        echo "[BEGIN]: Building frontend assets"
-        ${linkNodePackages}
-        make web-build
-        rm -fr node_modules
-        rm -fr node_binaries
-        echo "[DONE ]: Building frontend assets"
-      '';
-
       postInstall = ''
         # check required files
         STATIC_CHECK="/robots.txt /502.html
                       /js/scripts.min.js /js/rhodecode-components.js
                       /css/style.css /css/style-polymer.css /css/style-ipython.css"
 
         for file in $STATIC_CHECK;
         do
             if [ ! -f rhodecode/public/$file ]; then
               echo "Missing $file"
               exit 1
             fi
         done
 
         echo "Writing enterprise-ce meta information for rccontrol to nix-support/rccontrol"
         mkdir -p $out/nix-support/rccontrol
         cp -v rhodecode/VERSION $out/nix-support/rccontrol/version
         echo "[DONE ]: enterprise-ce meta information for rccontrol written"
 
         mkdir -p $out/etc
         cp configs/production.ini $out/etc
         echo "[DONE ]: saved enterprise-ce production.ini into $out/etc"
 
         cp -Rf rhodecode/config/rcextensions $out/etc/rcextensions.tmpl
         echo "[DONE ]: saved enterprise-ce rcextensions into $out/etc/rcextensions.tmpl"
 
         # python based programs need to be wrapped
         mkdir -p $out/bin
 
         # required binaries from dependencies
         ln -s ${self.supervisor}/bin/supervisorctl $out/bin/
         ln -s ${self.supervisor}/bin/supervisord $out/bin/
         ln -s ${self.pastescript}/bin/paster $out/bin/
         ln -s ${self.channelstream}/bin/channelstream $out/bin/
         ln -s ${self.celery}/bin/celery $out/bin/
         ln -s ${self.gunicorn}/bin/gunicorn $out/bin/
         ln -s ${self.pyramid}/bin/prequest $out/bin/
         ln -s ${self.pyramid}/bin/pserve $out/bin/
 
         echo "[DONE ]: created symlinks into $out/bin"
         DEPS="$out/bin/supervisorctl \
               $out/bin/supervisord \
               $out/bin/paster \
               $out/bin/channelstream \
               $out/bin/celery \
               $out/bin/gunicorn \
               $out/bin/prequest \
               $out/bin/pserve"
 
         # wrap only dependency scripts, they require to have full PYTHONPATH set
         # to be able to import all packages
         for file in $DEPS;
         do
           wrapProgram $file \
             --prefix PATH : $PATH \
             --prefix PYTHONPATH : $PYTHONPATH \
             --set PYTHONHASHSEED random
         done
 
         echo "[DONE ]: enterprise-ce binary wrapping"
-
         # rhodecode-tools don't need wrapping
         ln -s ${self.rhodecode-tools}/bin/rhodecode-* $out/bin/
 
         # expose sources of CE
         ln -s $out $out/etc/rhodecode_enterprise_ce_source
 
         # expose static files folder
         cp -Rf $out/lib/${self.python.libPrefix}/site-packages/rhodecode/public/ $out/etc/static
         chmod 755 -R $out/etc/static
 
       '';
-    });
 
+    });
   };
 
+
   basePythonPackages = with builtins;
     if isAttrs pythonPackages then
       pythonPackages
     else
       getAttr pythonPackages pkgs;
 
   pythonGeneratedPackages = import ./pkgs/python-packages.nix {
-    inherit
-      pkgs;
-    inherit
-      (pkgs)
-      fetchurl
-      fetchgit
-      fetchhg;
+    inherit pkgs;
+    inherit (pkgs) fetchurl fetchgit fetchhg;
   };
 
   pythonCommunityOverrides = import ./pkgs/python-packages-overrides.nix {
     inherit pkgs basePythonPackages;
   };
 
   # Apply all overrides and fix the final package set
   myPythonPackagesUnfix = with pkgs.lib;
     (extends pythonExternalOverrides
     (extends pythonLocalOverrides
     (extends pythonCommunityOverrides
     (extends pythonGeneratedPackages
              basePythonPackagesUnfix))));
 
   myPythonPackages = (pkgs.lib.fix myPythonPackagesUnfix);
 
 in myPythonPackages.rhodecode-enterprise-ce

package.json

@@ -1,62 +1,62 b''
 {
   "name": "rhodecode-enterprise",
-  "version": "2.0.0",
+  "version": "4.19.0",
   "private": true,
   "description" : "RhodeCode JS packaged",
   "license": "SEE LICENSE IN LICENSE.txt",
   "repository" : {
     "type" : "hg",
     "url" : "https://code.rhodecode.com/rhodecode-enterprise-ce"
   },
   "devDependencies": {
     "appenlight-client": "git+https://[e-mail]/AppEnlight/appenlight-client-js.git#0.5.1",
     "clipboard": "^2.0.1",
     "exports-loader": "^0.6.4",
     "favico.js": "^0.3.10",
     "dropzone": "^5.5.0",
     "grunt": "^0.4.5",
     "grunt-cli": "^1.3.1",
     "grunt-contrib-concat": "^0.5.1",
     "grunt-contrib-copy": "^1.0.0",
     "grunt-contrib-jshint": "^0.12.0",
     "grunt-contrib-less": "^1.1.0",
     "grunt-contrib-watch": "^0.6.1",
     "grunt-webpack": "^3.1.3",
     "grunt-contrib-uglify": "^4.0.1",
     "sweetalert2": "^9.10.12",
     "jquery": "1.11.3",
     "mark.js": "8.11.1",
     "jshint": "^2.9.1-rc3",
     "moment": "^2.18.1",
     "mousetrap": "^1.6.1",
     "qrious": "^4.0.2",
     "sticky-sidebar": "3.3.1",
     "waypoints": "4.0.1",
     "webpack": "4.23.1",
     "webpack-cli": "3.1.2",
     "babel-core": "^6.26.3",
     "babel-loader": "^7.1.2",
     "babel-plugin-transform-object-rest-spread": "^6.26.0",
     "babel-preset-env": "^1.6.0",
     "copy-webpack-plugin": "^4.4.2",
     "css-loader": "^0.28.11",
     "html-loader": "^0.4.4",
     "html-webpack-plugin": "^3.2.0",
     "imports-loader": "^0.7.1",
     "polymer-webpack-loader": "^2.0.1",
     "style-loader": "^0.21.0",
     "webpack-uglify-js-plugin": "^1.1.9",
     "raw-loader": "1.0.0-beta.0",
     "ts-loader": "^1.3.3",
     "@webcomponents/webcomponentsjs": "^2.0.0",
     "@polymer/polymer": "^3.0.0",
     "@polymer/paper-button": "^3.0.0",
     "@polymer/paper-spinner": "^3.0.0",
     "@polymer/paper-tooltip": "^3.0.0",
     "@polymer/paper-toast": "^3.0.0",
     "@polymer/paper-toggle-button": "^3.0.0",
     "@polymer/iron-ajax": "^3.0.0",
     "@polymer/iron-autogrow-textarea": "^3.0.0",
     "@polymer/iron-a11y-keys": "^3.0.0"
   }
 }

pkgs/node-default.nix

@@ -1,17 +1,17 b''
-# This file has been generated by node2nix 1.6.0. Do not edit!
+# This file has been generated by node2nix 1.8.0. Do not edit!
 
 {pkgs ? import <nixpkgs> {
     inherit system;
-  }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-8_x"}:
+  }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-12_x"}:
 
 let
   nodeEnv = import ./node-env.nix {
     inherit (pkgs) stdenv python2 utillinux runCommand writeTextFile;
     inherit nodejs;
     libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null;
   };
 in
 import ./node-packages.nix {
   inherit (pkgs) fetchurl fetchgit;
   inherit nodeEnv;
 }
  No newline at end of file

pkgs/python-packages-overrides.nix

@@ -1,279 +1,315 b''
 # Overrides for the generated python-packages.nix
 #
 # This function is intended to be used as an extension to the generated file
 # python-packages.nix. The main objective is to add needed dependencies of C
 # libraries and tweak the build instructions where needed.
 
 { pkgs
 , basePythonPackages
 }:
 
 let
   sed = "sed -i";
 
   localLicenses = {
     repoze = {
       fullName = "Repoze License";
       url =  http://www.repoze.org/LICENSE.txt;
     };
+
+    unlicense = {
+      spdxId = "Unlicense";
+      fullName = "The Unlicense";
+      url = http://unlicense.org/;
+    };
   };
 
 in
 
 self: super: {
 
-  "appenlight-client" = super."appenlight-client".override (attrs: {
-    meta = {
-      license = [ pkgs.lib.licenses.bsdOriginal ];
-    };
-  });
-
   "beaker" = super."beaker".override (attrs: {
     patches = [
       ./patches/beaker/patch-beaker-lock-func-debug.diff
       ./patches/beaker/patch-beaker-metadata-reuse.diff
     ];
   });
 
   "cffi" = super."cffi".override (attrs: {
-    buildInputs = [
+    buildInputs = with self; attrs.buildInputs ++ [
       pkgs.libffi
     ];
   });
 
   "cryptography" = super."cryptography".override (attrs: {
-    buildInputs = [
+    buildInputs = with self; attrs.buildInputs ++ [
       pkgs.openssl
     ];
   });
 
   "gevent" = super."gevent".override (attrs: {
-    propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
+    propagatedBuildInputs = with self; attrs.propagatedBuildInputs ++ [
       # NOTE: (marcink) odd requirements from gevent aren't set properly,
       # thus we need to inject psutil manually
       self."psutil"
     ];
   });
 
-  "future" = super."future".override (attrs: {
-    meta = {
-      license = [ pkgs.lib.licenses.mit ];
-    };
-  });
-
-  "testpath" = super."testpath".override (attrs: {
-    meta = {
-      license = [ pkgs.lib.licenses.mit ];
-    };
-  });
-
   "gnureadline" = super."gnureadline".override (attrs: {
-    buildInputs = [
+    buildInputs = with self; attrs.buildInputs ++ [
       pkgs.ncurses
     ];
     patchPhase = ''
       substituteInPlace setup.py --replace "/bin/bash" "${pkgs.bash}/bin/bash"
     '';
   });
 
   "gunicorn" = super."gunicorn".override (attrs: {
-    propagatedBuildInputs = [
+    propagatedBuildInputs = with self; attrs.propagatedBuildInputs ++ [
       # johbo: futures is needed as long as we are on Python 2, otherwise
       # gunicorn explodes if used with multiple threads per worker.
       self."futures"
     ];
   });
 
   "nbconvert" = super."nbconvert".override (attrs: {
-    propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
+    propagatedBuildInputs = with self; attrs.propagatedBuildInputs ++ [
       # marcink: plug in jupyter-client for notebook rendering
       self."jupyter-client"
     ];
   });
 
   "ipython" = super."ipython".override (attrs: {
-    propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
+    propagatedBuildInputs = with self; attrs.propagatedBuildInputs ++ [
       self."gnureadline"
     ];
   });
 
   "lxml" = super."lxml".override (attrs: {
-    buildInputs = [
+    nativeBuildInputs = with self; attrs.nativeBuildInputs ++ [
+      pkgs.libxml2.dev
+      pkgs.libxslt.dev
+      self.cython
+    ];
+
+    buildInputs = with self; attrs.buildInputs ++ [
       pkgs.libxml2
       pkgs.libxslt
+      pkgs.zlib
     ];
-    propagatedBuildInputs = [
-      # Needed, so that "setup.py bdist_wheel" does work
-      self."wheel"
-    ];
+
+#    propagatedBuildInputs = [
+#      # Needed, so that "setup.py bdist_wheel" does work
+#      self."wheel"
+#    ];
   });
 
-  "mysql-python" = super."mysql-python".override (attrs: {
-    buildInputs = [
+  "mysqlclient" = super."mysqlclient".override (attrs: {
+    nativeBuildInputs = with self; attrs.nativeBuildInputs ++ [
+      pkgs.libmysqlclient
       pkgs.openssl
-    ];
-    propagatedBuildInputs = [
-      pkgs.libmysql
       pkgs.zlib
     ];
+
   });
 
   "psycopg2" = super."psycopg2".override (attrs: {
-    propagatedBuildInputs = [
+    nativeBuildInputs = with self; attrs.nativeBuildInputs ++ [
       pkgs.postgresql
     ];
     meta = {
       license = pkgs.lib.licenses.lgpl3Plus;
     };
   });
 
   "pycurl" = super."pycurl".override (attrs: {
-    propagatedBuildInputs = [
+    nativeBuildInputs = with self; attrs.nativeBuildInputs ++ [
+      pkgs.curl
+      pkgs.openssl
+    ];
+
+    propagatedBuildInputs = with self; attrs.propagatedBuildInputs ++ [
       pkgs.curl
       pkgs.openssl
     ];
 
     preConfigure = ''
       substituteInPlace setup.py --replace '--static-libs' '--libs'
       export PYCURL_SSL_LIBRARY=openssl
     '';
 
     meta = {
       license = pkgs.lib.licenses.mit;
     };
   });
 
-  "pyramid" = super."pyramid".override (attrs: {
-    meta = {
-      license = localLicenses.repoze;
-    };
-  });
-
-  "pyramid-debugtoolbar" = super."pyramid-debugtoolbar".override (attrs: {
-    meta = {
-      license = [ pkgs.lib.licenses.bsdOriginal localLicenses.repoze ];
-    };
-  });
-
   "pysqlite" = super."pysqlite".override (attrs: {
-    propagatedBuildInputs = [
+    propagatedBuildInputs = with self; attrs.propagatedBuildInputs ++ [
       pkgs.sqlite
     ];
     meta = {
       license = [ pkgs.lib.licenses.zlib pkgs.lib.licenses.libpng ];
     };
   });
 
   "python-ldap" = super."python-ldap".override (attrs: {
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
       pkgs.openldap
       pkgs.cyrus_sasl
       pkgs.openssl
     ];
   });
 
   "python-pam" = super."python-pam".override (attrs: {
-    propagatedBuildInputs = [
+    propagatedBuildInputs = with self; attrs.propagatedBuildInputs ++ [
         pkgs.pam
     ];
 
     # TODO: johbo: Check if this can be avoided, or transform into
     # a real patch
     patchPhase = ''
       substituteInPlace pam.py \
         --replace 'find_library("pam")' '"${pkgs.pam}/lib/libpam.so.0"'
     '';
 
     });
 
   "python-saml" = super."python-saml".override (attrs: {
-    buildInputs = [
+    nativeBuildInputs = with self; attrs.nativeBuildInputs ++ [
       pkgs.libxml2
       pkgs.libxslt
     ];
   });
 
   "dm.xmlsec.binding" = super."dm.xmlsec.binding".override (attrs: {
-    buildInputs = [
+    nativeBuildInputs = with self; attrs.nativeBuildInputs ++ [
+      pkgs.libxml2.dev
+      pkgs.libxslt.dev
+      pkgs.xmlsec
+      pkgs.libtool
+      pkgs.zlib
+    ];
+
+    buildInputs = with self; attrs.buildInputs ++ [
       pkgs.libxml2
       pkgs.libxslt
       pkgs.xmlsec
       pkgs.libtool
+      pkgs.zlib
     ];
   });
 
   "pyzmq" = super."pyzmq".override (attrs: {
     buildInputs = [
       pkgs.czmq
     ];
   });
 
+  "supervisor" = super."supervisor".override (attrs: {
+    patches = [
+      ./patches/supervisor/patch-rlimits-old-kernel.diff
+    ];
+  });
+
+  "py" = super."py".override (attrs: {
+    buildInputs = with self; attrs.buildInputs ++ [
+      self."setuptools-scm"
+    ];
+  });
+
+  "importlib-metadata" = super."importlib-metadata".override (attrs: {
+    buildInputs = with self; attrs.buildInputs ++ [
+      self."setuptools-scm"
+    ];
+  });
+
+  # License fixes
+  "appenlight-client" = super."appenlight-client".override (attrs: {
+    meta = {
+      license = [ pkgs.lib.licenses.bsdOriginal ];
+    };
+  });
+
   "urlobject" = super."urlobject".override (attrs: {
     meta = {
-      license = {
-        spdxId = "Unlicense";
-        fullName = "The Unlicense";
-        url = http://unlicense.org/;
-      };
+      license = localLicenses.unlicense;
+    };
+  });
+
+  "future" = super."future".override (attrs: {
+    meta = {
+      license = [ pkgs.lib.licenses.mit ];
+    };
+  });
+
+  "testpath" = super."testpath".override (attrs: {
+    meta = {
+      license = [ pkgs.lib.licenses.mit ];
     };
   });
 
   "docutils" = super."docutils".override (attrs: {
     meta = {
       license = pkgs.lib.licenses.bsd2;
     };
   });
 
   "colander" = super."colander".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
   "pyramid-beaker"  = super."pyramid-beaker".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
   "pyramid-mako" = super."pyramid-mako".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
+  "pyramid" = super."pyramid".override (attrs: {
+    meta = {
+      license = localLicenses.repoze;
+    };
+  });
+
+  "pyramid-debugtoolbar" = super."pyramid-debugtoolbar".override (attrs: {
+    meta = {
+      license = [ pkgs.lib.licenses.bsdOriginal localLicenses.repoze ];
+    };
+  });
+
   "repoze.lru" = super."repoze.lru".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
   "python-editor" = super."python-editor".override (attrs: {
     meta = {
       license = pkgs.lib.licenses.asl20;
     };
   });
 
   "translationstring" = super."translationstring".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
   "venusian" = super."venusian".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
-  "supervisor" = super."supervisor".override (attrs: {
-    patches = [
-      ./patches/supervisor/patch-rlimits-old-kernel.diff
-    ];
-  });
-
   # Avoid that base packages screw up the build process
   inherit (basePythonPackages)
     setuptools;
 
 }

pkgs/shell-generate.nix

@@ -1,55 +1,76 b''
 { pkgs ? (import <nixpkgs> {})
 , pythonPackages ? "python27Packages"
 }:
 
 with pkgs.lib;
 
-let _pythonPackages = pythonPackages; in
+let
+  _pythonPackages = pythonPackages;
+
+in
+
 let
   pythonPackages = getAttr _pythonPackages pkgs;
 
   pip2nix = import ./nix-common/pip2nix.nix {
     inherit
       pkgs
       pythonPackages;
   };
 
 in
 
 pkgs.stdenv.mkDerivation {
   name = "pip2nix-generated";
+
   buildInputs = [
     # Allows to generate python packages
     pip2nix.pip2nix
     pythonPackages.pip-tools
+    # compile using ffi
+    pkgs.libffi
+
+    pythonPackages.cython
 
     # Allows to generate node dependencies
     pkgs.nodePackages.node2nix
 
     # We need mysql_config to be around
-    pkgs.mysql
+    pkgs.libmysqlclient
 
     # We need postgresql to be around
     pkgs.postgresql
 
     # we need the below for saml
     pkgs.libxml2
     pkgs.libxslt
     pkgs.xmlsec
 
     # Curl is needed for pycurl
     pkgs.curl
   ];
 
+  LC_ALL = "en_US.UTF-8";
+  LOCALE_ARCHIVE =
+    if pkgs.stdenv.isLinux
+    then "${pkgs.glibcLocales}/lib/locale/locale-archive"
+    else "";
+
   shellHook = ''
     runHook preShellHook
     runHook postShellHook
   '';
 
   preShellHook = ''
     echo "Starting Generate Shell"
+    # set unpack source date to 1980 to fix ZIP problems that does not support <1980
+    export SOURCE_DATE_EPOCH=315532800
+    export TMPDIR=/tmp
+
     # Custom prompt to distinguish from other dev envs.
-    export PS1="\n\[\033[1;32m\][Generate-shell:\w]$\[\033[0m\] "
+    export PS1="\n\[\033[1;32m\][pip2nix-generate-shell]$\[\033[0m\] "
+
     export PYCURL_SSL_LIBRARY=openssl
+
   '';
 }

requirements.txt

@@ -1,123 +1,123 b''
 ## dependencies
 
 amqp==2.5.2
 babel==1.3
 beaker==1.9.1
 bleach==3.1.3
 celery==4.3.0
 channelstream==0.5.2
-click==7.0
+click==7.1.2
 colander==1.7.0
 # our custom configobj
 https://code.rhodecode.com/upstream/configobj/artifacts/download/0-012de99a-b1e1-4f64-a5c0-07a98a41b324.tar.gz?md5=6a513f51fe04b2c18cf84c1395a7c626#egg=configobj==5.0.6
 cssselect==1.0.3
 cryptography==2.6.1
+cython==0.29.17
 decorator==4.1.2
 deform==2.0.8
 docutils==0.16.0
 dogpile.cache==0.9.0
 dogpile.core==0.4.1
 formencode==1.2.4
 future==0.14.3
 futures==3.0.2
 infrae.cache==1.0.1
 iso8601==0.1.12
 itsdangerous==0.24
 kombu==4.6.6
-lxml==4.2.5
-mako==1.1.0
+lxml==4.5.0
+mako==1.1.2
 markdown==2.6.11
 markupsafe==1.1.1
 msgpack-python==0.5.6
 pyotp==2.3.0
 packaging==20.3
 pathlib2==2.3.5
 paste==3.4.0
 pastedeploy==2.1.0
 pastescript==3.2.0
 peppercorn==0.6
 premailer==3.6.1
 psutil==5.7.0
 py-bcrypt==0.4
 pycurl==7.43.0.3
 pycrypto==2.6.1
 pygments==2.4.2
 pyparsing==2.4.7
 pyramid-debugtoolbar==4.6.1
 pyramid-mako==1.1.0
 pyramid==1.10.4
 pyramid_mailer==0.15.1
 python-dateutil==2.8.1
 python-ldap==3.2.0
 python-memcached==1.59
 python-pam==1.8.4
 python-saml==2.4.2
 pytz==2019.3
 tzlocal==1.5.1
 pyzmq==14.6.0
 py-gfm==0.1.4
 redis==3.4.1
 repoze.lru==0.7
 requests==2.22.0
 routes==2.4.1
 simplejson==3.16.0
 six==1.11.0
 sqlalchemy==1.3.15
 sshpubkeys==3.1.0
 subprocess32==3.5.4
 supervisor==4.1.0
 translationstring==1.3
 urllib3==1.25.2
 urlobject==2.4.3
 venusian==1.2.0
 weberror==0.13.1
 webhelpers2==2.0
 webob==1.8.5
 whoosh==2.7.4
 wsgiref==0.1.2
 zope.cachedescriptors==4.3.1
 zope.deprecation==4.4.0
 zope.event==4.4.0
 zope.interface==4.6.0
 
 # DB drivers
-mysql-python==1.2.5
+mysqlclient==1.4.6
 pymysql==0.8.1
 pysqlite==2.8.3
 psycopg2==2.8.4
 
 # IPYTHON RENDERING
 # entrypoints backport, pypi version doesn't support egg installs
 https://code.rhodecode.com/upstream/entrypoints/artifacts/download/0-8e9ee9e4-c4db-409c-b07e-81568fd1832d.tar.gz?md5=3a027b8ff1d257b91fe257de6c43357d#egg=entrypoints==0.2.2.rhodecode-upstream1
 nbconvert==5.3.1
 nbformat==4.4.0
 jupyter-client==5.0.0
 jupyter-core==4.5.0
 
 ## cli tools
 alembic==1.4.2
 invoke==0.13.0
 bumpversion==0.5.3
 
 ## http servers
 gevent==1.5.0
 greenlet==0.4.15
 gunicorn==19.9.0
 waitress==1.3.1
 
 ## debug
 ipdb==0.13.2
-ipython==5.1.0
+ipython==5.10.0
 
 ## rhodecode-tools, special case, use file://PATH.tar.gz#egg=rhodecode-tools==X.Y.Z, to test local version
-https://code.rhodecode.com/rhodecode-tools-ce/artifacts/download/0-ed54e749-2ef5-4bc7-ae7f-7900e3c2aa15.tar.gz?sha256=76f024bad3a1e55fdb3d64f13f5b77ff21a12fee699918de2110fe21effd5a3a#egg=rhodecode-tools==1.4.0
-
+https://code.rhodecode.com/rhodecode-tools-ce/artifacts/download/0-0cf09d55-fcb5-4ab0-ad48-e71f65090875.tar.gz?sha256=ae458b6845f278aed1bcb90939ef01cdb581016ba0e7f58602d1ac6513f94707#egg=rhodecode-tools==2.0.0
 
 ## appenlight
 appenlight-client==0.6.26
 
 ## test related requirements
 -r requirements_test.txt
 
 ## uncomment to add the debug libraries
 #-r requirements_debug.txt

requirements_pinned.txt

@@ -1,27 +1,33 b''
 # contains not directly required libraries we want to pin the version.
 
-atomicwrites==1.3.0
+atomicwrites==1.4.0
 attrs==19.3.0
 asn1crypto==0.24.0
 billiard==3.6.1.0
 cffi==1.12.3
 chameleon==2.24
 configparser==4.0.2
 contextlib2==0.6.0.post1
 ecdsa==0.13.2
 gnureadline==6.3.8
 hupper==1.10.2
 ipaddress==1.0.23
 importlib-metadata==1.6.0
 jinja2==2.9.6
 jsonschema==2.6.0
 pluggy==0.13.1
 pyasn1-modules==0.2.6
 pyramid-jinja2==2.7
 scandir==1.10.0
 setproctitle==1.1.10
 tempita==0.5.2
-testpath==0.4.4
+
 transaction==2.4.0
 vine==1.3.0
 wcwidth==0.1.9
+
+# ptyprocess backport, pypi version doesn't support egg/source installs on python 2.X
+https://code.rhodecode.com/upstream/ptyprocess/artifacts/download/0-c8b019b1-c4d3-46ac-a0ad-1206ec3fb3cb.tar.gz?sha256=50394f2c5e117fcab4360bf99c8bc40be7211ee1a5860aeb3809b44249550c3e#egg=ptyprocess==0.6.0.rhodecode-upstream1
+
+# testpath backport, pypi version doesn't support egg/source installs on python 2.X
+https://code.rhodecode.com/upstream/testpath/artifacts/download/0-618e6b32-6ca5-428a-bda0-494bb347a56d.tar.gz?sha256=fd95bafd89ee2fb2bb0d82be34c9c5bba3a290f52cafc67a12a74ef825527019#egg=testpath==0.4.4.rhodecode-upstream1

requirements_test.txt

@@ -1,16 +1,16 b''
 # test related requirements
-pytest==4.6.5
-py==1.8.0
-pytest-cov==2.7.1
-pytest-sugar==0.9.2
-pytest-runner==5.1.0
+pytest==4.6.9
+py==1.8.1
+pytest-cov==2.8.1
+pytest-sugar==0.9.3
+pytest-runner==5.2.0
 pytest-profiling==1.7.0
 pytest-timeout==1.3.3
 gprof2dot==2017.9.19
 
 mock==3.0.5
 cov-core==1.15.0
 coverage==4.5.4
 
 webtest==2.0.34
 beautifulsoup4==4.6.3

setup.cfg

@@ -1,43 +1,39 b''
 [aliases]
 test = pytest
 
-[egg_info]
-tag_build =
-tag_svn_revision = false
-
 # Babel configuration
 [compile_catalog]
 domain = rhodecode
 directory = rhodecode/i18n
 statistics = true
 
 [extract_messages]
 add_comments = TRANSLATORS:
 output_file = rhodecode/i18n/rhodecode.pot
 msgid-bugs-address = [e-mail]
 copyright-holder = RhodeCode GmbH
 no-wrap = true
 keywords =
     lazy_ugettext
     _ngettext
     _gettext
     gettext_translator
 
 [init_catalog]
 domain = rhodecode
 input_file = rhodecode/i18n/rhodecode.pot
 output_dir = rhodecode/i18n
 
 [update_catalog]
 domain = rhodecode
 input_file = rhodecode/i18n/rhodecode.pot
 output_dir = rhodecode/i18n
 previous = true
 
 [build_sphinx]
 source-dir = docs/
 build-dir  = docs/_build
 all_files  = 1
 
 [upload_sphinx]
 upload-dir = docs/_build/html

shell.nix

@@ -1,119 +1,139 b''
 # This file contains the adjustments which are desired for a development
 # environment.
 
 { pkgs ? (import <nixpkgs> {})
 , pythonPackages ? "python27Packages"
 , doCheck ? false
 , sourcesOverrides ? {}
 , doDevelopInstall ? true
 , doReleaseInstall ? false
 }:
 
 let
   # Get sources from config and update them with overrides.
   sources = (pkgs.config.rc.sources or {}) // sourcesOverrides;
 
   enterprise-ce = import ./default.nix {
     inherit
       pythonPackages
       doCheck;
   };
 
   ce-pythonPackages = enterprise-ce.pythonPackages;
 
   # This method looks up a path from `pkgs.config.rc.sources` and returns a
   # shell script which does a `python setup.py develop` installation of it. If
   # no path is found it will return an empty string.
   optionalDevelopInstall = attributeName:
     let
       path = pkgs.lib.attrByPath [attributeName] null sources;
-      doIt = doDevelopInstall && path != null;
+      doDI = doDevelopInstall && path != null;
 
     in
       # do develop installation with empty hosts to skip any package duplicates to
       # be replaced. This only pushes the package to be locally available
-      pkgs.lib.optionalString doIt (''
+      pkgs.lib.optionalString doDI (''
         echo "[BEGIN] Develop install of '${attributeName}' from '${path}'"
         pushd ${path}
         python setup.py develop --prefix $tmp_path --allow-hosts ""
         popd
         echo "[DONE] Develop install of '${attributeName}' from '${path}'"
         echo ""
       '');
 
   # This method looks up a path from `pkgs.config.rc.sources` and imports the
   # default.nix file if it exists. It returns the list of build inputs. If no
   # path is found it will return an empty list.
   optionalDevelopInstallBuildInputs = attributeName:
     let
       path = pkgs.lib.attrByPath [attributeName] null sources;
-      doIt = doDevelopInstall && path != null && pkgs.lib.pathExists "${nixFile}";
       nixFile = "${path}/default.nix";
+      doDI = doDevelopInstall && path != null && pkgs.lib.pathExists "${nixFile}";
 
       derivate = import "${nixFile}" {
         inherit
           doCheck
           pythonPackages;
       };
     in
-      pkgs.lib.lists.optionals doIt (
+      pkgs.lib.lists.optionals doDI (
         derivate.propagatedBuildInputs
       );
 
+  optionalBinDeps = attributeName:
+    let
+      path = pkgs.lib.attrByPath [attributeName] null sources;
+      nixFile = "${path}/default.nix";
+      doDI = doDevelopInstall && path != null && pkgs.lib.pathExists "${nixFile}";
+
+      derivate = import "${nixFile}" {
+        inherit
+          doCheck
+          pythonPackages;
+      };
+    in
+      pkgs.lib.optionalString doDI (''
+            echo "Wrapping PATH with vcsserver vcs binaries"
+            export PATH="${derivate.vcs_pkgs.subversion}/bin:${derivate.vcs_pkgs.git}/bin:${derivate.pythonPackages.mercurial}/bin:$PATH"
+      '');
+
   developInstalls = [ "rhodecode-vcsserver" ];
 
-in enterprise-ce.override (attrs: {
+in enterprise-ce.override (attrs: rec {
   # Avoid that we dump any sources into the store when entering the shell and
   # make development a little bit more convenient.
   src = null;
 
   # Add dependencies which are useful for the development environment.
   buildInputs =
     attrs.buildInputs ++
     (with ce-pythonPackages;
       [ ipdb ]
       ++ pkgs.lib.lists.optionals doReleaseInstall (
         [invoke bumpversion]
       )
     );
 
   # place to inject some required libs from develop installs
   propagatedBuildInputs =
     attrs.propagatedBuildInputs ++
     pkgs.lib.lists.concatMap optionalDevelopInstallBuildInputs developInstalls;
 
 
   # Make sure we execute both hooks
   shellHook = ''
     runHook preShellHook
     runHook postShellHook
   '';
 
   preShellHook = ''
-    echo "Entering CE-Shell"
+    echo "Entering rhodecode-ce"
 
     # Custom prompt to distinguish from other dev envs.
-    export PS1="\n\[\033[1;32m\][CE-shell:\w]$\[\033[0m\] "
+    export PS1="\n\[\033[1;32m\][rhodecode-ce-shell:\w]$\[\033[0m\] "
+
+    # Set locale
+    export LOCALE_ARCHIVE="${pkgs.glibcLocales}/lib/locale/locale-archive"
+    export LC_ALL="en_US.UTF-8"
 
     echo "Building frontend assets"
     ${enterprise-ce.linkNodePackages}
 
     # Setup a temporary directory.
     tmp_path=$(mktemp -d)
     export PATH="$tmp_path/bin:$PATH"
     export PYTHONPATH="$tmp_path/${ce-pythonPackages.python.sitePackages}:$PYTHONPATH"
     mkdir -p $tmp_path/${ce-pythonPackages.python.sitePackages}
 
     # Develop installation
     echo "[BEGIN]: develop install of rhodecode-enterprise-ce"
     python setup.py develop --prefix $tmp_path --allow-hosts ""
   '';
 
   postShellHook = ''
     echo "** Additional develop installs **"
-    '' +
-    pkgs.lib.strings.concatMapStrings optionalDevelopInstall developInstalls
-    + ''
+    '' + pkgs.lib.strings.concatMapStrings optionalDevelopInstall developInstalls + ''
+    '' + pkgs.lib.strings.concatMapStrings optionalBinDeps [ "rhodecode-vcsserver" ] + ''
   '';
 
 })