pip2nix: cleanups and NEW package

default.nix

@@ -1,208 +1,208 b''
 # Nix environment for the community edition
 #
 # This shall be as lean as possible, just producing the rhodecode-vcsserver
 # derivation. For advanced tweaks to pimp up the development environment we use
 # "shell.nix" so that it does not have to clutter this file.
 
 args@
 { system ? builtins.currentSystem
 , pythonPackages ? "python27Packages"
 , pythonExternalOverrides ? self: super: {}
 , doCheck ? false
 , ...
 }:
 
 let
   pkgs_ = args.pkgs or (import <nixpkgs> { inherit system; });
 in
 
 let
   pkgs = import <nixpkgs> {
     overlays = [
       (import ./pkgs/overlays.nix)
     ];
     inherit
       (pkgs_)
       system;
   };
 
   # Works with the new python-packages, still can fallback to the old
   # variant.
   basePythonPackagesUnfix = basePythonPackages.__unfix__ or (
     self: basePythonPackages.override (a: { inherit self; }));
 
   # Evaluates to the last segment of a file system path.
   basename = path: with pkgs.lib; last (splitString "/" path);
   startsWith = prefix: full: let
     actualPrefix = builtins.substring 0 (builtins.stringLength prefix) full;
   in actualPrefix == prefix;
 
   # source code filter used as arugment to builtins.filterSource.
   src-filter = path: type: with pkgs.lib;
     let
       ext = last (splitString "." path);
       parts = last (splitString "/" path);
     in
       !builtins.elem (basename path) [
         ".git" ".hg" "__pycache__" ".eggs" ".idea" ".dev"
         "node_modules" "node_binaries"
         "build" "data" "result" "tmp"] &&
       !builtins.elem ext ["egg-info" "pyc"] &&
       !startsWith "result" (basename path);
 
   sources =
     let
       inherit
         (pkgs.lib)
         all
         isString
         attrValues;
 
       sourcesConfig = pkgs.config.rc.sources or {};
     in
       # Ensure that sources are configured as strings. Using a path
       # would result in a copy into the nix store.
       assert all isString (attrValues sourcesConfig);
       sourcesConfig;
 
   version = builtins.readFile "${rhodecode-vcsserver-src}/vcsserver/VERSION";
   rhodecode-vcsserver-src = builtins.filterSource src-filter ./.;
 
   pythonLocalOverrides = self: super: {
     rhodecode-vcsserver =
       let
         releaseName = "RhodeCodeVCSServer-${version}";
       in super.rhodecode-vcsserver.override (attrs: {
       inherit
         doCheck
         version;
 
       name = "rhodecode-vcsserver-${version}";
       releaseName = releaseName;
       src = rhodecode-vcsserver-src;
       dontStrip = true; # prevent strip, we don't need it.
 
       # expose following attributed outside
       passthru = {
         pythonPackages = self;
         vcs_pkgs = pkgs;
       };
 
       buildInputs =
         attrs.buildInputs or [] ++ [
 
         ];
 
       #NOTE: option to inject additional propagatedBuildInputs
       propagatedBuildInputs =
         attrs.propagatedBuildInputs or [] ++ [
         pkgs.git
         pkgs.subversion
       ];
 
       preBuild = ''
         export NIX_PATH=nixpkgs=${pkgs.path}
         export SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt
       '';
 
       # Add bin directory to path so that tests can find 'vcsserver'.
       preCheck = ''
         export PATH="$out/bin:$PATH"
       '';
 
       # custom check phase for testing
       checkPhase = ''
         runHook preCheck
         PYTHONHASHSEED=random py.test -vv -p no:sugar -r xw --cov-config=.coveragerc --cov=vcsserver --cov-report=term-missing vcsserver
         runHook postCheck
       '';
 
       postCheck = ''
         echo "Cleanup of vcsserver/tests"
         rm -rf $out/lib/${self.python.libPrefix}/site-packages/vcsserver/tests
       '';
 
       postInstall = ''
         echo "Writing vcsserver meta information for rccontrol to nix-support/rccontrol"
         mkdir -p $out/nix-support/rccontrol
         cp -v vcsserver/VERSION $out/nix-support/rccontrol/version
         echo "DONE: vcsserver meta information for rccontrol written"
 
         mkdir -p $out/etc
         cp configs/production.ini $out/etc
         echo "DONE: saved vcsserver production.ini into $out/etc"
 
         echo "saving env in $out/etc/env_vars.txt"
         touch $out/etc/env_vars.txt
         echo "# RhodeCode build env vars" >> $out/etc/env_vars.txt
         echo "LOCALE_ARCHIVE=\"${pkgs.glibcLocales}/lib/locale/locale-archive\"" >> $out/etc/env_vars.txt
         echo "LC_ALL=\"en_US.UTF-8\"" >> $out/etc/env_vars.txt
 
         # python based programs need to be wrapped
         mkdir -p $out/bin
 
         # expose python
-        ln -s ${self.python}/bin/python $out/bin/
+        ln -s ${pkgs.pythonWithSetuptools}/bin/python $out/bin/
 
         # required binaries from dependencies
         ln -s ${self.gunicorn}/bin/gunicorn $out/bin/
         ln -s ${self.pyramid}/bin/prequest $out/bin/
         ln -s ${self.pyramid}/bin/pserve $out/bin/
 
         # Symlink version control utilities
         # We ensure that always the correct version is available as a symlink.
         # So that users calling them via the profile path will always use the
         # correct version. Wrapping is required so those can "import"
         # vcsserver python hooks.
 
         ln -s ${pkgs.git}/bin/git $out/bin
         ln -s ${self.mercurial}/bin/hg $out/bin
         ln -s ${pkgs.subversion}/bin/svn* $out/bin
 
         echo "DONE: created symlinks into $out/bin"
         DEPS="$out/bin/*"
 
         # wrap only dependency scripts, they require to have full PYTHONPATH set
         # to be able to import all packages
         for file in $DEPS;
         do
           wrapProgram $file \
             --prefix PATH : $PATH \
             --prefix PYTHONPATH : $PYTHONPATH \
             --set PYTHONHASHSEED random
         done
 
         echo "DONE: vcsserver binary wrapping"
 
       '';
 
     });
   };
 
 
   basePythonPackages = with builtins;
     if isAttrs pythonPackages then
       pythonPackages
     else
       getAttr pythonPackages pkgs;
 
   pythonGeneratedPackages = import ./pkgs/python-packages.nix {
     inherit pkgs;
     inherit (pkgs) fetchurl fetchgit fetchhg;
   };
 
   pythonVCSServerOverrides = import ./pkgs/python-packages-overrides.nix {
     inherit pkgs basePythonPackages;
   };
 
   # Apply all overrides and fix the final package set
   myPythonPackagesUnfix = with pkgs.lib;
     (extends pythonExternalOverrides
     (extends pythonLocalOverrides
     (extends pythonVCSServerOverrides
     (extends pythonGeneratedPackages
              basePythonPackagesUnfix))));
 
   myPythonPackages = (pkgs.lib.fix myPythonPackagesUnfix);
 
 in myPythonPackages.rhodecode-vcsserver

pkgs/nix-common/pip2nix.nix

@@ -1,17 +1,25 b''
 { pkgs
 , pythonPackages
 }:
 
 rec {
   pip2nix-src = pkgs.fetchzip {
     url = https://github.com/johbo/pip2nix/archive/51e6fdae34d0e8ded9efeef7a8601730249687a6.tar.gz;
     sha256 = "02a4jjgi7lsvf8mhrxsd56s9a3yg20081rl9bgc2m84w60v2gbz2";
   };
 
   pip2nix = import pip2nix-src {
     inherit
       pkgs
       pythonPackages;
   };
 
+  pip-tools = pythonPackages.pip-tools;
+
+  setuptools = pythonPackages.setuptools;
+
+  wheel = pythonPackages.wheel;
+
+  pip = pythonPackages.pip;
+
 }

pkgs/overlays.nix

@@ -1,72 +1,75 b''
 self: super: {
 
+  pythonWithSetuptools = self.python.withPackages(ps: with ps; [
+    setuptools
+  ]);
+
   # change GIT version
   # latest supported are in: https://github.com/NixOS/nixpkgs/tree/master/pkgs/applications/version-management/git-and-tools/git
   git = super.lib.overrideDerivation super.git (oldAttrs: {
     name = "git-2.25.3";
     src = self.fetchurl {
       url = "https://www.kernel.org/pub/software/scm/git/git-2.25.3.tar.xz";
       sha256 = "0yvr97cl0dvj3fwblq1mb0cp97v8hrn9l98p8b1jx8815mbsnz9h";
     };
 
     # patches come from: https://github.com/NixOS/nixpkgs/tree/master/pkgs/applications/version-management/git-and-tools/git
     patches = [
       ./patches/git/docbook2texi.patch
       ./patches/git/git-sh-i18n.patch
       ./patches/git/ssh-path.patch
       ./patches/git/git-send-email-honor-PATH.patch
       ./patches/git/installCheck-path.patch
     ];
 
   });
 
   libgit2rc = super.lib.overrideDerivation super.libgit2 (oldAttrs: {
     name = "libgit2-0.28.2";
     version = "0.28.2";
 
     src = self.fetchFromGitHub {
         owner = "libgit2";
         repo = "libgit2";
         rev = "v0.28.2";
         sha256 = "0cm8fvs05rj0baigs2133q5a0sm3pa234y8h6hmwhl2bz9xq3k4b";
     };
 
     cmakeFlags = [ "-DTHREADSAFE=ON" "-DUSE_HTTPS=no"];
 
     buildInputs = [
         super.zlib
         super.libssh2
         super.openssl
         super.curl
     ];
 
 
   });
 
   # Override subversion derivation to
   #  - activate python bindings
   subversion =
   let
     subversionWithPython = super.subversion.override {
       httpSupport = true;
       pythonBindings = true;
       python = self.python27Packages.python;
     };
   in
     super.lib.overrideDerivation subversionWithPython (oldAttrs: {
       name = "subversion-1.13.0";
       src = self.fetchurl {
         url = "https://archive.apache.org/dist/subversion/subversion-1.13.0.tar.gz";
         sha256 = "0cb9p7f5hg0l4k32hz8vmvy2r45igchq5sh4m366za5q0c649bfs";
       };
 
       ## use internal lz4/utf8proc because it is stable and shipped with SVN
       configureFlags = oldAttrs.configureFlags ++ [
         " --with-lz4=internal"
         " --with-utf8proc=internal"
       ];
 
   });
 
-
 }

pkgs/shell-generate.nix

@@ -1,59 +1,60 b''
 { pkgs ? (import <nixpkgs> {})
 , pythonPackages ? "python27Packages"
 }:
 
 with pkgs.lib;
 
 let
   _pythonPackages = pythonPackages;
 
 in
 
 let
   pythonPackages = getAttr _pythonPackages pkgs;
 
   pip2nix = import ./nix-common/pip2nix.nix {
     inherit
       pkgs
       pythonPackages;
   };
 
 in
 
 pkgs.stdenv.mkDerivation {
   name = "pip2nix-generated";
 
   buildInputs = [
     # Allows to generate python packages
     pip2nix.pip2nix
-    pythonPackages.pip-tools
+    pip2nix.pip
+    pip2nix.pip-tools
+
     # compile using ffi
     pkgs.libffi
 
     pkgs.apr
     pkgs.aprutil
   ];
 
   shellHook = ''
     runHook preShellHook
     echo "Setting SVN_* variables"
     export SVN_LIBRARY_PATH=${pkgs.subversion}/lib
     export SVN_HEADER_PATH=${pkgs.subversion.dev}/include
     runHook postShellHook
   '';
 
   preShellHook = ''
     echo "Starting Generate Shell"
     # set unpack source date to 1980 to fix ZIP problems that does not support <1980
     export SOURCE_DATE_EPOCH=315532800
     export TMPDIR=/tmp
     export LOCALE_ARCHIVE="${pkgs.glibcLocales}/lib/locale/locale-archive"
     export LC_ALL="en_US.UTF-8"
+    export PYCURL_SSL_LIBRARY=openssl
 
     # Custom prompt to distinguish from other dev envs.
     export PS1="\n\[\033[1;32m\][pip2nix-generate-shell]$\[\033[0m\] "
 
-    export PYCURL_SSL_LIBRARY=openssl
-
   '';
 }