nix: Pass the backwards compatible fetchgit to python and node packages.

default.nix

@@ -1,228 +1,250 b''
 # Nix environment for the community edition
 #
 # This shall be as lean as possible, just producing the Enterprise
 # derivation. For advanced tweaks to pimp up the development environment we use
 # "shell.nix" so that it does not have to clutter this file.
 
-{ pkgs ? (import <nixpkgs> {})
-, pythonPackages ? "python27Packages"
+args@
+{ pythonPackages ? "python27Packages"
 , pythonExternalOverrides ? self: super: {}
 , doCheck ? true
+, ...
 }:
 
 let
-  inherit (pkgs.lib) fix extends;
+
+  # Use nixpkgs from args or import them. We use this indirect approach
+  # through args to be able to use the name `pkgs` for our customized packages.
+  # Otherwise we will end up with an infinite recursion.
+  nixpkgs = args.pkgs or (import <nixpkgs> { });
+
+  # johbo: Interim bridge which allows us to build with the upcoming
+  # nixos.16.09 branch (unstable at the moment of writing this note) and the
+  # current stable nixos-16.03.
+  backwardsCompatibleFetchgit = { ... }@args:
+    let
+      origSources = nixpkgs.fetchgit args;
+    in
+    nixpkgs.lib.overrideDerivation origSources (oldAttrs: {
+      NIX_PREFETCH_GIT_CHECKOUT_HOOK = ''
+        find $out -name '.git*' -print0 | xargs -0 rm -rf
+      '';
+    });
+
+  # Create a customized version of nixpkgs which should be used throughout the
+  # rest of this file.
+  pkgs = nixpkgs.overridePackages (self: super: {
+    fetchgit = backwardsCompatibleFetchgit;
+  });
+
+  # Evaluates to the last segment of a file system path.
+  basename = path: with pkgs.lib; last (splitString "/" path);
+
+  # source code filter used as arugment to builtins.filterSource.
+  src-filter = path: type: with pkgs.lib;
+    let
+      ext = last (splitString "." path);
+    in
+      !builtins.elem (basename path) [
+        ".git" ".hg" "__pycache__" ".eggs"
+        "bower_components" "node_modules"
+        "build" "data" "result" "tmp"] &&
+      !builtins.elem ext ["egg-info" "pyc"] &&
+      # TODO: johbo: This check is wrong, since "path" contains an absolute path,
+      # it would still be good to restore it since we want to ignore "result-*".
+      !hasPrefix "result" path;
 
   basePythonPackages = with builtins; if isAttrs pythonPackages
     then pythonPackages
     else getAttr pythonPackages pkgs;
 
   buildBowerComponents =
     pkgs.buildBowerComponents or
     (import ./pkgs/backport-16.03-build-bower-components.nix { inherit pkgs; });
 
-  elem = builtins.elem;
-  basename = path: with pkgs.lib; last (splitString "/" path);
-  startsWith = prefix: full: let
-    actualPrefix = builtins.substring 0 (builtins.stringLength prefix) full;
-  in actualPrefix == prefix;
-
-  src-filter = path: type: with pkgs.lib;
-    let
-      ext = last (splitString "." path);
-    in
-      !elem (basename path) [
-        ".git" ".hg" "__pycache__" ".eggs"
-        "bower_components" "node_modules"
-        "build" "data" "result" "tmp"] &&
-      !elem ext ["egg-info" "pyc"] &&
-      # TODO: johbo: This check is wrong, since "path" contains an absolute path,
-      # it would still be good to restore it since we want to ignore "result-*".
-      !startsWith "result" path;
-
   sources = pkgs.config.rc.sources or {};
   version = builtins.readFile ./rhodecode/VERSION;
   rhodecode-enterprise-ce-src = builtins.filterSource src-filter ./.;
 
   nodeEnv = import ./pkgs/node-default.nix {
     inherit pkgs;
   };
   nodeDependencies = nodeEnv.shell.nodeDependencies;
 
   bowerComponents = buildBowerComponents {
     name = "enterprise-ce-${version}";
     generated = ./pkgs/bower-packages.nix;
     src = rhodecode-enterprise-ce-src;
   };
 
   pythonGeneratedPackages = self: basePythonPackages.override (a: {
     inherit self;
   })
   // (scopedImport {
     self = self;
     super = basePythonPackages;
     inherit pkgs;
     inherit (pkgs) fetchurl fetchgit;
   } ./pkgs/python-packages.nix);
 
   pythonOverrides = import ./pkgs/python-packages-overrides.nix {
     inherit
       basePythonPackages
       pkgs;
   };
 
   pythonLocalOverrides = self: super: {
     rhodecode-enterprise-ce =
       let
         linkNodeAndBowerPackages = ''
           echo "Export RhodeCode CE path"
           export RHODECODE_CE_PATH=${rhodecode-enterprise-ce-src}
           echo "Link node packages"
           rm -fr node_modules
           mkdir node_modules
           # johbo: Linking individual packages allows us to run "npm install"
           # inside of a shell to try things out. Re-entering the shell will
           # restore a clean environment.
           ln -s ${nodeDependencies}/lib/node_modules/* node_modules/
 
           echo "DONE: Link node packages"
 
           echo "Link bower packages"
           rm -fr bower_components
           mkdir bower_components
 
           ln -s ${bowerComponents}/bower_components/* bower_components/
           echo "DONE: Link bower packages"
         '';
       in super.rhodecode-enterprise-ce.override (attrs: {
 
       inherit
         doCheck
         version;
       name = "rhodecode-enterprise-ce-${version}";
       releaseName = "RhodeCodeEnterpriseCE-${version}";
       src = rhodecode-enterprise-ce-src;
 
       buildInputs =
         attrs.buildInputs ++
         (with self; [
           pkgs.nodePackages.bower
           pkgs.nodePackages.grunt-cli
           pkgs.subversion
           pytest-catchlog
           rhodecode-testdata
         ]);
 
       propagatedBuildInputs = attrs.propagatedBuildInputs ++ (with self; [
         rhodecode-tools
       ]);
 
       # TODO: johbo: Make a nicer way to expose the parts. Maybe
       # pkgs/default.nix?
       passthru = {
         inherit
           bowerComponents
           linkNodeAndBowerPackages
           myPythonPackagesUnfix
           pythonLocalOverrides;
         pythonPackages = self;
       };
 
       LC_ALL = "en_US.UTF-8";
       LOCALE_ARCHIVE =
         if pkgs.stdenv ? glibc
         then "${pkgs.glibcLocales}/lib/locale/locale-archive"
         else "";
 
       # Somewhat snappier setup of the development environment
       # TODO: move into shell.nix
       # TODO: think of supporting a stable path again, so that multiple shells
       #       can share it.
       shellHook = ''
         tmp_path=$(mktemp -d)
         export PATH="$tmp_path/bin:$PATH"
         export PYTHONPATH="$tmp_path/${self.python.sitePackages}:$PYTHONPATH"
         mkdir -p $tmp_path/${self.python.sitePackages}
         python setup.py develop --prefix $tmp_path --allow-hosts ""
       '' + linkNodeAndBowerPackages;
 
       preCheck = ''
         export PATH="$out/bin:$PATH"
       '';
 
       postCheck = ''
         rm -rf $out/lib/${self.python.libPrefix}/site-packages/pytest_pylons
         rm -rf $out/lib/${self.python.libPrefix}/site-packages/rhodecode/tests
       '';
 
       preBuild = linkNodeAndBowerPackages + ''
         grunt
         rm -fr node_modules
       '';
 
       postInstall = ''
         # python based programs need to be wrapped
         ln -s ${self.supervisor}/bin/supervisor* $out/bin/
         ln -s ${self.gunicorn}/bin/gunicorn $out/bin/
         ln -s ${self.PasteScript}/bin/paster $out/bin/
         ln -s ${self.channelstream}/bin/channelstream $out/bin/
         ln -s ${self.pyramid}/bin/* $out/bin/  #*/
 
         # rhodecode-tools
         # TODO: johbo: re-think this. Do the tools import anything from enterprise?
         ln -s ${self.rhodecode-tools}/bin/rhodecode-* $out/bin/
 
         # note that condition should be restricted when adding further tools
         for file in $out/bin/*; do  #*/
           wrapProgram $file \
               --prefix PYTHONPATH : $PYTHONPATH \
               --prefix PATH : $PATH \
               --set PYTHONHASHSEED random
         done
 
         mkdir $out/etc
         cp configs/production.ini $out/etc
 
         echo "Writing meta information for rccontrol to nix-support/rccontrol"
         mkdir -p $out/nix-support/rccontrol
         cp -v rhodecode/VERSION $out/nix-support/rccontrol/version
         echo "DONE: Meta information for rccontrol written"
 
         # TODO: johbo: Make part of ac-tests
         if [ ! -f rhodecode/public/js/scripts.js ]; then
           echo "Missing scripts.js"
           exit 1
         fi
         if [ ! -f rhodecode/public/css/style.css ]; then
           echo "Missing style.css"
           exit 1
         fi
       '';
 
     });
 
     rhodecode-testdata = import "${rhodecode-testdata-src}/default.nix" {
     inherit
       doCheck
       pkgs
       pythonPackages;
     };
 
   };
 
   rhodecode-testdata-src = sources.rhodecode-testdata or (
     pkgs.fetchhg {
       url = "https://code.rhodecode.com/upstream/rc_testdata";
       rev = "v0.8.0";
       sha256 = "0hy1ba134rq2f9si85yx7j4qhc9ky0hjzdk553s3q026i7km809m";
   });
 
   # Apply all overrides and fix the final package set
-  myPythonPackagesUnfix =
+  myPythonPackagesUnfix = with pkgs.lib;
     (extends pythonExternalOverrides
     (extends pythonLocalOverrides
     (extends pythonOverrides
              pythonGeneratedPackages)));
-  myPythonPackages = (fix myPythonPackagesUnfix);
+  myPythonPackages = (pkgs.lib.fix myPythonPackagesUnfix);
 
 in myPythonPackages.rhodecode-enterprise-ce

pkgs/python-packages-overrides.nix

@@ -1,296 +1,283 b''
 # Overrides for the generated python-packages.nix
 #
 # This function is intended to be used as an extension to the generated file
 # python-packages.nix. The main objective is to add needed dependencies of C
 # libraries and tweak the build instructions where needed.
 
 { pkgs, basePythonPackages }:
 
 let
   sed = "sed -i";
   localLicenses = {
     repoze = {
       fullName = "Repoze License";
       url =  http://www.repoze.org/LICENSE.txt;
     };
   };
 
-  # johbo: Interim bridge which allows us to build with the upcoming
-  # nixos.16.09 branch (unstable at the moment of writing this note) and the
-  # current stable nixos-16.03.
-  backwardsCompatibleFetchgit = { ... }@args:
-    let
-      origSources = pkgs.fetchgit args;
-    in
-    pkgs.lib.overrideDerivation origSources (oldAttrs: {
-      NIX_PREFETCH_GIT_CHECKOUT_HOOK = ''
-        find $out -name '.git*' -print0 | xargs -0 rm -rf
-      '';
-    });
-
 in
 
 self: super: {
 
   appenlight-client = super.appenlight-client.override (attrs: {
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   });
 
   future = super.future.override (attrs: {
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   });
 
   gnureadline = super.gnureadline.override (attrs: {
     buildInputs = attrs.buildInputs ++ [
       pkgs.ncurses
     ];
     patchPhase = ''
       substituteInPlace setup.py --replace "/bin/bash" "${pkgs.bash}/bin/bash"
     '';
   });
 
   gunicorn = super.gunicorn.override (attrs: {
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
       # johbo: futures is needed as long as we are on Python 2, otherwise
       # gunicorn explodes if used with multiple threads per worker.
       self.futures
     ];
   });
 
   ipython = super.ipython.override (attrs: {
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
       self.gnureadline
     ];
   });
 
   kombu = super.kombu.override (attrs: {
     # The current version of kombu needs some patching to work with the
     # other libs. Should be removed once we update celery and kombu.
     patches = [
       ./patch-kombu-py-2-7-11.diff
       ./patch-kombu-msgpack.diff
     ];
   });
 
   lxml = super.lxml.override (attrs: {
     buildInputs = with self; [
       pkgs.libxml2
       pkgs.libxslt
     ];
   });
 
   MySQL-python = super.MySQL-python.override (attrs: {
     buildInputs = attrs.buildInputs ++ [
       pkgs.openssl
     ];
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
       pkgs.mysql.lib
       pkgs.zlib
     ];
   });
 
   psutil = super.psutil.override (attrs: {
     buildInputs = attrs.buildInputs ++
       pkgs.lib.optional pkgs.stdenv.isDarwin pkgs.darwin.IOKit;
   });
 
   psycopg2 = super.psycopg2.override (attrs: {
     buildInputs = attrs.buildInputs ++
       pkgs.lib.optional pkgs.stdenv.isDarwin pkgs.openssl;
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
       pkgs.postgresql
     ];
     meta = {
       license = pkgs.lib.licenses.lgpl3Plus;
     };
   });
 
   py-gfm = super.py-gfm.override  {
-    src = backwardsCompatibleFetchgit {
+    src = pkgs.fetchgit {
       url = "https://code.rhodecode.com/upstream/py-gfm";
       rev = "0d66a19bc16e3d49de273c0f797d4e4781e8c0f2";
       sha256 = "0ryp74jyihd3ckszq31bml5jr3bciimhfp7va7kw6ld92930ksv3";
     };
   };
 
   pycurl = super.pycurl.override (attrs: {
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
       pkgs.curl
       pkgs.openssl
     ];
     preConfigure = ''
       substituteInPlace setup.py --replace '--static-libs' '--libs'
       export PYCURL_SSL_LIBRARY=openssl
     '';
     meta = {
       # TODO: It is LGPL and MIT
       license = pkgs.lib.licenses.mit;
     };
   });
 
   Pylons = super.Pylons.override (attrs: {
     name = "Pylons-1.0.1-patch1";
-    src = backwardsCompatibleFetchgit {
+    src = pkgs.fetchgit {
       url = "https://code.rhodecode.com/upstream/pylons";
       rev = "707354ee4261b9c10450404fc9852ccea4fd667d";
       sha256 = "b2763274c2780523a335f83a1df65be22ebe4ff413a7bc9e9288d23c1f62032e";
     };
   });
 
   pyramid = super.pyramid.override (attrs: {
     postFixup = ''
       wrapPythonPrograms
       # TODO: johbo: "wrapPython" adds this magic line which
       # confuses pserve.
       ${sed} '/import sys; sys.argv/d' $out/bin/.pserve-wrapped
     '';
     meta = {
       license = localLicenses.repoze;
     };
   });
 
   pyramid-debugtoolbar = super.pyramid-debugtoolbar.override (attrs: {
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal localLicenses.repoze ];
     };
   });
 
   Pyro4 = super.Pyro4.override (attrs: {
     # TODO: Was not able to generate this version, needs further
     # investigation.
     name = "Pyro4-4.35";
     src = pkgs.fetchurl {
       url = "https://pypi.python.org/packages/source/P/Pyro4/Pyro4-4.35.src.tar.gz";
       md5 = "cbe6cb855f086a0f092ca075005855f3";
     };
   });
 
   pysqlite = super.pysqlite.override (attrs: {
     propagatedBuildInputs = [
       pkgs.sqlite
     ];
     meta = {
       license = [ pkgs.lib.licenses.zlib pkgs.lib.licenses.libpng ];
     };
   });
 
   pytest-runner = super.pytest-runner.override (attrs: {
     propagatedBuildInputs = [
       self.setuptools-scm
     ];
   });
 
   python-ldap = super.python-ldap.override (attrs: {
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
       pkgs.cyrus_sasl
       pkgs.openldap
       pkgs.openssl
     ];
     # TODO: johbo: Remove the "or" once we drop 16.03 support.
     NIX_CFLAGS_COMPILE = "-I${pkgs.cyrus_sasl.dev or pkgs.cyrus_sasl}/include/sasl";
   });
 
   python-pam = super.python-pam.override (attrs:
     let
       includeLibPam = pkgs.stdenv.isLinux;
     in {
       # TODO: johbo: Move the option up into the default.nix, we should
       # include python-pam only on supported platforms.
       propagatedBuildInputs = attrs.propagatedBuildInputs ++
         pkgs.lib.optional includeLibPam [
           pkgs.pam
         ];
       # TODO: johbo: Check if this can be avoided, or transform into
       # a real patch
       patchPhase = pkgs.lib.optionals includeLibPam ''
         substituteInPlace pam.py \
           --replace 'find_library("pam")' '"${pkgs.pam}/lib/libpam.so.0"'
       '';
     });
 
   rhodecode-tools = super.rhodecode-tools.override (attrs: {
     patches = [
       ./patch-rhodecode-tools-setup.diff
     ];
   });
 
   URLObject = super.URLObject.override (attrs: {
     meta = {
       license = {
         spdxId = "Unlicense";
         fullName = "The Unlicense";
         url = http://unlicense.org/;
       };
     };
   });
 
   amqplib = super.amqplib.override (attrs: {
     meta = {
       license = pkgs.lib.licenses.lgpl3;
     };
   });
 
   docutils = super.docutils.override (attrs: {
     meta = {
       license = pkgs.lib.licenses.bsd2;
     };
   });
 
   colander = super.colander.override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
   pyramid-beaker  = super.pyramid-beaker.override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
   pyramid-mako = super.pyramid-mako.override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
   repoze.lru = super.repoze.lru.override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
   recaptcha-client = super.recaptcha-client.override (attrs: {
     meta = {
       # TODO: It is MIT/X11
       license = pkgs.lib.licenses.mit;
     };
   });
 
   python-editor = super.python-editor.override (attrs: {
     meta = {
       license = pkgs.lib.licenses.asl20;
     };
   });
 
   translationstring = super.translationstring.override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
   venusian = super.venusian.override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
   # Avoid that setuptools is replaced, this leads to trouble
   # with buildPythonPackage.
   setuptools = basePythonPackages.setuptools;
 
 }