nix: updated to 18.03 nix packages

- dependencies: updated setuptools-scm to 2.1.0
- new strucutre
- cleanups and optimize of build chain

pkgs/README.rst

@@ -0,0 +1,55 b''
+
+==============================
+ Generate the Nix expressions
+==============================
+
+Details can be found in the repository of `RhodeCode Enterprise CE`_ inside of
+the file `docs/contributing/dependencies.rst`.
+
+Start the environment as follows:
+
+.. code:: shell
+
+   nix-shell pkgs/shell-generate.nix
+
+
+
+Python dependencies
+===================
+
+.. code:: shell
+
+   pip2nix generate --licenses
+   # or
+   nix-shell pkgs/shell-generate.nix --command "pip2nix generate --licenses"
+
+
+NodeJS dependencies
+===================
+
+.. code:: shell
+
+   # switch to pkgs dir
+   pushd pkgs
+   node2nix --input ../package.json \
+            -o node-packages.nix \
+            -e node-env.nix \
+            -c node-default.nix \
+            -d --flatten --nodejs-6
+   popd
+
+
+
+Bower dependencies
+==================
+
+.. code:: shell
+
+   bower2nix bower.json pkgs/bower-packages.nix
+   # or
+   nix-shell pkgs/shell-generate.nix --command "bower2nix bower.json pkgs/bower-packages.nix"
+
+
+.. Links
+
+.. _RhodeCode Enterprise CE: https://code.rhodecode.com/rhodecode-enterprise-ce

pkgs/nix-common/pip2nix.nix

@@ -0,0 +1,17 b''
+{ pkgs
+, pythonPackages
+}:
+
+rec {
+  pip2nix-src = pkgs.fetchzip {
+    url = https://github.com/johbo/pip2nix/archive/51e6fdae34d0e8ded9efeef7a8601730249687a6.tar.gz;
+    sha256 = "02a4jjgi7lsvf8mhrxsd56s9a3yg20081rl9bgc2m84w60v2gbz2";
+  };
+
+  pip2nix = import pip2nix-src {
+    inherit
+      pkgs
+      pythonPackages;
+  };
+
+}

pkgs/shell-generate.nix

@@ -0,0 +1,52 b''
+{ pkgs ? (import <nixpkgs> {})
+, pythonPackages ? "python27Packages"
+}:
+
+with pkgs.lib;
+
+let _pythonPackages = pythonPackages; in
+let
+  pythonPackages = getAttr _pythonPackages pkgs;
+
+  pip2nix = import ./nix-common/pip2nix.nix {
+    inherit
+      pkgs
+      pythonPackages;
+  };
+
+in
+
+pkgs.stdenv.mkDerivation {
+  name = "pip2nix-generated";
+  buildInputs = [
+    # Allows to generate python packages
+    pip2nix.pip2nix
+    pythonPackages.pip-tools
+
+    # Allows to generate bower dependencies
+    pkgs.nodePackages.bower2nix
+
+    # Allows to generate node dependencies
+    pkgs.nodePackages.node2nix
+
+    # We need mysql_config to be around
+    pkgs.mysql
+
+    # We need postgresql to be around
+    pkgs.postgresql
+
+    # Curl is needed for pycurl
+    pkgs.curl
+  ];
+
+  shellHook = ''
+    runHook preShellHook
+    runHook postShellHook
+  '';
+
+  preShellHook = ''
+    echo "Starting Generate Shell"
+    # Custom prompt to distinguish from other dev envs.
+    export PS1="\n\[\033[1;32m\][Generate-shell:\w]$\[\033[0m\] "
+  '';
+}

bower.json

@@ -10,7 +10,7 b''
     "paper-tooltip": "PolymerElements/paper-tooltip#^1.1.2",
     "paper-toast": "PolymerElements/paper-toast#^1.3.0",
     "paper-toggle-button": "PolymerElements/paper-toggle-button#^1.2.0",
-    "iron-ajax": "PolymerElements/iron-ajax#^1.4.3",
+    "iron-ajax": "PolymerElements/iron-ajax#^1.4.4",
     "iron-autogrow-textarea": "PolymerElements/iron-autogrow-textarea#^1.0.13",
     "iron-a11y-keys": "PolymerElements/iron-a11y-keys#^1.0.6"
   }

default.nix

@@ -1,41 +1,40 b''
 # Nix environment for the community edition
 #
-# This shall be as lean as possible, just producing the Enterprise
+# This shall be as lean as possible, just producing the enterprise-ce
 # derivation. For advanced tweaks to pimp up the development environment we use
 # "shell.nix" so that it does not have to clutter this file.
+#
+# Configuration, set values in "~/.nixpkgs/config.nix".
+# example
+#  {
+#    # Thoughts on how to configure the dev environment
+#    rc = {
+#      codeInternalUrl = "https://usr:[e-mail]";
+#      sources = {
+#        rhodecode-vcsserver = "/home/user/work/rhodecode-vcsserver";
+#        rhodecode-enterprise-ce = "/home/user/work/rhodecode-enterprise-ce";
+#        rhodecode-enterprise-ee = "/home/user/work/rhodecode-enterprise-ee";
+#      };
+#    };
+#  }
 
 args@
 { pythonPackages ? "python27Packages"
 , pythonExternalOverrides ? self: super: {}
-, doCheck ? true
+, doCheck ? false
 , ...
 }:
 
 let
-
   # Use nixpkgs from args or import them. We use this indirect approach
   # through args to be able to use the name `pkgs` for our customized packages.
   # Otherwise we will end up with an infinite recursion.
-  nixpkgs = args.pkgs or (import <nixpkgs> { });
+  pkgs = args.pkgs or (import <nixpkgs> { });
 
-  # johbo: Interim bridge which allows us to build with the upcoming
-  # nixos.16.09 branch (unstable at the moment of writing this note) and the
-  # current stable nixos-16.03.
-  backwardsCompatibleFetchgit = { ... }@args:
-    let
-      origSources = nixpkgs.fetchgit args;
-    in
-    nixpkgs.lib.overrideDerivation origSources (oldAttrs: {
-      NIX_PREFETCH_GIT_CHECKOUT_HOOK = ''
-        find $out -name '.git*' -print0 | xargs -0 rm -rf
-      '';
-    });
-
-  # Create a customized version of nixpkgs which should be used throughout the
-  # rest of this file.
-  pkgs = nixpkgs.overridePackages (self: super: {
-    fetchgit = backwardsCompatibleFetchgit;
-  });
+  # Works with the new python-packages, still can fallback to the old
+  # variant.
+  basePythonPackagesUnfix = basePythonPackages.__unfix__ or (
+    self: basePythonPackages.override (a: { inherit self; }));
 
   # Evaluates to the last segment of a file system path.
   basename = path: with pkgs.lib; last (splitString "/" path);
@@ -46,7 +45,7 b' let'
       ext = last (splitString "." path);
     in
       !builtins.elem (basename path) [
-        ".git" ".hg" "__pycache__" ".eggs"
+        ".git" ".hg" "__pycache__" ".eggs" ".idea" ".dev"
         "bower_components" "node_modules"
         "build" "data" "result" "tmp"] &&
       !builtins.elem ext ["egg-info" "pyc"] &&
@@ -54,18 +53,20 b' let'
       # it would still be good to restore it since we want to ignore "result-*".
       !hasPrefix "result" path;
 
-  basePythonPackages = with builtins; if isAttrs pythonPackages
-    then pythonPackages
-    else getAttr pythonPackages pkgs;
+  sources =
+    let
+      inherit (pkgs.lib) all isString attrValues;
+      sourcesConfig = pkgs.config.rc.sources or {};
+    in
+      # Ensure that sources are configured as strings. Using a path
+      # would result in a copy into the nix store.
+      assert all isString (attrValues sourcesConfig);
+      sourcesConfig;
 
-  buildBowerComponents =
-    pkgs.buildBowerComponents or
-    (import ./pkgs/backport-16.03-build-bower-components.nix { inherit pkgs; });
-
-  sources = pkgs.config.rc.sources or {};
-  version = builtins.readFile ./rhodecode/VERSION;
+  version = builtins.readFile "${rhodecode-enterprise-ce-src}/rhodecode/VERSION";
   rhodecode-enterprise-ce-src = builtins.filterSource src-filter ./.;
 
+  buildBowerComponents = pkgs.buildBowerComponents;
   nodeEnv = import ./pkgs/node-default.nix {
     inherit pkgs;
   };
@@ -77,133 +78,145 b' let'
     src = rhodecode-enterprise-ce-src;
   };
 
-  pythonGeneratedPackages = self: basePythonPackages.override (a: {
-    inherit self;
-  })
-  // (scopedImport {
-    self = self;
-    super = basePythonPackages;
-    inherit pkgs;
-    inherit (pkgs) fetchurl fetchgit;
-  } ./pkgs/python-packages.nix);
+  rhodecode-testdata-src = sources.rhodecode-testdata or (
+    pkgs.fetchhg {
+      url = "https://code.rhodecode.com/upstream/rc_testdata";
+      rev = "v0.10.0";
+      sha256 = "0zn9swwvx4vgw4qn8q3ri26vvzgrxn15x6xnjrysi1bwmz01qjl0";
+  });
 
-  pythonOverrides = import ./pkgs/python-packages-overrides.nix {
-    inherit
-      basePythonPackages
-      pkgs;
+  rhodecode-testdata = import "${rhodecode-testdata-src}/default.nix" {
+  inherit
+    doCheck
+    pkgs
+    pythonPackages;
   };
 
   pythonLocalOverrides = self: super: {
     rhodecode-enterprise-ce =
       let
         linkNodeAndBowerPackages = ''
-          echo "Export RhodeCode CE path"
           export RHODECODE_CE_PATH=${rhodecode-enterprise-ce-src}
-          echo "Link node packages"
+
+          echo "[BEGIN]: Link node packages"
           rm -fr node_modules
           mkdir node_modules
           # johbo: Linking individual packages allows us to run "npm install"
           # inside of a shell to try things out. Re-entering the shell will
           # restore a clean environment.
           ln -s ${nodeDependencies}/lib/node_modules/* node_modules/
+          echo "[DONE]: Link node packages"
 
-          echo "DONE: Link node packages"
-
-          echo "Link bower packages"
+          echo "[BEGIN]: Link bower packages"
           rm -fr bower_components
           mkdir bower_components
+          ln -s ${bowerComponents}/bower_components/* bower_components/
+          echo "[DONE]: Link bower packages"
+        '';
 
-          ln -s ${bowerComponents}/bower_components/* bower_components/
-          echo "DONE: Link bower packages"
-        '';
+        releaseName = "RhodeCodeEnterpriseCE-${version}";
       in super.rhodecode-enterprise-ce.override (attrs: {
-
       inherit
         doCheck
         version;
+
       name = "rhodecode-enterprise-ce-${version}";
-      releaseName = "RhodeCodeEnterpriseCE-${version}";
+      releaseName = releaseName;
       src = rhodecode-enterprise-ce-src;
       dontStrip = true; # prevent strip, we don't need it.
 
-      buildInputs =
-        attrs.buildInputs ++
-        (with self; [
-          pkgs.nodePackages.bower
-          pkgs.nodePackages.grunt-cli
-          pkgs.subversion
-          rhodecode-testdata
-        ]);
-
-      #TODO: either move this into overrides, OR use the new machanics from
-      # pip2nix and requiremtn.txt file
-      propagatedBuildInputs = attrs.propagatedBuildInputs ++ (with self; [
-        rhodecode-tools
-      ]);
-
-      # TODO: johbo: Make a nicer way to expose the parts. Maybe
-      # pkgs/default.nix?
+      # expose following attributed outside
       passthru = {
         inherit
+          rhodecode-testdata
           bowerComponents
           linkNodeAndBowerPackages
           myPythonPackagesUnfix
-          pythonLocalOverrides;
+          pythonLocalOverrides
+          pythonCommunityOverrides;
+
         pythonPackages = self;
       };
 
+      buildInputs =
+        attrs.buildInputs or [] ++ [
+          rhodecode-testdata
+          pkgs.nodePackages.bower
+          pkgs.nodePackages.grunt-cli
+        ];
+
+      #NOTE: option to inject additional propagatedBuildInputs
+      propagatedBuildInputs =
+        attrs.propagatedBuildInputs or [] ++ [
+
+        ];
+
       LC_ALL = "en_US.UTF-8";
       LOCALE_ARCHIVE =
-        if pkgs.stdenv ? glibc
+        if pkgs.stdenv.isLinux
         then "${pkgs.glibcLocales}/lib/locale/locale-archive"
         else "";
 
+      # Add bin directory to path so that tests can find 'rhodecode'.
       preCheck = ''
         export PATH="$out/bin:$PATH"
       '';
 
+      # custom check phase for testing
+      checkPhase = ''
+        runHook preCheck
+        PYTHONHASHSEED=random py.test -vv -p no:sugar -r xw --cov-config=.coveragerc --cov=rhodecode --cov-report=term-missing rhodecode
+        runHook postCheck
+      '';
+
       postCheck = ''
-        rm -rf $out/lib/${self.python.libPrefix}/site-packages/pytest_pylons
+        echo "Cleanup of rhodecode/tests"
         rm -rf $out/lib/${self.python.libPrefix}/site-packages/rhodecode/tests
       '';
 
-      preBuild = linkNodeAndBowerPackages + ''
+      preBuild = ''
+
+        echo "Building frontend assets"
+        ${linkNodeAndBowerPackages}
         grunt
         rm -fr node_modules
       '';
 
       postInstall = ''
-        echo "Writing meta information for rccontrol to nix-support/rccontrol"
+        echo "Writing enterprise-ce meta information for rccontrol to nix-support/rccontrol"
         mkdir -p $out/nix-support/rccontrol
         cp -v rhodecode/VERSION $out/nix-support/rccontrol/version
-        echo "DONE: Meta information for rccontrol written"
+        echo "[DONE]: enterprise-ce meta information for rccontrol written"
+
+        mkdir -p $out/etc
+        cp configs/production.ini $out/etc
+        echo "[DONE]: saved enterprise-ce production.ini into $out/etc"
 
         # python based programs need to be wrapped
-        #ln -s ${self.python}/bin/* $out/bin/
+        mkdir -p $out/bin
+        # rhodecode-tools
+        ln -s ${self.rhodecode-tools}/bin/rhodecode-* $out/bin/
+
+        # required binaries from dependencies
+        #ln -s ${self.python}/bin/python $out/bin
         ln -s ${self.pyramid}/bin/* $out/bin/
         ln -s ${self.gunicorn}/bin/gunicorn $out/bin/
         ln -s ${self.supervisor}/bin/supervisor* $out/bin/
         ln -s ${self.pastescript}/bin/paster $out/bin/
         ln -s ${self.channelstream}/bin/channelstream $out/bin/
         ln -s ${self.celery}/bin/celery $out/bin/
+        echo "[DONE]: created symlinks into $out/bin"
 
-        # rhodecode-tools
-        ln -s ${self.rhodecode-tools}/bin/rhodecode-* $out/bin/
-
-        # note that condition should be restricted when adding further tools
         for file in $out/bin/*;
         do
           wrapProgram $file \
-              --prefix PATH : $PATH \
-              --prefix PYTHONPATH : $PYTHONPATH \
-              --set PYTHONHASHSEED random
+            --prefix PATH : $PATH \
+            --prefix PYTHONPATH : $PYTHONPATH \
+            --set PYTHONHASHSEED random
         done
 
-        mkdir $out/etc
-        cp configs/production.ini $out/etc
+        echo "[DONE]: enterprise-ce binary wrapping"
 
-
-        # TODO: johbo: Make part of ac-tests
         if [ ! -f rhodecode/public/js/scripts.js ]; then
           echo "Missing scripts.js"
           exit 1
@@ -213,31 +226,33 b' let'
           exit 1
         fi
       '';
-
     });
 
-    rhodecode-testdata = import "${rhodecode-testdata-src}/default.nix" {
-    inherit
-      doCheck
-      pkgs
-      pythonPackages;
-    };
-
   };
 
-  rhodecode-testdata-src = sources.rhodecode-testdata or (
-    pkgs.fetchhg {
-      url = "https://code.rhodecode.com/upstream/rc_testdata";
-      rev = "v0.10.0";
-      sha256 = "0zn9swwvx4vgw4qn8q3ri26vvzgrxn15x6xnjrysi1bwmz01qjl0";
-  });
+  basePythonPackages = with builtins;
+    if isAttrs pythonPackages then
+      pythonPackages
+    else
+      getAttr pythonPackages pkgs;
+
+  pythonGeneratedPackages = import ./pkgs/python-packages.nix {
+    inherit pkgs;
+    inherit (pkgs) fetchurl fetchgit fetchhg;
+  };
+
+  pythonCommunityOverrides = import ./pkgs/python-packages-overrides.nix {
+    inherit pkgs basePythonPackages;
+  };
 
   # Apply all overrides and fix the final package set
   myPythonPackagesUnfix = with pkgs.lib;
     (extends pythonExternalOverrides
     (extends pythonLocalOverrides
-    (extends pythonOverrides
-             pythonGeneratedPackages)));
+    (extends pythonCommunityOverrides
+    (extends pythonGeneratedPackages
+             basePythonPackagesUnfix))));
+
   myPythonPackages = (pkgs.lib.fix myPythonPackagesUnfix);
 
 in myPythonPackages.rhodecode-enterprise-ce

docs/contributing/dev-setup.rst

@@ -47,7 +47,7 b' Switch nix to the latest STABLE channel'
 
 run::
 
-   nix-channel --add https://nixos.org/channels/nixos-16.03 nixpkgs
+   nix-channel --add https://nixos.org/channels/nixos-18.03 nixpkgs
 
 Followed by::
 

package.json

@@ -1,6 +1,13 b''
 {
   "name": "rhodecode-enterprise",
-  "version": "0.0.1",
+  "version": "1.0.0",
+  "private": true,
+  "description" : "RhodeCode JS packaged",
+  "license": "SEE LICENSE IN LICENSE.txt",
+  "repository" : {
+    "type" : "hg",
+    "url" : "https://code.rhodecode.com/rhodecode-enterprise-ce"
+  },
   "devDependencies": {
     "grunt": "^0.4.5",
     "grunt-contrib-copy": "^1.0.0",
@@ -14,7 +21,7 b''
     "grunt-vulcanize": "^1.0.0",
     "node2nix": "^1.0.0",
     "jshint": "^2.9.1-rc3",
-    "bower": "^1.7.9",
+    "bower": "^1.8.4",
     "jquery": "1.11.3",
     "favico.js": "^0.3.10",
     "clipboard": "^1.7.1",

pkgs/bower-packages.nix

@@ -1,33 +1,34 b''
+# Generated by bower2nix v3.2.0 (https://github.com/rvl/bower2nix)
 { fetchbower, buildEnv }:
 buildEnv { name = "bower-env"; ignoreCollisions = true; paths = [
-  (fetchbower "webcomponentsjs" "0.7.22" "^0.7.22" "0ggh3k8ssafd056ib1m5bvzi7cpz3ry7gr5176d79na1w0c3i7dz")
-  (fetchbower "polymer" "Polymer/polymer#1.6.1" "Polymer/polymer#^1.6.1" "09mm0jgk457gvwqlc155swch7gjr6fs3g7spnvhi6vh5b6518540")
-  (fetchbower "paper-button" "PolymerElements/paper-button#1.0.13" "PolymerElements/paper-button#^1.0.13" "0i3y153nqk06pn0gk282vyybnl3g1w3w41d5i9z659cgn27g3fvm")
-  (fetchbower "paper-spinner" "PolymerElements/paper-spinner#1.2.0" "PolymerElements/paper-spinner#^1.2.0" "1av1m6y81jw3hjhz1yqy3rwcgxarjzl58ldfn4q6sn51pgzngfqb")
-  (fetchbower "paper-tooltip" "PolymerElements/paper-tooltip#1.1.3" "PolymerElements/paper-tooltip#^1.1.2" "0vmrm1n8k9sk9nvqy03q177axy22pia6i3j1gxbk72j3pqiqvg6k")
-  (fetchbower "paper-toast" "PolymerElements/paper-toast#1.3.0" "PolymerElements/paper-toast#^1.3.0" "0x9rqxsks5455s8pk4aikpp99ijdn6kxr9gvhwh99nbcqdzcxq1m")
-  (fetchbower "paper-toggle-button" "PolymerElements/paper-toggle-button#1.2.0" "PolymerElements/paper-toggle-button#^1.2.0" "0mphcng3ngspbpg4jjn0mb91nvr4xc1phq3qswib15h6sfww1b2w")
-  (fetchbower "iron-ajax" "PolymerElements/iron-ajax#1.4.4" "PolymerElements/iron-ajax#^1.4.4" "0jpi7ik3zljw8yh2ccc85r26lcpzmkc2nl1kn6fqdx57zkzk9v5b")
-  (fetchbower "iron-autogrow-textarea" "PolymerElements/iron-autogrow-textarea#1.0.13" "PolymerElements/iron-autogrow-textarea#^1.0.13" "0zwhpl97vii1s8k0lgain8i9dnw29b0mxc5ixdscx9las13n2lqq")
-  (fetchbower "iron-a11y-keys" "PolymerElements/iron-a11y-keys#1.0.6" "PolymerElements/iron-a11y-keys#^1.0.6" "1xz3mgghfcxixq28sdb654iaxj4nyi1bzcwf77ydkms6fviqs9mv")
-  (fetchbower "iron-flex-layout" "PolymerElements/iron-flex-layout#1.3.1" "PolymerElements/iron-flex-layout#^1.0.0" "0nswv3ih3bhflgcd2wjfmddqswzgqxb2xbq65jk9w3rkj26hplbl")
-  (fetchbower "paper-behaviors" "PolymerElements/paper-behaviors#1.0.12" "PolymerElements/paper-behaviors#^1.0.0" "012bqk97awgz55cn7rm9g7cckrdhkqhls3zvp8l6nd4rdwcrdzq8")
-  (fetchbower "paper-material" "PolymerElements/paper-material#1.0.6" "PolymerElements/paper-material#^1.0.0" "0rljmknfdbm5aabvx9pk77754zckj3l127c3mvnmwkpkkr353xnh")
-  (fetchbower "paper-styles" "PolymerElements/paper-styles#1.1.4" "PolymerElements/paper-styles#^1.0.0" "0j8vg74xrcxlni8i93dsab3y80f34kk30lv4yblqpkp9c3nrilf7")
-  (fetchbower "neon-animation" "PolymerElements/neon-animation#1.2.4" "PolymerElements/neon-animation#^1.0.0" "16mz9i2n5w0k5j8d6gha23cnbdgm5syz3fawyh89gdbq97bi2q5j")
-  (fetchbower "iron-a11y-announcer" "PolymerElements/iron-a11y-announcer#1.0.5" "PolymerElements/iron-a11y-announcer#^1.0.0" "0n7c7j1pwk3835s7s2jd9125wdcsqf216yi5gj07wn5s8h8p7m9d")
-  (fetchbower "iron-overlay-behavior" "PolymerElements/iron-overlay-behavior#1.8.6" "PolymerElements/iron-overlay-behavior#^1.0.9" "14brn9gz6qqskarg3fxk91xs7vg02vgcsz9a9743kidxr0l0413m")
-  (fetchbower "iron-fit-behavior" "PolymerElements/iron-fit-behavior#1.2.5" "PolymerElements/iron-fit-behavior#^1.1.0" "1msnlh8lp1xg6v4h6dkjwj9kzac5q5q208ayla3x9hi483ki6rlf")
-  (fetchbower "iron-checked-element-behavior" "PolymerElements/iron-checked-element-behavior#1.0.5" "PolymerElements/iron-checked-element-behavior#^1.0.0" "0l0yy4ah454s8bzfv076s8by7h67zy9ni6xb932qwyhx8br6c1m7")
-  (fetchbower "promise-polyfill" "polymerlabs/promise-polyfill#1.0.1" "polymerlabs/promise-polyfill#^1.0.0" "045bj2caav3famr5hhxgs1dx7n08r4s46mlzwb313vdy17is38xb")
-  (fetchbower "iron-behaviors" "PolymerElements/iron-behaviors#1.0.17" "PolymerElements/iron-behaviors#^1.0.0" "021qvkmbk32jrrmmphpmwgby4bzi5jyf47rh1bxmq2ip07ly4bpr")
-  (fetchbower "iron-validatable-behavior" "PolymerElements/iron-validatable-behavior#1.1.1" "PolymerElements/iron-validatable-behavior#^1.0.0" "1yhxlvywhw2klbbgm3f3cmanxfxggagph4ii635zv0c13707wslv")
-  (fetchbower "iron-form-element-behavior" "PolymerElements/iron-form-element-behavior#1.0.6" "PolymerElements/iron-form-element-behavior#^1.0.0" "0rdhxivgkdhhz2yadgdbjfc70l555p3y83vjh8rfj5hr0asyn6q1")
-  (fetchbower "iron-a11y-keys-behavior" "polymerelements/iron-a11y-keys-behavior#1.1.9" "polymerelements/iron-a11y-keys-behavior#^1.0.0" "1imm4gc84qizihhbyhfa8lwjh3myhj837f79i5m04xjgwrjmkaf6")
-  (fetchbower "paper-ripple" "PolymerElements/paper-ripple#1.0.8" "PolymerElements/paper-ripple#^1.0.0" "0r9sq8ik7wwrw0qb82c3rw0c030ljwd3s466c9y4qbcrsbvfjnns")
-  (fetchbower "font-roboto" "PolymerElements/font-roboto#1.0.1" "PolymerElements/font-roboto#^1.0.1" "02jz43r0wkyr3yp7rq2rc08l5cwnsgca9fr54sr4rhsnl7cjpxrj")
-  (fetchbower "iron-meta" "PolymerElements/iron-meta#1.1.2" "PolymerElements/iron-meta#^1.0.0" "1wl4dx8fnsknw9z9xi8bpc4cy9x70c11x4zxwxnj73hf3smifppl")
-  (fetchbower "iron-resizable-behavior" "PolymerElements/iron-resizable-behavior#1.0.5" "PolymerElements/iron-resizable-behavior#^1.0.0" "1fd5zmbr2hax42vmcasncvk7lzi38fmb1kyii26nn8pnnjak7zkn")
-  (fetchbower "iron-selector" "PolymerElements/iron-selector#1.5.2" "PolymerElements/iron-selector#^1.0.0" "1ajv46llqzvahm5g6g75w7nfyjcslp53ji0wm96l2k94j87spv3r")
-  (fetchbower "web-animations-js" "web-animations/web-animations-js#2.2.2" "web-animations/web-animations-js#^2.2.0" "1izfvm3l67vwys0bqbhidi9rqziw2f8wv289386sc6jsxzgkzhga")
+  (fetchbower "webcomponentsjs" "0.7.24" "^0.7.22" "0d6hfsd51n7qykzci9cw5vlsrajvffaf5ic3azlp2rfz76m651qj")
+  (fetchbower "polymer" "Polymer/polymer#1.11.3" "Polymer/polymer#^1.6.1" "0n11ag2pmczw5yv3m76bh0a7hvicqvcaiv7knixx1r704pw107s6")
+  (fetchbower "paper-button" "PolymerElements/paper-button#1.0.15" "PolymerElements/paper-button#^1.0.13" "0zabrp8p4s9md1hlwg0rqmbx0k87a41lsg9pzk747hcb349gblg0")
+  (fetchbower "paper-spinner" "PolymerElements/paper-spinner#1.2.1" "PolymerElements/paper-spinner#^1.2.0" "0d6xc9fd2ipcli7w77yrn1k0z9j373c9y1f16db2840cyb4rvii8")
+  (fetchbower "paper-tooltip" "PolymerElements/paper-tooltip#1.1.4" "PolymerElements/paper-tooltip#^1.1.2" "0j8s09dxqql8mgnvb7x382scq98xk2vjgylk06bsd1gphp3d3qzm")
+  (fetchbower "paper-toast" "PolymerElements/paper-toast#1.3.1" "PolymerElements/paper-toast#^1.3.0" "1s0csv8dwgdyg4psq1zrd6vivlpsgzi4sjqllwqmlwhfnxfl5ql4")
+  (fetchbower "paper-toggle-button" "PolymerElements/paper-toggle-button#1.3.0" "PolymerElements/paper-toggle-button#^1.2.0" "0hvv2y406lzlrkkcmv9nnd99bmcgcrhcx86q3axxv8k3580gqq97")
+  (fetchbower "iron-ajax" "PolymerElements/iron-ajax#1.4.4" "PolymerElements/iron-ajax#^1.4.4" "0vs4dqcw5y02kj11ivzs901s5nwn97fk01xz2jmpy2fgh6l9q5yr")
+  (fetchbower "iron-autogrow-textarea" "PolymerElements/iron-autogrow-textarea#1.0.15" "PolymerElements/iron-autogrow-textarea#^1.0.13" "1jw40ki5w21il0i9pwjywk4y6mk9lrj8fm57vfg9nlpbiqm2vswb")
+  (fetchbower "iron-a11y-keys" "PolymerElements/iron-a11y-keys#1.0.9" "PolymerElements/iron-a11y-keys#^1.0.6" "07c2wm1p9g52qidl67a43yb7pzd88ygycgghlwzjbh2vkwrs40kp")
+  (fetchbower "iron-flex-layout" "PolymerElements/iron-flex-layout#1.3.9" "PolymerElements/iron-flex-layout#^1.0.0" "1r54la4n8n0lq97vdxnlpdrarxsiwp2b3vfvby9il3j4y4s8vi4h")
+  (fetchbower "paper-behaviors" "PolymerElements/paper-behaviors#1.0.13" "PolymerElements/paper-behaviors#^1.0.0" "0yljykkdg9p67dinplmp6hc5ma6sp95ykah8kz6id5z8gjmsd05b")
+  (fetchbower "paper-material" "PolymerElements/paper-material#1.0.7" "PolymerElements/paper-material#^1.0.0" "1q9r3i5f61y6hmd18h3fcmn7y29yznraz83f9256z8cc0vglfjdb")
+  (fetchbower "paper-styles" "PolymerElements/paper-styles#1.3.1" "PolymerElements/paper-styles#^1.0.0" "11fcxp9kx6sqp2yq0883psn8xyw5d3i753mimqbx8aqa5abvrk4q")
+  (fetchbower "neon-animation" "PolymerElements/neon-animation#1.2.5" "PolymerElements/neon-animation#^1.0.0" "144sq9ijw1nnp2jagpa1ammrc018kp1y6nlmgq1v1iishv4ylsl5")
+  (fetchbower "iron-a11y-announcer" "PolymerElements/iron-a11y-announcer#1.0.6" "PolymerElements/iron-a11y-announcer#^1.0.0" "1az02v91s17v9bir868pifv0s2lwxchm0i4l20176f98366813zk")
+  (fetchbower "iron-overlay-behavior" "PolymerElements/iron-overlay-behavior#1.10.4" "PolymerElements/iron-overlay-behavior#^1.0.9" "0px6s756cgqzxzq53fgk1297j07gyfykqkhdzmj9fwyyrwiv1g8z")
+  (fetchbower "iron-fit-behavior" "PolymerElements/iron-fit-behavior#1.2.7" "PolymerElements/iron-fit-behavior#^1.1.0" "0b864x9cdxadvzkdcn1d3yvi32kqccfv8j467337rgcq193l60jb")
+  (fetchbower "iron-checked-element-behavior" "PolymerElements/iron-checked-element-behavior#1.0.6" "PolymerElements/iron-checked-element-behavior#^1.0.0" "165nh5vkdsr4d5vkq4sj1sz0igfy6vgx844ys164mqg8gkjncvxm")
+  (fetchbower "promise-polyfill" "polymerlabs/promise-polyfill#1.0.1" "polymerlabs/promise-polyfill#^1.0.0" "0warxr9fk2d3cvgiq81djxwiky73gxxr5s3xa97d3c83q1lidlzy")
+  (fetchbower "iron-behaviors" "PolymerElements/iron-behaviors#1.0.18" "PolymerElements/iron-behaviors#^1.0.0" "1icx212kh1cfdvk3v7pdqic9c48pdhnknq8ajx0dlps1l5sm69xh")
+  (fetchbower "iron-validatable-behavior" "PolymerElements/iron-validatable-behavior#1.1.2" "PolymerElements/iron-validatable-behavior#^1.0.0" "1gjjn08y5s43p7aizif24i2yvkd1sasy77dix62irkwzig3favqr")
+  (fetchbower "iron-form-element-behavior" "PolymerElements/iron-form-element-behavior#1.0.7" "PolymerElements/iron-form-element-behavior#^1.0.0" "02qbxqsqxjzy086cqbv6pgibmi0888q757p95ig6x3nc62xk81an")
+  (fetchbower "iron-a11y-keys-behavior" "PolymerElements/iron-a11y-keys-behavior#1.1.9" "PolymerElements/iron-a11y-keys-behavior#^1.0.0" "0dy1ca1nb9v9y968q998vgs147fkmn4irnnrdfl40ln1bln45qx9")
+  (fetchbower "paper-ripple" "PolymerElements/paper-ripple#1.0.10" "PolymerElements/paper-ripple#^1.0.0" "0d5gjf7cw7hk520h6xnav1xrpd948qc8mzvqgqycqkad4j4vdck7")
+  (fetchbower "font-roboto" "PolymerElements/font-roboto#1.1.0" "PolymerElements/font-roboto#^1.0.1" "0z4msvaa5pnr84j2r957g313fmdbdbrknhdw1axy5g48845yv04s")
+  (fetchbower "iron-meta" "PolymerElements/iron-meta#1.1.3" "PolymerElements/iron-meta#^1.0.0" "13lsj648ibkyw3lha6g6r7afmk4yxvgdi63bkpy54wx63gfx7xir")
+  (fetchbower "iron-resizable-behavior" "PolymerElements/iron-resizable-behavior#1.0.6" "PolymerElements/iron-resizable-behavior#^1.0.0" "1r8qk670nigqpw50x1m5yvbx7p9sahiwlf0f9z71v508d63vrbi1")
+  (fetchbower "iron-selector" "PolymerElements/iron-selector#1.5.3" "PolymerElements/iron-selector#^1.0.0" "1362pq6vy113h4y6hn31hhp52hh8g269s5aj7vsq266v7y59igf6")
+  (fetchbower "web-animations-js" "web-animations/web-animations-js#2.3.1" "web-animations/web-animations-js#^2.2.0" "16haz886711qrcf9h9wrjwf5hrz2c69l4jxlq0iyzar823c51qkq")
 ]; }

pkgs/node-default.nix

@@ -1,12 +1,14 b''
-# This file has been generated by node2nix 1.0.0. Do not edit!
+# This file has been generated by node2nix 1.5.3. Do not edit!
 
 {pkgs ? import <nixpkgs> {
     inherit system;
-  }, system ? builtins.currentSystem}:
+  }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-6_x"}:
 
 let
   nodeEnv = import ./node-env.nix {
-    inherit (pkgs) stdenv python utillinux runCommand writeTextFile nodejs;
+    inherit (pkgs) stdenv python2 utillinux runCommand writeTextFile;
+    inherit nodejs;
+    libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null;
   };
 in
 import ./node-packages.nix {

pkgs/node-env.nix

@@ -1,24 +1,26 b''
 # This file originates from node2nix
 
-{stdenv, python, nodejs, utillinux, runCommand, writeTextFile}:
+{stdenv, nodejs, python2, utillinux, libtool, runCommand, writeTextFile}:
 
 let
+  python = if nodejs ? python then nodejs.python else python2;
+
   # Create a tar wrapper that filters all the 'Ignoring unknown extended header keyword' noise
   tarWrapper = runCommand "tarWrapper" {} ''
     mkdir -p $out/bin
-    
+
     cat > $out/bin/tar <<EOF
     #! ${stdenv.shell} -e
     $(type -p tar) "\$@" --warning=no-unknown-keyword
     EOF
-    
+
     chmod +x $out/bin/tar
   '';
-  
+
   # Function that generates a TGZ file from a NPM project
   buildNodeSourceDist =
     { name, version, src, ... }:
-    
+
     stdenv.mkDerivation {
       name = "node-tarball-${name}-${version}";
       inherit src;
@@ -42,151 +44,311 b' let'
           # Bundle the dependencies of the package
           mkdir -p node_modules
           cd node_modules
-          
+
           # Only include dependencies if they don't exist. They may also be bundled in the package.
           if [ ! -e "${dependency.name}" ]
           then
               ${composePackage dependency}
           fi
-          
+
           cd ..
         ''
       ) dependencies);
 
   # Recursively composes the dependencies of a package
   composePackage = { name, packageName, src, dependencies ? [], ... }@args:
-    let
-      fixImpureDependencies = writeTextFile {
-        name = "fixDependencies.js";
-        text = ''
-          var fs = require('fs');
-          var url = require('url');
-          
-          /*
-           * Replaces an impure version specification by *
-           */
-          function replaceImpureVersionSpec(versionSpec) {
-              var parsedUrl = url.parse(versionSpec);
-              
-              if(versionSpec == "latest" || versionSpec == "unstable" ||
-                  versionSpec.substr(0, 2) == ".." || dependency.substr(0, 2) == "./" || dependency.substr(0, 2) == "~/" || dependency.substr(0, 1) == '/')
-                  return '*';
-              else if(parsedUrl.protocol == "git:" || parsedUrl.protocol == "git+ssh:" || parsedUrl.protocol == "git+http:" || parsedUrl.protocol == "git+https:" ||
-                  parsedUrl.protocol == "http:" || parsedUrl.protocol == "https:")
-                  return '*';
-              else
-                  return versionSpec;
-          }
-      
-          var packageObj = JSON.parse(fs.readFileSync('./package.json'));
-      
-          /* Replace dependencies */
-          if(packageObj.dependencies !== undefined) {
-              for(var dependency in packageObj.dependencies) {
-                  var versionSpec = packageObj.dependencies[dependency];
-                  packageObj.dependencies[dependency] = replaceImpureVersionSpec(versionSpec);
-              }
-          }
-          
-          /* Replace development dependencies */
-          if(packageObj.devDependencies !== undefined) {
-              for(var dependency in packageObj.devDependencies) {
-                  var versionSpec = packageObj.devDependencies[dependency];
-                  packageObj.devDependencies[dependency] = replaceImpureVersionSpec(versionSpec);
-              }
-          }
-          
-          /* Replace optional dependencies */
-          if(packageObj.optionalDependencies !== undefined) {
-              for(var dependency in packageObj.optionalDependencies) {
-                  var versionSpec = packageObj.optionalDependencies[dependency];
-                  packageObj.optionalDependencies[dependency] = replaceImpureVersionSpec(versionSpec);
-              }
-          }
-          
-          /* Write the fixed JSON file */
-          fs.writeFileSync("package.json", JSON.stringify(packageObj));
-        '';
-      };
-    in
     ''
       DIR=$(pwd)
       cd $TMPDIR
-      
+
       unpackFile ${src}
-      
+
       # Make the base dir in which the target dependency resides first
       mkdir -p "$(dirname "$DIR/${packageName}")"
 
       if [ -f "${src}" ]
       then
           # Figure out what directory has been unpacked
-          packageDir=$(find . -type d -maxdepth 1 | tail -1)
-          
+          packageDir="$(find . -maxdepth 1 -type d | tail -1)"
+
           # Restore write permissions to make building work
+          find "$packageDir" -type d -print0 | xargs -0 chmod u+x
           chmod -R u+w "$packageDir"
-          
+
           # Move the extracted tarball into the output folder
           mv "$packageDir" "$DIR/${packageName}"
       elif [ -d "${src}" ]
       then
+          # Get a stripped name (without hash) of the source directory.
+          # On old nixpkgs it's already set internally.
+          if [ -z "$strippedName" ]
+          then
+              strippedName="$(stripHash ${src})"
+          fi
+
           # Restore write permissions to make building work
-           chmod -R u+w $strippedName
-           
-           # Move the extracted directory into the output folder
-           mv $strippedName "$DIR/${packageName}"
+          chmod -R u+w "$strippedName"
+
+          # Move the extracted directory into the output folder
+          mv "$strippedName" "$DIR/${packageName}"
       fi
-      
+
       # Unset the stripped name to not confuse the next unpack step
       unset strippedName
-      
-      # Some version specifiers (latest, unstable, URLs, file paths) force NPM to make remote connections or consult paths outside the Nix store.
-      # The following JavaScript replaces these by * to prevent that
+
+      # Include the dependencies of the package
       cd "$DIR/${packageName}"
-      node ${fixImpureDependencies}
-      
-      # Include the dependencies of the package
       ${includeDependencies { inherit dependencies; }}
       cd ..
       ${stdenv.lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."}
     '';
 
+  pinpointDependencies = {dependencies, production}:
+    let
+      pinpointDependenciesFromPackageJSON = writeTextFile {
+        name = "pinpointDependencies.js";
+        text = ''
+          var fs = require('fs');
+          var path = require('path');
+
+          function resolveDependencyVersion(location, name) {
+              if(location == process.env['NIX_STORE']) {
+                  return null;
+              } else {
+                  var dependencyPackageJSON = path.join(location, "node_modules", name, "package.json");
+
+                  if(fs.existsSync(dependencyPackageJSON)) {
+                      var dependencyPackageObj = JSON.parse(fs.readFileSync(dependencyPackageJSON));
+
+                      if(dependencyPackageObj.name == name) {
+                          return dependencyPackageObj.version;
+                      }
+                  } else {
+                      return resolveDependencyVersion(path.resolve(location, ".."), name);
+                  }
+              }
+          }
+
+          function replaceDependencies(dependencies) {
+              if(typeof dependencies == "object" && dependencies !== null) {
+                  for(var dependency in dependencies) {
+                      var resolvedVersion = resolveDependencyVersion(process.cwd(), dependency);
+
+                      if(resolvedVersion === null) {
+                          process.stderr.write("WARNING: cannot pinpoint dependency: "+dependency+", context: "+process.cwd()+"\n");
+                      } else {
+                          dependencies[dependency] = resolvedVersion;
+                      }
+                  }
+              }
+          }
+
+          /* Read the package.json configuration */
+          var packageObj = JSON.parse(fs.readFileSync('./package.json'));
+
+          /* Pinpoint all dependencies */
+          replaceDependencies(packageObj.dependencies);
+          if(process.argv[2] == "development") {
+              replaceDependencies(packageObj.devDependencies);
+          }
+          replaceDependencies(packageObj.optionalDependencies);
+
+          /* Write the fixed package.json file */
+          fs.writeFileSync("package.json", JSON.stringify(packageObj, null, 2));
+        '';
+      };
+    in
+    ''
+      node ${pinpointDependenciesFromPackageJSON} ${if production then "production" else "development"}
+
+      ${stdenv.lib.optionalString (dependencies != [])
+        ''
+          if [ -d node_modules ]
+          then
+              cd node_modules
+              ${stdenv.lib.concatMapStrings (dependency: pinpointDependenciesOfPackage dependency) dependencies}
+              cd ..
+          fi
+        ''}
+    '';
+
+  # Recursively traverses all dependencies of a package and pinpoints all
+  # dependencies in the package.json file to the versions that are actually
+  # being used.
+
+  pinpointDependenciesOfPackage = { packageName, dependencies ? [], production ? true, ... }@args:
+    ''
+      if [ -d "${packageName}" ]
+      then
+          cd "${packageName}"
+          ${pinpointDependencies { inherit dependencies production; }}
+          cd ..
+          ${stdenv.lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."}
+      fi
+    '';
+
   # Extract the Node.js source code which is used to compile packages with
   # native bindings
   nodeSources = runCommand "node-sources" {} ''
     tar --no-same-owner --no-same-permissions -xf ${nodejs.src}
     mv node-* $out
   '';
-  
+
+  # Script that adds _integrity fields to all package.json files to prevent NPM from consulting the cache (that is empty)
+  addIntegrityFieldsScript = writeTextFile {
+    name = "addintegrityfields.js";
+    text = ''
+      var fs = require('fs');
+      var path = require('path');
+
+      function augmentDependencies(baseDir, dependencies) {
+          for(var dependencyName in dependencies) {
+              var dependency = dependencies[dependencyName];
+
+              // Open package.json and augment metadata fields
+              var packageJSONDir = path.join(baseDir, "node_modules", dependencyName);
+              var packageJSONPath = path.join(packageJSONDir, "package.json");
+
+              if(fs.existsSync(packageJSONPath)) { // Only augment packages that exist. Sometimes we may have production installs in which development dependencies can be ignored
+                  console.log("Adding metadata fields to: "+packageJSONPath);
+                  var packageObj = JSON.parse(fs.readFileSync(packageJSONPath));
+
+                  if(dependency.integrity) {
+                      packageObj["_integrity"] = dependency.integrity;
+                  } else {
+                      packageObj["_integrity"] = "sha1-000000000000000000000000000="; // When no _integrity string has been provided (e.g. by Git dependencies), add a dummy one. It does not seem to harm and it bypasses downloads.
+                  }
+
+                  packageObj["_resolved"] = dependency.version; // Set the resolved version to the version identifier. This prevents NPM from cloning Git repositories.
+                  fs.writeFileSync(packageJSONPath, JSON.stringify(packageObj, null, 2));
+              }
+
+              // Augment transitive dependencies
+              if(dependency.dependencies !== undefined) {
+                  augmentDependencies(packageJSONDir, dependency.dependencies);
+              }
+          }
+      }
+
+      if(fs.existsSync("./package-lock.json")) {
+          var packageLock = JSON.parse(fs.readFileSync("./package-lock.json"));
+
+          if(packageLock.lockfileVersion !== 1) {
+             process.stderr.write("Sorry, I only understand lock file version 1!\n");
+             process.exit(1);
+          }
+
+          if(packageLock.dependencies !== undefined) {
+              augmentDependencies(".", packageLock.dependencies);
+          }
+      }
+    '';
+  };
+
+  # Reconstructs a package-lock file from the node_modules/ folder structure and package.json files with dummy sha1 hashes
+  reconstructPackageLock = writeTextFile {
+    name = "addintegrityfields.js";
+    text = ''
+      var fs = require('fs');
+      var path = require('path');
+
+      var packageObj = JSON.parse(fs.readFileSync("package.json"));
+
+      var lockObj = {
+          name: packageObj.name,
+          version: packageObj.version,
+          lockfileVersion: 1,
+          requires: true,
+          dependencies: {}
+      };
+
+      function augmentPackageJSON(filePath, dependencies) {
+          var packageJSON = path.join(filePath, "package.json");
+          if(fs.existsSync(packageJSON)) {
+              var packageObj = JSON.parse(fs.readFileSync(packageJSON));
+              dependencies[packageObj.name] = {
+                  version: packageObj.version,
+                  integrity: "sha1-000000000000000000000000000=",
+                  dependencies: {}
+              };
+              processDependencies(path.join(filePath, "node_modules"), dependencies[packageObj.name].dependencies);
+          }
+      }
+
+      function processDependencies(dir, dependencies) {
+          if(fs.existsSync(dir)) {
+              var files = fs.readdirSync(dir);
+
+              files.forEach(function(entry) {
+                  var filePath = path.join(dir, entry);
+                  var stats = fs.statSync(filePath);
+
+                  if(stats.isDirectory()) {
+                      if(entry.substr(0, 1) == "@") {
+                          // When we encounter a namespace folder, augment all packages belonging to the scope
+                          var pkgFiles = fs.readdirSync(filePath);
+
+                          pkgFiles.forEach(function(entry) {
+                              if(stats.isDirectory()) {
+                                  var pkgFilePath = path.join(filePath, entry);
+                                  augmentPackageJSON(pkgFilePath, dependencies);
+                              }
+                          });
+                      } else {
+                          augmentPackageJSON(filePath, dependencies);
+                      }
+                  }
+              });
+          }
+      }
+
+      processDependencies("node_modules", lockObj.dependencies);
+
+      fs.writeFileSync("package-lock.json", JSON.stringify(lockObj, null, 2));
+    '';
+  };
+
   # Builds and composes an NPM package including all its dependencies
-  buildNodePackage = { name, packageName, version, dependencies ? [], production ? true, npmFlags ? "", dontNpmInstall ? false, preRebuild ? "", ... }@args:
-    
+  buildNodePackage = { name, packageName, version, dependencies ? [], production ? true, npmFlags ? "", dontNpmInstall ? false, bypassCache ? false, preRebuild ? "", ... }@args:
+
+    let
+      forceOfflineFlag = if bypassCache then "--offline" else "--registry http://www.example.com";
+    in
     stdenv.lib.makeOverridable stdenv.mkDerivation (builtins.removeAttrs args [ "dependencies" ] // {
       name = "node-${name}-${version}";
-      buildInputs = [ tarWrapper python nodejs ] ++ stdenv.lib.optional (stdenv.isLinux) utillinux ++ args.buildInputs or [];
+      buildInputs = [ tarWrapper python nodejs ]
+        ++ stdenv.lib.optional (stdenv.isLinux) utillinux
+        ++ stdenv.lib.optional (stdenv.isDarwin) libtool
+        ++ args.buildInputs or [];
       dontStrip = args.dontStrip or true; # Striping may fail a build for some package deployments
-      
+
       inherit dontNpmInstall preRebuild;
-      
+
       unpackPhase = args.unpackPhase or "true";
-      
+
       buildPhase = args.buildPhase or "true";
-      
+
       compositionScript = composePackage args;
-      passAsFile = [ "compositionScript" ];
-      
+      pinpointDependenciesScript = pinpointDependenciesOfPackage args;
+
+      passAsFile = [ "compositionScript" "pinpointDependenciesScript" ];
+
       installPhase = args.installPhase or ''
         # Create and enter a root node_modules/ folder
         mkdir -p $out/lib/node_modules
         cd $out/lib/node_modules
-          
+
         # Compose the package and all its dependencies
         source $compositionScriptPath
-        
+
+        # Pinpoint the versions of all dependencies to the ones that are actually being used
+        echo "pinpointing versions of dependencies..."
+        source $pinpointDependenciesScriptPath
+
         # Patch the shebangs of the bundled modules to prevent them from
         # calling executables outside the Nix store as much as possible
         patchShebangs .
-        
+
         # Deploy the Node.js package by running npm install. Since the
         # dependencies have been provided already by ourselves, it should not
         # attempt to install them again, which is good, because we want to make
@@ -196,23 +358,37 b' let'
         #
         # The other responsibilities of NPM are kept -- version checks, build
         # steps, postprocessing etc.
-        
+
         export HOME=$TMPDIR
         cd "${packageName}"
         runHook preRebuild
-        npm --registry http://www.example.com --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} rebuild
-        
+
+        ${stdenv.lib.optionalString bypassCache ''
+          if [ ! -f package-lock.json ]
+          then
+              echo "No package-lock.json file found, reconstructing..."
+              node ${reconstructPackageLock}
+          fi
+
+          node ${addIntegrityFieldsScript}
+        ''}
+
+        npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} rebuild
+
         if [ "$dontNpmInstall" != "1" ]
         then
-            npm --registry http://www.example.com --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} install
+            # NPM tries to download packages even when they already exist if npm-shrinkwrap is used.
+            rm -f npm-shrinkwrap.json
+
+            npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} install
         fi
-        
+
         # Create symlink to the deployed executable folder, if applicable
         if [ -d "$out/lib/node_modules/.bin" ]
         then
             ln -s $out/lib/node_modules/.bin $out/bin
         fi
-        
+
         # Create symlinks to the deployed manual page folders, if applicable
         if [ -d "$out/lib/node_modules/${packageName}/man" ]
         then
@@ -226,51 +402,86 b' let'
                 done
             done
         fi
+
+        # Run post install hook, if provided
+        runHook postInstall
       '';
     });
 
   # Builds a development shell
-  buildNodeShell = { name, packageName, version, src, dependencies ? [], production ? true, npmFlags ? "", dontNpmInstall ? false, ... }@args:
+  buildNodeShell = { name, packageName, version, src, dependencies ? [], production ? true, npmFlags ? "", dontNpmInstall ? false, bypassCache ? false, ... }@args:
     let
+      forceOfflineFlag = if bypassCache then "--offline" else "--registry http://www.example.com";
+
       nodeDependencies = stdenv.mkDerivation {
         name = "node-dependencies-${name}-${version}";
-        
-        buildInputs = [ tarWrapper python nodejs ] ++ stdenv.lib.optional (stdenv.isLinux) utillinux ++ args.buildInputs or [];
-        
+
+        buildInputs = [ tarWrapper python nodejs ]
+          ++ stdenv.lib.optional (stdenv.isLinux) utillinux
+          ++ stdenv.lib.optional (stdenv.isDarwin) libtool
+          ++ args.buildInputs or [];
+
         includeScript = includeDependencies { inherit dependencies; };
-        passAsFile = [ "includeScript" ];
-        
+        pinpointDependenciesScript = pinpointDependenciesOfPackage args;
+
+        passAsFile = [ "includeScript" "pinpointDependenciesScript" ];
+
         buildCommand = ''
-          mkdir -p $out/lib
-          cd $out/lib
+          mkdir -p $out/${packageName}
+          cd $out/${packageName}
+
           source $includeScriptPath
-          
+
           # Create fake package.json to make the npm commands work properly
-          cat > package.json <<EOF
-          {
-              "name": "${packageName}",
-              "version": "${version}"
-          }
-          EOF
-          
+          cp ${src}/package.json .
+          chmod 644 package.json
+          ${stdenv.lib.optionalString bypassCache ''
+            if [ -f ${src}/package-lock.json ]
+            then
+              cp ${src}/package-lock.json .
+            fi
+          ''}
+
+          # Pinpoint the versions of all dependencies to the ones that are actually being used
+          echo "pinpointing versions of dependencies..."
+          cd ..
+          source $pinpointDependenciesScriptPath
+          cd ${packageName}
+
           # Patch the shebangs of the bundled modules to prevent them from
           # calling executables outside the Nix store as much as possible
           patchShebangs .
-          
-          export HOME=$TMPDIR
-          npm --registry http://www.example.com --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} rebuild
-          
-          ${stdenv.lib.optionalString (!dontNpmInstall) ''
-            npm --registry http://www.example.com --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} install
+
+          export HOME=$PWD
+
+          ${stdenv.lib.optionalString bypassCache ''
+            if [ ! -f package-lock.json ]
+            then
+                echo "No package-lock.json file found, reconstructing..."
+                node ${reconstructPackageLock}
+            fi
+
+            node ${addIntegrityFieldsScript}
           ''}
 
+          npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} rebuild
+
+          ${stdenv.lib.optionalString (!dontNpmInstall) ''
+            # NPM tries to download packages even when they already exist if npm-shrinkwrap is used.
+            rm -f npm-shrinkwrap.json
+
+            npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} install
+          ''}
+
+          cd ..
+          mv ${packageName} lib
           ln -s $out/lib/node_modules/.bin $out/bin
         '';
       };
     in
     stdenv.lib.makeOverridable stdenv.mkDerivation {
       name = "node-shell-${name}-${version}";
-      
+
       buildInputs = [ python nodejs ] ++ stdenv.lib.optional (stdenv.isLinux) utillinux ++ args.buildInputs or [];
       buildCommand = ''
         mkdir -p $out/bin
@@ -281,7 +492,7 b' let'
         EOF
         chmod +x $out/bin/shell
       '';
-      
+
       # Provide the dependencies in a development shell through the NODE_PATH environment variable
       inherit nodeDependencies;
       shellHook = stdenv.lib.optionalString (dependencies != []) ''

pkgs/python-packages-overrides.nix

@@ -4,10 +4,13 b''
 # python-packages.nix. The main objective is to add needed dependencies of C
 # libraries and tweak the build instructions where needed.
 
-{ pkgs, basePythonPackages }:
+{ pkgs
+, basePythonPackages
+}:
 
 let
   sed = "sed -i";
+
   localLicenses = {
     repoze = {
       fullName = "Repoze License";
@@ -19,33 +22,33 b' in'
 
 self: super: {
 
-  appenlight-client = super.appenlight-client.override (attrs: {
+  "appenlight-client" = super."appenlight-client".override (attrs: {
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   });
 
-  beaker = super.beaker.override (attrs: {
+  "beaker" = super."beaker".override (attrs: {
     patches = [
-      ./patch-beaker-lock-func-debug.diff
-      ./patch-beaker-metadata-reuse.diff
+      ./patch_beaker/patch-beaker-lock-func-debug.diff
+      ./patch_beaker/patch-beaker-metadata-reuse.diff
     ];
   });
 
-  future = super.future.override (attrs: {
+  "future" = super."future".override (attrs: {
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   });
 
-  testpath = super.testpath.override (attrs: {
+  "testpath" = super."testpath".override (attrs: {
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   });
 
-  gnureadline = super.gnureadline.override (attrs: {
-    buildInputs = attrs.buildInputs ++ [
+  "gnureadline" = super."gnureadline".override (attrs: {
+    buildInputs = [
       pkgs.ncurses
     ];
     patchPhase = ''
@@ -53,56 +56,50 b' self: super: {'
     '';
   });
 
-  gunicorn = super.gunicorn.override (attrs: {
-    propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
+  "gunicorn" = super."gunicorn".override (attrs: {
+    propagatedBuildInputs = [
       # johbo: futures is needed as long as we are on Python 2, otherwise
       # gunicorn explodes if used with multiple threads per worker.
-      self.futures
+      self."futures"
     ];
   });
 
-  nbconvert = super.nbconvert.override (attrs: {
+ "nbconvert" = super."nbconvert".override (attrs: {
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
       # marcink: plug in jupyter-client for notebook rendering
-      self.jupyter-client
+      self."jupyter-client"
     ];
   });
 
-  ipython = super.ipython.override (attrs: {
+  "ipython" = super."ipython".override (attrs: {
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
-      self.gnureadline
+      self."gnureadline"
     ];
   });
 
-  lxml = super.lxml.override (attrs: {
-    # johbo: On 16.09 we need this to compile on darwin, otherwise compilation
-    # fails on Darwin.
-    hardeningDisable = if pkgs.stdenv.isDarwin then [ "format" ] else null;
-    buildInputs = with self; [
+  "lxml" = super."lxml".override (attrs: {
+    buildInputs = [
       pkgs.libxml2
       pkgs.libxslt
     ];
+    propagatedBuildInputs = [
+      # Needed, so that "setup.py bdist_wheel" does work
+      self."wheel"
+    ];
   });
 
-  mysql-python = super.mysql-python.override (attrs: {
-    buildInputs = attrs.buildInputs ++ [
+  "mysql-python" = super."mysql-python".override (attrs: {
+    buildInputs = [
       pkgs.openssl
     ];
-    propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
+    propagatedBuildInputs = [
       pkgs.libmysql
       pkgs.zlib
     ];
   });
 
-  psutil = super.psutil.override (attrs: {
-    buildInputs = attrs.buildInputs ++
-      pkgs.lib.optional pkgs.stdenv.isDarwin pkgs.darwin.IOKit;
-  });
-
-  psycopg2 = super.psycopg2.override (attrs: {
-    buildInputs = attrs.buildInputs ++
-      pkgs.lib.optional pkgs.stdenv.isDarwin pkgs.openssl;
-    propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
+  "psycopg2" = super."psycopg2".override (attrs: {
+    propagatedBuildInputs = [
       pkgs.postgresql
     ];
     meta = {
@@ -110,8 +107,8 b' self: super: {'
     };
   });
 
-  pycurl = super.pycurl.override (attrs: {
-    propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
+  "pycurl" = super."pycurl".override (attrs: {
+    propagatedBuildInputs = [
       pkgs.curl
       pkgs.openssl
     ];
@@ -120,30 +117,23 b' self: super: {'
       export PYCURL_SSL_LIBRARY=openssl
     '';
     meta = {
-      # TODO: It is LGPL and MIT
       license = pkgs.lib.licenses.mit;
     };
   });
 
-  pyramid = super.pyramid.override (attrs: {
-    postFixup = ''
-      wrapPythonPrograms
-      # TODO: johbo: "wrapPython" adds this magic line which
-      # confuses pserve.
-      ${sed} '/import sys; sys.argv/d' $out/bin/.pserve-wrapped
-    '';
+  "pyramid" = super."pyramid".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
-  pyramid-debugtoolbar = super.pyramid-debugtoolbar.override (attrs: {
+  "pyramid-debugtoolbar" = super."pyramid-debugtoolbar".override (attrs: {
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal localLicenses.repoze ];
     };
   });
 
-  pysqlite = super.pysqlite.override (attrs: {
+  "pysqlite" = super."pysqlite".override (attrs: {
     propagatedBuildInputs = [
       pkgs.sqlite
     ];
@@ -152,41 +142,39 b' self: super: {'
     };
   });
 
-  pytest-runner = super.pytest-runner.override (attrs: {
+  "pytest-runner" = super."pytest-runner".override (attrs: {
     propagatedBuildInputs = [
-      self.setuptools-scm
+      self."setuptools-scm"
+    ];
+  });
+
+  "python-ldap" = super."python-ldap".override (attrs: {
+    propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
+      pkgs.openldap
+      pkgs.cyrus_sasl
+      pkgs.openssl
     ];
   });
 
-  python-ldap = super.python-ldap.override (attrs: {
-    propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
-      pkgs.cyrus_sasl
-      pkgs.openldap
-      pkgs.openssl
+  "python-pam" = super."python-pam".override (attrs: {
+    propagatedBuildInputs = [
+        pkgs.pam
     ];
-    # TODO: johbo: Remove the "or" once we drop 16.03 support.
-    NIX_CFLAGS_COMPILE = "-I${pkgs.cyrus_sasl.dev or pkgs.cyrus_sasl}/include/sasl";
+    # TODO: johbo: Check if this can be avoided, or transform into
+    # a real patch
+    patchPhase = ''
+      substituteInPlace pam.py \
+        --replace 'find_library("pam")' '"${pkgs.pam}/lib/libpam.so.0"'
+    '';
+    });
+
+  "pyzmq" = super."pyzmq".override (attrs: {
+    buildInputs = [
+      pkgs.czmq
+    ];
   });
 
-  python-pam = super.python-pam.override (attrs:
-    let
-      includeLibPam = pkgs.stdenv.isLinux;
-    in {
-      # TODO: johbo: Move the option up into the default.nix, we should
-      # include python-pam only on supported platforms.
-      propagatedBuildInputs = attrs.propagatedBuildInputs ++
-        pkgs.lib.optional includeLibPam [
-          pkgs.pam
-        ];
-      # TODO: johbo: Check if this can be avoided, or transform into
-      # a real patch
-      patchPhase = pkgs.lib.optionals includeLibPam ''
-        substituteInPlace pam.py \
-          --replace 'find_library("pam")' '"${pkgs.pam}/lib/libpam.so.0"'
-      '';
-    });
-
-  urlobject = super.urlobject.override (attrs: {
+  "urlobject" = super."urlobject".override (attrs: {
     meta = {
       license = {
         spdxId = "Unlicense";
@@ -196,56 +184,56 b' self: super: {'
     };
   });
 
-  docutils = super.docutils.override (attrs: {
+  "docutils" = super."docutils".override (attrs: {
     meta = {
       license = pkgs.lib.licenses.bsd2;
     };
   });
 
-  colander = super.colander.override (attrs: {
+  "colander" = super."colander".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
-  pyramid-beaker  = super.pyramid-beaker.override (attrs: {
+  "pyramid-beaker"  = super."pyramid-beaker".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
-  pyramid-mako = super.pyramid-mako.override (attrs: {
+  "pyramid-mako" = super."pyramid-mako".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
-  repoze.lru = super.repoze.lru.override (attrs: {
+  "repoze.lru" = super."repoze.lru".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
-  python-editor = super.python-editor.override (attrs: {
+  "python-editor" = super."python-editor".override (attrs: {
     meta = {
       license = pkgs.lib.licenses.asl20;
     };
   });
 
-  translationstring = super.translationstring.override (attrs: {
+  "translationstring" = super."translationstring".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
-  venusian = super.venusian.override (attrs: {
+  "venusian" = super."venusian".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
-  # Avoid that setuptools is replaced, this leads to trouble
-  # with buildPythonPackage.
-  setuptools = basePythonPackages.setuptools;
+  # Avoid that base packages screw up the build process
+  inherit (basePythonPackages)
+    setuptools;
 
 }

release.nix

@@ -1,232 +1,16 b''
-#
-# About
-# =====
-#
-# This file defines jobs for our CI system and the attribute "build" is used
-# as the input for packaging.
-#
-#
-# CI details
-# ==========
-#
-# This file defines an attribute set of derivations. Each of these attributes is
-# then used in our CI system as one job to run. This way we keep the
-# configuration for the CI jobs as well under version control.
-#
-# Run CI jobs locally
-# -------------------
-#
-# Since it is all based on normal Nix derivations, the jobs can be tested
-# locally with a run of "nix-build" like the following example:
-#
-#    nix-build release.nix -A test-api -I vcsserver=~/rhodecode-vcsserver
-#
-# Note: Replace "~/rhodecode-vcsserver" with a path where a clone of the
-# vcsserver resides.
+# This file defines how to "build" for packaging.
 
 { pkgs ? import <nixpkgs> {}
 , doCheck ? true
 }:
 
 let
-
-  inherit (pkgs)
-    stdenv
-    system;
-
-  testing = import <nixpkgs/nixos/lib/testing.nix> {
-    inherit system;
-  };
-
-  runInMachine = testing.runInMachine;
-
-  sphinx = import ./docs/default.nix {};
-
-  mkDocs = kind: stdenv.mkDerivation {
-    name = kind;
-    srcs = [
-      (./. + (builtins.toPath "/${kind}"))
-      (builtins.filterSource
-        (path: type: baseNameOf path == "VERSION")
-        ./rhodecode)
-    ];
-    sourceRoot = kind;
-    buildInputs = [ sphinx ];
-    configurePhase = null;
-    buildPhase = ''
-      make SPHINXBUILD=sphinx-build html
-    '';
-    installPhase = ''
-      mkdir -p $out
-      mv _build/html $out/
-
-      mkdir -p $out/nix-support
-      echo "doc manual $out/html index.html" >> \
-        "$out/nix-support/hydra-build-products"
-    '';
-  };
-
-  enterprise = import ./default.nix {
+  enterprise_ce = import ./default.nix {
     inherit
+      doCheck
       pkgs;
-
-    # TODO: for quick local testing
-    doCheck = false;
-  };
-
-  test-cfg = stdenv.mkDerivation {
-    name = "test-cfg";
-    unpackPhase = "true";
-    buildInputs = [
-      enterprise.src
-    ];
-    installPhase = ''
-      mkdir -p $out/etc
-      cp ${enterprise.src}/test.ini $out/etc/enterprise.ini
-      # TODO: johbo: Needed, so that the login works, this causes
-      # probably some side effects
-      substituteInPlace $out/etc/enterprise.ini --replace "is_test = True" ""
-
-      # Gevent configuration
-      cp $out/etc/enterprise.ini $out/etc/enterprise-gevent.ini;
-      cat >> $out/etc/enterprise-gevent.ini <<EOF
-
-      [server:main]
-      use = egg:gunicorn#main
-      worker_class = gevent
-      EOF
-
-      cp ${enterprise.src}/vcsserver/test.ini $out/etc/vcsserver.ini
-    '';
-  };
-
-  ac-test-drv = import ./acceptance_tests {
-    withExternals = false;
   };
 
-  # TODO: johbo: Currently abusing buildPythonPackage to make the
-  # needed environment for the ac-test tools.
-  mkAcTests = {
-    # Path to an INI file which will be used to run Enterprise.
-    #
-    # Intended usage is to provide different configuration files to
-    # run the tests against a different configuration.
-    enterpriseCfg ? "${test-cfg}/etc/enterprise.ini"
-
-    # Path to an INI file which will be used to run the VCSServer.
-  , vcsserverCfg ? "${test-cfg}/etc/vcsserver.ini"
-  }: pkgs.pythonPackages.buildPythonPackage {
-    name = "enterprise-ac-tests";
-    src = ./acceptance_tests;
-
-    buildInputs = with pkgs; [
-      curl
-      enterprise
-      ac-test-drv
-    ];
-
-    buildPhase = ''
-      cp ${enterpriseCfg} enterprise.ini
-
-      echo "Creating a fake home directory"
-      mkdir fake-home
-      export HOME=$PWD/fake-home
-
-      echo "Creating a repository directory"
-      mkdir repos
-
-      echo "Preparing the database"
-      rc-setup-app \
-        --user=admin \
-        --email=[e-mail] \
-        --password=secret \
-        --api-key=9999999999999999999999999999999999999999 \
-        --force-yes \
-        --repos=$PWD/repos \
-        enterprise.ini > /dev/null
-
-      echo "Starting rc-server"
-      vcsserver --config ${vcsserverCfg} >vcsserver.log 2>&1 &
-      rc-server enterprise.ini >rc-server.log 2>&1 &
-
-      while ! curl -f -s http://localhost:5000 > /dev/null
-      do
-          echo "Waiting for server to be ready..."
-          sleep 3
-      done
-      echo "Webserver is ready."
-
-      echo "Starting the test run"
-      py.test -c example.ini -vs --maxfail=5 tests
-
-      echo "Kill rc-server"
-      kill %2
-      kill %1
-    '';
-
-    # TODO: johbo: Use the install phase again once the normal mkDerivation
-    # can be used again.
-    postInstall = ''
-      mkdir -p $out
-      cp enterprise.ini $out
-      cp ${vcsserverCfg} $out/vcsserver.ini
-      cp rc-server.log $out
-      cp vcsserver.log $out
-
-      mkdir -p $out/nix-support
-      echo "report config $out enterprise.ini" >> $out/nix-support/hydra-build-products
-      echo "report config $out vcsserver.ini" >> $out/nix-support/hydra-build-products
-      echo "report rc-server $out rc-server.log" >> $out/nix-support/hydra-build-products
-      echo "report vcsserver $out vcsserver.log" >> $out/nix-support/hydra-build-products
-    '';
-  };
-
-  vcsserver = import <vcsserver> {
-    inherit pkgs;
-
-    # TODO: johbo: Think of a more elegant solution to this problem
-    pythonExternalOverrides = self: super: (enterprise.myPythonPackagesUnfix self);
-  };
-
-  runTests = optionString: (enterprise.override (attrs: {
-    doCheck = true;
-    name = "test-run";
-    buildInputs = attrs.buildInputs ++ [
-      vcsserver
-    ];
-    checkPhase = ''
-      py.test ${optionString} -vv -ra
-    '';
-    buildPhase = attrs.shellHook;
-    installPhase = ''
-      echo "Intentionally not installing anything"
-    '';
-    meta.description = "Enterprise test run ${optionString}";
-  }));
-
-  jobs = {
-
-    build = enterprise;
-
-    # johbo: Currently this is simply running the tests against the sources. Nicer
-    # would be to run xdist and against the installed application, so that we also
-    # cover the impact of installing the application.
-    test-api = runTests "rhodecode/api";
-    test-functional = runTests "rhodecode/tests/functional";
-    test-rest = runTests "rhodecode/tests --ignore=rhodecode/tests/functional";
-    test-full = runTests "rhodecode";
-
-    docs = mkDocs "docs";
-
-    aggregate = pkgs.releaseTools.aggregate {
-      name = "aggregated-jobs";
-      constituents = [
-        jobs.build
-        jobs.test-api
-        jobs.test-rest
-        jobs.docs
-      ];
-    };
-  };
-
-in jobs
+in {
+  build = enterprise_ce;
+}

requirements.txt

@@ -1,7 +1,6 b''
-## core
-setuptools==39.2.0
-setuptools-scm==1.15.6
+## dependencies
 
+setuptools-scm==2.1.0
 amqp==2.3.1
 authomatic==0.1.0.post1
 babel==1.3
@@ -59,7 +58,7 b' pyramid-mako==1.0.2'
 pyramid==1.9.2
 pysqlite==2.8.3
 python-dateutil
-python-ldap==2.4.45
+python-ldap==3.1.0
 python-memcached==1.59
 python-pam==1.8.2
 pytz==2018.4

rhodecode/config/licenses.json

@@ -310,11 +310,7 b''
   }, 
   "python2.7-requests-2.9.1": {
     "Apache License 2.0": "http://spdx.org/licenses/Apache-2.0"
-  }, 
-  "python2.7-setuptools-19.4": {
-    "Python Software Foundation License version 2": "http://spdx.org/licenses/Python-2.0", 
-    "Zope Public License 2.0": "http://spdx.org/licenses/ZPL-2.0"
-  }, 
+  },
   "python2.7-setuptools-scm-1.15.6": {
     "MIT License": "http://spdx.org/licenses/MIT"
   }, 

shell.nix

@@ -1,6 +1,9 b''
+# This file contains the adjustments which are desired for a development
+# environment.
+
 { pkgs ? (import <nixpkgs> {})
 , pythonPackages ? "python27Packages"
-, doCheck ? true
+, doCheck ? false
 , sourcesOverrides ? {}
 , doDevelopInstall ? true
 }:
@@ -10,7 +13,10 b' let'
   sources = (pkgs.config.rc.sources or {}) // sourcesOverrides;
 
   enterprise-ce = import ./default.nix {
-    inherit pkgs pythonPackages doCheck;
+    inherit
+      pkgs
+      pythonPackages
+      doCheck;
   };
 
   ce-pythonPackages = enterprise-ce.pythonPackages;
@@ -22,14 +28,17 b' let'
     let
       path = pkgs.lib.attrByPath [attributeName] null sources;
       doIt = doDevelopInstall && path != null;
+
     in
-      pkgs.lib.optionalString doIt (
-      builtins.trace "Develop install of ${attributeName} from ${path}" ''
-        echo "Develop install of '${attributeName}' from '${path}' [BEGIN]"
+      # do develop installation with empty hosts to skip any package duplicates to
+      # be replaced. This only pushes the package to be locally available
+      pkgs.lib.optionalString doIt (''
+        echo "[BEGIN] Develop install of '${attributeName}' from '${path}'"
         pushd ${path}
         python setup.py develop --prefix $tmp_path --allow-hosts ""
         popd
-        echo "Develop install of '${attributeName}' from '${path}' [DONE]"
+        echo "[DONE] Develop install of '${attributeName}' from '${path}'"
+        echo ""
       '');
 
   # This method looks up a path from `pkgs.config.rc.sources` and imports the
@@ -38,13 +47,16 b' let'
   optionalDevelopInstallBuildInputs = attributeName:
     let
       path = pkgs.lib.attrByPath [attributeName] null sources;
+      doIt = doDevelopInstall && path != null && pkgs.lib.pathExists "${nixFile}";
       nixFile = "${path}/default.nix";
-      doIt = doDevelopInstall && path != null && pkgs.lib.pathExists "${nixFile}";
+
       derivate = import "${nixFile}" {
         inherit doCheck pkgs pythonPackages;
       };
     in
-      pkgs.lib.lists.optionals doIt derivate.propagatedNativeBuildInputs;
+      pkgs.lib.lists.optionals doIt (
+        derivate.propagatedBuildInputs
+      );
 
   developInstalls = [ "rhodecode-vcsserver" ];
 
@@ -53,31 +65,52 b' in enterprise-ce.override (attrs: {'
   # make development a little bit more convenient.
   src = null;
 
+  # Add dependencies which are useful for the development environment.
   buildInputs =
     attrs.buildInputs ++
-    pkgs.lib.lists.concatMap optionalDevelopInstallBuildInputs developInstalls ++
     (with ce-pythonPackages; [
       bumpversion
       invoke
       ipdb
     ]);
 
-  # Somewhat snappier setup of the development environment
-  # TODO: think of supporting a stable path again, so that multiple shells
-  #       can share it.
-  preShellHook = enterprise-ce.linkNodeAndBowerPackages + ''
+  # place to inject some required libs from develop installs
+  propagatedBuildInputs =
+    attrs.propagatedBuildInputs ++
+    pkgs.lib.lists.concatMap optionalDevelopInstallBuildInputs developInstalls;
+
+
+  # Make sure we execute both hooks
+  shellHook = ''
+    runHook preShellHook
+    runHook postShellHook
+  '';
+
+  preShellHook = ''
+    echo "Entering CE-Shell"
+
     # Custom prompt to distinguish from other dev envs.
     export PS1="\n\[\033[1;32m\][CE-shell:\w]$\[\033[0m\] "
 
+    echo "Building frontend assets"
+    ${enterprise-ce.linkNodeAndBowerPackages}
+
     # Setup a temporary directory.
     tmp_path=$(mktemp -d)
     export PATH="$tmp_path/bin:$PATH"
     export PYTHONPATH="$tmp_path/${ce-pythonPackages.python.sitePackages}:$PYTHONPATH"
     mkdir -p $tmp_path/${ce-pythonPackages.python.sitePackages}
 
-    # Develop installations
+    # Develop installation
+    echo "[BEGIN]: develop install of rhodecode-enterprise-ce"
     python setup.py develop --prefix $tmp_path --allow-hosts ""
-    echo "Additional develop installs"
-  '' + pkgs.lib.strings.concatMapStrings optionalDevelopInstall developInstalls;
+  '';
+
+  postShellHook = ''
+    echo "** Additional develop installs **"
+    '' +
+    pkgs.lib.strings.concatMapStrings optionalDevelopInstall developInstalls
+    + ''
+  '';
 
 })