merge with default

MANIFEST.in

@@ -1,52 +1,53 b''
 # top level files
 
 include MANIFEST.in
 include README.rst
 include CHANGES.rst
 include LICENSE.txt
 
 include rhodecode/VERSION
 
 # docs
 recursive-include docs *
 
 # all config files
 recursive-include configs *
 
 # translations
 recursive-include rhodecode/i18n *
 
 # non-python core stuff
 recursive-include rhodecode *.cfg
 recursive-include rhodecode *.json
 recursive-include rhodecode *.ini_tmpl
 recursive-include rhodecode *.sh
 recursive-include rhodecode *.mako
 
 # 502 page
 include rhodecode/public/502.html
 
 # robots
 include rhodecode/public/robots.txt
 
 # images, css
 include rhodecode/public/css/*.css
 include rhodecode/public/images/*.*
 include rhodecode/public/images/ee_features/*.*
 
 # sound files
 include rhodecode/public/sounds/*.mp3
 include rhodecode/public/sounds/*.wav
 
 # fonts
 recursive-include rhodecode/public/fonts/RCIcons *
 
 # js
 recursive-include rhodecode/public/js *
 
 # templates
 recursive-include rhodecode/templates *
 
 # skip any tests files
+recursive-exclude result *
 recursive-exclude rhodecode/tests *
 

Makefile

@@ -1,98 +1,135 b''
-.DEFAULT_GOAL := help
 
 # set by: PATH_TO_OUTDATED_PACKAGES=/some/path/outdated_packages.py
 OUTDATED_PACKAGES = ${PATH_TO_OUTDATED_PACKAGES}
 
 NODE_PATH=./node_modules
 WEBPACK=./node_binaries/webpack
 GRUNT=./node_binaries/grunt
 
 .PHONY: clean
-clean: ## full clean
+## Cleanup compiled and cache py files
+clean:
 	make test-clean
 	find . -type f \( -iname '*.c' -o -iname '*.pyc' -o -iname '*.so' -o -iname '*.orig' \) -exec rm '{}' ';'
 
 
 .PHONY: test
-test: ## run test-clean and tests
+## run test-clean and tests
+test:
 	make test-clean
 	make test-only
 
 
 .PHONY:test-clean
-test-clean:  ## run test-clean and tests
+## run test-clean and tests
+test-clean:
 	rm -rf coverage.xml htmlcov junit.xml pylint.log result
 	find . -type d -name "__pycache__" -prune -exec rm -rf '{}' ';'
 	find . -type f \( -iname '.coverage.*' \) -exec rm '{}' ';'
 
 
 .PHONY: test-only
-test-only: ## run tests
+## Run tests only without cleanup
+test-only:
 	PYTHONHASHSEED=random \
 	py.test -x -vv -r xw -p no:sugar \
-	--cov=rhodecode --cov-report=term-missing --cov-report=html \
-    rhodecode
-
+	--cov-report=term-missing --cov-report=html \
+	--cov=rhodecode rhodecode
 
 .PHONY: test-only-mysql
-test-only-mysql: ## run tests against mysql
+## run tests against mysql
+test-only-mysql:
 	PYTHONHASHSEED=random \
 	py.test -x -vv -r xw -p no:sugar \
-	--cov=rhodecode --cov-report=term-missing --cov-report=html \
+	--cov-report=term-missing --cov-report=html \
     --ini-config-override='{"app:main": {"sqlalchemy.db1.url": "mysql://root:qweqwe@localhost/rhodecode_test?charset=utf8"}}' \
-    rhodecode
+    --cov=rhodecode rhodecode
 
 
 .PHONY: test-only-postgres
-test-only-postgres: ## run tests against postgres
+## run tests against postgres
+test-only-postgres:
 	PYTHONHASHSEED=random \
 	py.test -x -vv -r xw -p no:sugar \
-	--cov=rhodecode --cov-report=term-missing --cov-report=html \
+	--cov-report=term-missing --cov-report=html \
     --ini-config-override='{"app:main": {"sqlalchemy.db1.url": "postgresql://postgres:qweqwe@localhost/rhodecode_test"}}' \
-    rhodecode
+    --cov=rhodecode rhodecode
 
 .PHONY: docs
-docs: ## build docs
+## build docs
+docs:
 	(cd docs; nix-build default.nix -o result; make clean html)
 
 
 .PHONY: docs-clean
-docs-clean: ## Cleanup docs
+## Cleanup docs
+docs-clean:
 	(cd docs; make clean)
 
 
 .PHONY: docs-cleanup
-docs-cleanup: ## Cleanup docs
+## Cleanup docs
+docs-cleanup:
 	(cd docs; make cleanup)
 
 
 .PHONY: web-build
-web-build: ## Build static/js
+## Build JS packages static/js
+web-build:
 	NODE_PATH=$(NODE_PATH) $(GRUNT)
 
 
-.PHONY: generate-pkgs
-generate-pkgs: ## generate new python packages
-	nix-shell pkgs/shell-generate.nix --command "pip2nix generate --licenses"
-
-
 .PHONY: pip-packages
-pip-packages: ## show outdated packages
+## show outdated packages
+pip-packages:
 	python ${OUTDATED_PACKAGES}
 
 
-.PHONY: generate-js-pkgs
-generate-js-pkgs: ## generate js packages
-	rm -rf node_modules && \
-	nix-shell pkgs/shell-generate.nix --command "node2nix --input package.json -o pkgs/node-packages.nix -e pkgs/node-env.nix -c pkgs/node-default.nix -d --flatten --nodejs-8" && \
-	sed -i -e 's/http:\/\//https:\/\//g' pkgs/node-packages.nix
+.PHONY: sdist
+## Build sdist
+sdist:
+	python setup.py sdist
 
 
-.PHONY: generate-license-meta
-generate-license-meta: ## Generate license metadata
-	nix-build pkgs/license-generate.nix -o result-license && \
-	cat result-license/licenses.json | python -m json.tool > rhodecode/config/licenses.json
+# Default command on calling make
+.DEFAULT_GOAL := show-help
 
-.PHONY: help
-help:
-	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-24s\033[0m %s\n", $$1, $$2}'
+.PHONY: show-help
+show-help:
+	@echo "$$(tput bold)Available rules:$$(tput sgr0)"
+	@echo
+	@sed -n -e "/^## / { \
+		h; \
+		s/.*//; \
+		:doc" \
+		-e "H; \
+		n; \
+		s/^## //; \
+		t doc" \
+		-e "s/:.*//; \
+		G; \
+		s/\\n## /---/; \
+		s/\\n/ /g; \
+		p; \
+	}" ${MAKEFILE_LIST} \
+	| LC_ALL='C' sort --ignore-case \
+	| awk -F '---' \
+		-v ncol=$$(tput cols) \
+		-v indent=19 \
+		-v col_on="$$(tput setaf 6)" \
+		-v col_off="$$(tput sgr0)" \
+	'{ \
+		printf "%s%*s%s ", col_on, -indent, $$1, col_off; \
+		n = split($$2, words, " "); \
+		line_length = ncol - indent; \
+		for (i = 1; i <= n; i++) { \
+			line_length -= length(words[i]) + 1; \
+			if (line_length <= 0) { \
+				line_length = ncol - indent - length(words[i]) - 1; \
+				printf "\n%*s ", -indent, " "; \
+			} \
+			printf "%s ", words[i]; \
+		} \
+		printf "\n"; \
+	}'
+

default.nix

@@ -1,296 +1,302 b''
 # Nix environment for the community edition
 #
 # This shall be as lean as possible, just producing the enterprise-ce
 # derivation. For advanced tweaks to pimp up the development environment we use
 # "shell.nix" so that it does not have to clutter this file.
 #
 # Configuration, set values in "~/.nixpkgs/config.nix".
 # example
 #  {
 #    # Thoughts on how to configure the dev environment
 #    rc = {
 #      codeInternalUrl = "https://usr:[e-mail]/internal";
 #      sources = {
 #        rhodecode-vcsserver = "/home/user/work/rhodecode-vcsserver";
 #        rhodecode-enterprise-ce = "/home/user/work/rhodecode-enterprise-ce";
 #        rhodecode-enterprise-ee = "/home/user/work/rhodecode-enterprise-ee";
 #      };
 #    };
 #  }
 
 args@
 { system ? builtins.currentSystem
 , pythonPackages ? "python27Packages"
 , pythonExternalOverrides ? self: super: {}
 , doCheck ? false
 , ...
 }:
 
 let
   pkgs_ = args.pkgs or (import <nixpkgs> { inherit system; });
 in
 
 let
   pkgs = import <nixpkgs> {
     overlays = [
       (import ./pkgs/overlays.nix)
     ];
     inherit
       (pkgs_)
       system;
   };
 
   # Works with the new python-packages, still can fallback to the old
   # variant.
   basePythonPackagesUnfix = basePythonPackages.__unfix__ or (
     self: basePythonPackages.override (a: { inherit self; }));
 
   # Evaluates to the last segment of a file system path.
   basename = path: with pkgs.lib; last (splitString "/" path);
+  startsWith = prefix: full: let
+    actualPrefix = builtins.substring 0 (builtins.stringLength prefix) full;
+  in actualPrefix == prefix;
 
   # source code filter used as arugment to builtins.filterSource.
   src-filter = path: type: with pkgs.lib;
     let
       ext = last (splitString "." path);
+      parts = last (splitString "/" path);
     in
       !builtins.elem (basename path) [
         ".git" ".hg" "__pycache__" ".eggs" ".idea" ".dev"
         "node_modules" "node_binaries"
         "build" "data" "result" "tmp"] &&
       !builtins.elem ext ["egg-info" "pyc"] &&
-      # TODO: johbo: This check is wrong, since "path" contains an absolute path,
-      # it would still be good to restore it since we want to ignore "result-*".
-      !hasPrefix "result" path;
+      !startsWith "result" (basename path);
 
   sources =
     let
       inherit
         (pkgs.lib)
         all
         isString
         attrValues;
+
       sourcesConfig = pkgs.config.rc.sources or {};
     in
       # Ensure that sources are configured as strings. Using a path
       # would result in a copy into the nix store.
       assert all isString (attrValues sourcesConfig);
       sourcesConfig;
 
+  rhodecode-enterprise-ce-src = builtins.filterSource src-filter ./.;
   version = builtins.readFile "${rhodecode-enterprise-ce-src}/rhodecode/VERSION";
-  rhodecode-enterprise-ce-src = builtins.filterSource src-filter ./.;
 
   nodeEnv = import ./pkgs/node-default.nix {
     inherit
       pkgs
       system;
   };
   nodeDependencies = nodeEnv.shell.nodeDependencies;
 
   rhodecode-testdata-src = sources.rhodecode-testdata or (
     pkgs.fetchhg {
       url = "https://code.rhodecode.com/upstream/rc_testdata";
       rev = "v0.10.0";
       sha256 = "0zn9swwvx4vgw4qn8q3ri26vvzgrxn15x6xnjrysi1bwmz01qjl0";
   });
 
   rhodecode-testdata = import "${rhodecode-testdata-src}/default.nix" {
   inherit
     doCheck
     pkgs
     pythonPackages;
   };
 
   pythonLocalOverrides = self: super: {
     rhodecode-enterprise-ce =
       let
         linkNodePackages = ''
           export RHODECODE_CE_PATH=${rhodecode-enterprise-ce-src}
 
           echo "[BEGIN]: Link node packages and binaries"
           # johbo: Linking individual packages allows us to run "npm install"
           # inside of a shell to try things out. Re-entering the shell will
           # restore a clean environment.
           rm -fr node_modules
           mkdir node_modules
           ln -s ${nodeDependencies}/lib/node_modules/* node_modules/
           export NODE_PATH=./node_modules
 
           rm -fr node_binaries
           mkdir node_binaries
           ln -s ${nodeDependencies}/bin/* node_binaries/
           echo "[DONE ]: Link node packages and binaries"
         '';
 
         releaseName = "RhodeCodeEnterpriseCE-${version}";
+        pythonWithEnv =
+            self.python.buildEnv.override {
+                extraLibs = [  ] ++ self.rhodecode-enterprise-ce.propagatedBuildInputs;
+                ignoreCollisions = true;
+                #--set PYTHONHASHSEED random TODO
+            };
       in super.rhodecode-enterprise-ce.override (attrs: {
       inherit
         doCheck
         version;
 
       name = "rhodecode-enterprise-ce-${version}";
       releaseName = releaseName;
       src = rhodecode-enterprise-ce-src;
       dontStrip = true; # prevent strip, we don't need it.
 
       # expose following attributed outside
       passthru = {
         inherit
           rhodecode-testdata
           linkNodePackages
           myPythonPackagesUnfix
           pythonLocalOverrides
           pythonCommunityOverrides;
 
         pythonPackages = self;
+        rc_pkgs = pkgs;
       };
 
       buildInputs =
         attrs.buildInputs or [] ++ [
           rhodecode-testdata
         ];
 
       #NOTE: option to inject additional propagatedBuildInputs
       propagatedBuildInputs =
         attrs.propagatedBuildInputs or [] ++ [
 
-        ];
+      ];
+
+      preBuild = ''
+        export NIX_PATH=nixpkgs=${pkgs.path}
+        export SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt
 
-      LC_ALL = "en_US.UTF-8";
-      LOCALE_ARCHIVE =
-        if pkgs.stdenv.isLinux
-        then "${pkgs.glibcLocales}/lib/locale/locale-archive"
-        else "";
+        echo "[BEGIN]: Building frontend assets"
+        ${linkNodePackages}
+        make web-build
+        rm -fr node_modules
+        rm -fr node_binaries
+        echo "[DONE ]: Building frontend assets"
+      '';
 
       # Add bin directory to path so that tests can find 'rhodecode'.
       preCheck = ''
+        echo "Expanding PATH with $out/bin directory"
         export PATH="$out/bin:$PATH"
       '';
 
       # custom check phase for testing
       checkPhase = ''
         runHook preCheck
         PYTHONHASHSEED=random py.test -vv -p no:sugar -r xw --cov-config=.coveragerc --cov=rhodecode --cov-report=term-missing rhodecode
         runHook postCheck
       '';
 
       postCheck = ''
         echo "Cleanup of rhodecode/tests"
         rm -rf $out/lib/${self.python.libPrefix}/site-packages/rhodecode/tests
       '';
 
-      preBuild = ''
-        echo "[BEGIN]: Building frontend assets"
-        ${linkNodePackages}
-        make web-build
-        rm -fr node_modules
-        rm -fr node_binaries
-        echo "[DONE ]: Building frontend assets"
-      '';
-
       postInstall = ''
         # check required files
         STATIC_CHECK="/robots.txt /502.html
                       /js/scripts.min.js /js/rhodecode-components.js
                       /css/style.css /css/style-polymer.css /css/style-ipython.css"
 
         for file in $STATIC_CHECK;
         do
             if [ ! -f rhodecode/public/$file ]; then
               echo "Missing $file"
               exit 1
             fi
         done
 
         echo "Writing enterprise-ce meta information for rccontrol to nix-support/rccontrol"
         mkdir -p $out/nix-support/rccontrol
         cp -v rhodecode/VERSION $out/nix-support/rccontrol/version
         echo "[DONE ]: enterprise-ce meta information for rccontrol written"
 
         mkdir -p $out/etc
         cp configs/production.ini $out/etc
         echo "[DONE ]: saved enterprise-ce production.ini into $out/etc"
 
+        echo "saving env in $out/etc/env_vars.txt"
+        touch $out/etc/env_vars.txt
+        echo "# RhodeCode build env vars" >> $out/etc/env_vars.txt
+        echo "LOCALE_ARCHIVE=\"${pkgs.glibcLocales}/lib/locale/locale-archive\"" >> $out/etc/env_vars.txt
+        echo "LC_ALL=\"en_US.UTF-8\"" >> $out/etc/env_vars.txt
+
         cp -Rf rhodecode/config/rcextensions $out/etc/rcextensions.tmpl
         echo "[DONE ]: saved enterprise-ce rcextensions into $out/etc/rcextensions.tmpl"
 
         # python based programs need to be wrapped
         mkdir -p $out/bin
 
+        # expose python
+        ln -s ${pythonWithEnv}/bin/python $out/bin/
+
         # required binaries from dependencies
-        ln -s ${self.supervisor}/bin/supervisorctl $out/bin/
-        ln -s ${self.supervisor}/bin/supervisord $out/bin/
-        ln -s ${self.pastescript}/bin/paster $out/bin/
-        ln -s ${self.channelstream}/bin/channelstream $out/bin/
-        ln -s ${self.celery}/bin/celery $out/bin/
-        ln -s ${self.gunicorn}/bin/gunicorn $out/bin/
-        ln -s ${self.pyramid}/bin/prequest $out/bin/
-        ln -s ${self.pyramid}/bin/pserve $out/bin/
+        ln -s ${pythonWithEnv}/bin/supervisorctl $out/bin/
+        ln -s ${pythonWithEnv}/bin/supervisord $out/bin/
+        ln -s ${pythonWithEnv}/bin/paster $out/bin/
+        ln -s ${pythonWithEnv}/bin/channelstream $out/bin/
+        ln -s ${pythonWithEnv}/bin/celery $out/bin/
+        ln -s ${pythonWithEnv}/bin/gunicorn $out/bin/
+        ln -s ${pythonWithEnv}/bin/prequest $out/bin/
+        ln -s ${pythonWithEnv}/bin/pserve $out/bin/
 
         echo "[DONE ]: created symlinks into $out/bin"
-        DEPS="$out/bin/supervisorctl \
-              $out/bin/supervisord \
-              $out/bin/paster \
-              $out/bin/channelstream \
-              $out/bin/celery \
-              $out/bin/gunicorn \
-              $out/bin/prequest \
-              $out/bin/pserve"
+        DEPS="
+              "
 
         # wrap only dependency scripts, they require to have full PYTHONPATH set
         # to be able to import all packages
         for file in $DEPS;
         do
           wrapProgram $file \
             --prefix PATH : $PATH \
             --prefix PYTHONPATH : $PYTHONPATH \
             --set PYTHONHASHSEED random
         done
 
         echo "[DONE ]: enterprise-ce binary wrapping"
 
         # rhodecode-tools don't need wrapping
         ln -s ${self.rhodecode-tools}/bin/rhodecode-* $out/bin/
 
-        # expose sources of CE
-        ln -s $out $out/etc/rhodecode_enterprise_ce_source
-
         # expose static files folder
         cp -Rf $out/lib/${self.python.libPrefix}/site-packages/rhodecode/public/ $out/etc/static
         chmod 755 -R $out/etc/static
 
+        # expose sources of rhodecode-enterprise-ce
+        ln -s $out $out/etc/rhodecode_enterprise_ce_source
       '';
+
     });
+  };
 
-  };
 
   basePythonPackages = with builtins;
     if isAttrs pythonPackages then
       pythonPackages
     else
       getAttr pythonPackages pkgs;
 
   pythonGeneratedPackages = import ./pkgs/python-packages.nix {
-    inherit
-      pkgs;
-    inherit
-      (pkgs)
-      fetchurl
-      fetchgit
-      fetchhg;
+    inherit pkgs;
+    inherit (pkgs) fetchurl fetchgit fetchhg;
   };
 
   pythonCommunityOverrides = import ./pkgs/python-packages-overrides.nix {
     inherit pkgs basePythonPackages;
   };
 
   # Apply all overrides and fix the final package set
   myPythonPackagesUnfix = with pkgs.lib;
     (extends pythonExternalOverrides
     (extends pythonLocalOverrides
     (extends pythonCommunityOverrides
     (extends pythonGeneratedPackages
              basePythonPackagesUnfix))));
 
   myPythonPackages = (pkgs.lib.fix myPythonPackagesUnfix);
 
 in myPythonPackages.rhodecode-enterprise-ce

package.json

@@ -1,62 +1,62 b''
 {
   "name": "rhodecode-enterprise",
-  "version": "2.0.0",
+  "version": "4.19.0",
   "private": true,
   "description" : "RhodeCode JS packaged",
   "license": "SEE LICENSE IN LICENSE.txt",
   "repository" : {
     "type" : "hg",
     "url" : "https://code.rhodecode.com/rhodecode-enterprise-ce"
   },
   "devDependencies": {
     "appenlight-client": "git+https://[e-mail]/AppEnlight/appenlight-client-js.git#0.5.1",
     "clipboard": "^2.0.1",
     "exports-loader": "^0.6.4",
     "favico.js": "^0.3.10",
     "dropzone": "^5.5.0",
     "grunt": "^0.4.5",
     "grunt-cli": "^1.3.1",
     "grunt-contrib-concat": "^0.5.1",
     "grunt-contrib-copy": "^1.0.0",
     "grunt-contrib-jshint": "^0.12.0",
     "grunt-contrib-less": "^1.1.0",
     "grunt-contrib-watch": "^0.6.1",
     "grunt-webpack": "^3.1.3",
     "grunt-contrib-uglify": "^4.0.1",
     "sweetalert2": "^9.10.12",
     "jquery": "1.11.3",
     "mark.js": "8.11.1",
     "jshint": "^2.9.1-rc3",
     "moment": "^2.18.1",
     "mousetrap": "^1.6.1",
     "qrious": "^4.0.2",
     "sticky-sidebar": "3.3.1",
     "waypoints": "4.0.1",
     "webpack": "4.23.1",
     "webpack-cli": "3.1.2",
     "babel-core": "^6.26.3",
     "babel-loader": "^7.1.2",
     "babel-plugin-transform-object-rest-spread": "^6.26.0",
     "babel-preset-env": "^1.6.0",
     "copy-webpack-plugin": "^4.4.2",
     "css-loader": "^0.28.11",
     "html-loader": "^0.4.4",
     "html-webpack-plugin": "^3.2.0",
     "imports-loader": "^0.7.1",
     "polymer-webpack-loader": "^2.0.1",
     "style-loader": "^0.21.0",
     "webpack-uglify-js-plugin": "^1.1.9",
     "raw-loader": "1.0.0-beta.0",
     "ts-loader": "^1.3.3",
     "@webcomponents/webcomponentsjs": "^2.0.0",
     "@polymer/polymer": "^3.0.0",
     "@polymer/paper-button": "^3.0.0",
     "@polymer/paper-spinner": "^3.0.0",
     "@polymer/paper-tooltip": "^3.0.0",
     "@polymer/paper-toast": "^3.0.0",
     "@polymer/paper-toggle-button": "^3.0.0",
     "@polymer/iron-ajax": "^3.0.0",
     "@polymer/iron-autogrow-textarea": "^3.0.0",
     "@polymer/iron-a11y-keys": "^3.0.0"
   }
 }

pkgs/nix-common/pip2nix.nix

@@ -1,17 +1,25 b''
 { pkgs
 , pythonPackages
 }:
 
 rec {
   pip2nix-src = pkgs.fetchzip {
     url = https://github.com/johbo/pip2nix/archive/51e6fdae34d0e8ded9efeef7a8601730249687a6.tar.gz;
     sha256 = "02a4jjgi7lsvf8mhrxsd56s9a3yg20081rl9bgc2m84w60v2gbz2";
   };
 
   pip2nix = import pip2nix-src {
     inherit
       pkgs
       pythonPackages;
   };
 
+  pip-tools = pythonPackages.pip-tools;
+
+  setuptools = pythonPackages.setuptools;
+
+  wheel = pythonPackages.wheel;
+
+  pip = pythonPackages.pip;
+
 }

pkgs/node-default.nix

@@ -1,17 +1,17 b''
-# This file has been generated by node2nix 1.6.0. Do not edit!
+# This file has been generated by node2nix 1.8.0. Do not edit!
 
 {pkgs ? import <nixpkgs> {
     inherit system;
-  }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-8_x"}:
+  }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-12_x"}:
 
 let
   nodeEnv = import ./node-env.nix {
     inherit (pkgs) stdenv python2 utillinux runCommand writeTextFile;
     inherit nodejs;
     libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null;
   };
 in
 import ./node-packages.nix {
   inherit (pkgs) fetchurl fetchgit;
   inherit nodeEnv;
 }
  No newline at end of file

pkgs/node-env.nix

@@ -1,542 +1,542 b''
 # This file originates from node2nix
 
 {stdenv, nodejs, python2, utillinux, libtool, runCommand, writeTextFile}:
 
 let
   python = if nodejs ? python then nodejs.python else python2;
 
   # Create a tar wrapper that filters all the 'Ignoring unknown extended header keyword' noise
   tarWrapper = runCommand "tarWrapper" {} ''
     mkdir -p $out/bin
 
     cat > $out/bin/tar <<EOF
     #! ${stdenv.shell} -e
-    $(type -p tar) "\$@" --warning=no-unknown-keyword
+    $(type -p tar) "\$@" --warning=no-unknown-keyword --delay-directory-restore
     EOF
 
     chmod +x $out/bin/tar
   '';
 
   # Function that generates a TGZ file from a NPM project
   buildNodeSourceDist =
     { name, version, src, ... }:
 
     stdenv.mkDerivation {
       name = "node-tarball-${name}-${version}";
       inherit src;
       buildInputs = [ nodejs ];
       buildPhase = ''
         export HOME=$TMPDIR
         tgzFile=$(npm pack | tail -n 1) # Hooks to the pack command will add output (https://docs.npmjs.com/misc/scripts)
       '';
       installPhase = ''
         mkdir -p $out/tarballs
         mv $tgzFile $out/tarballs
         mkdir -p $out/nix-support
         echo "file source-dist $out/tarballs/$tgzFile" >> $out/nix-support/hydra-build-products
       '';
     };
 
   includeDependencies = {dependencies}:
     stdenv.lib.optionalString (dependencies != [])
       (stdenv.lib.concatMapStrings (dependency:
         ''
           # Bundle the dependencies of the package
           mkdir -p node_modules
           cd node_modules
 
           # Only include dependencies if they don't exist. They may also be bundled in the package.
           if [ ! -e "${dependency.name}" ]
           then
               ${composePackage dependency}
           fi
 
           cd ..
         ''
       ) dependencies);
 
   # Recursively composes the dependencies of a package
   composePackage = { name, packageName, src, dependencies ? [], ... }@args:
-    ''
+    builtins.addErrorContext "while evaluating node package '${packageName}'" ''
       DIR=$(pwd)
       cd $TMPDIR
 
       unpackFile ${src}
 
       # Make the base dir in which the target dependency resides first
       mkdir -p "$(dirname "$DIR/${packageName}")"
 
       if [ -f "${src}" ]
       then
           # Figure out what directory has been unpacked
           packageDir="$(find . -maxdepth 1 -type d | tail -1)"
 
           # Restore write permissions to make building work
-          find "$packageDir" -type d -print0 | xargs -0 chmod u+x
+          find "$packageDir" -type d -exec chmod u+x {} \;
           chmod -R u+w "$packageDir"
 
           # Move the extracted tarball into the output folder
           mv "$packageDir" "$DIR/${packageName}"
       elif [ -d "${src}" ]
       then
           # Get a stripped name (without hash) of the source directory.
           # On old nixpkgs it's already set internally.
           if [ -z "$strippedName" ]
           then
               strippedName="$(stripHash ${src})"
           fi
 
           # Restore write permissions to make building work
           chmod -R u+w "$strippedName"
 
           # Move the extracted directory into the output folder
           mv "$strippedName" "$DIR/${packageName}"
       fi
 
       # Unset the stripped name to not confuse the next unpack step
       unset strippedName
 
       # Include the dependencies of the package
       cd "$DIR/${packageName}"
       ${includeDependencies { inherit dependencies; }}
       cd ..
       ${stdenv.lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."}
     '';
 
   pinpointDependencies = {dependencies, production}:
     let
       pinpointDependenciesFromPackageJSON = writeTextFile {
         name = "pinpointDependencies.js";
         text = ''
           var fs = require('fs');
           var path = require('path');
 
           function resolveDependencyVersion(location, name) {
               if(location == process.env['NIX_STORE']) {
                   return null;
               } else {
                   var dependencyPackageJSON = path.join(location, "node_modules", name, "package.json");
 
                   if(fs.existsSync(dependencyPackageJSON)) {
                       var dependencyPackageObj = JSON.parse(fs.readFileSync(dependencyPackageJSON));
 
                       if(dependencyPackageObj.name == name) {
                           return dependencyPackageObj.version;
                       }
                   } else {
                       return resolveDependencyVersion(path.resolve(location, ".."), name);
                   }
               }
           }
 
           function replaceDependencies(dependencies) {
               if(typeof dependencies == "object" && dependencies !== null) {
                   for(var dependency in dependencies) {
                       var resolvedVersion = resolveDependencyVersion(process.cwd(), dependency);
 
                       if(resolvedVersion === null) {
                           process.stderr.write("WARNING: cannot pinpoint dependency: "+dependency+", context: "+process.cwd()+"\n");
                       } else {
                           dependencies[dependency] = resolvedVersion;
                       }
                   }
               }
           }
 
           /* Read the package.json configuration */
           var packageObj = JSON.parse(fs.readFileSync('./package.json'));
 
           /* Pinpoint all dependencies */
           replaceDependencies(packageObj.dependencies);
           if(process.argv[2] == "development") {
               replaceDependencies(packageObj.devDependencies);
           }
           replaceDependencies(packageObj.optionalDependencies);
 
           /* Write the fixed package.json file */
           fs.writeFileSync("package.json", JSON.stringify(packageObj, null, 2));
         '';
       };
     in
     ''
       node ${pinpointDependenciesFromPackageJSON} ${if production then "production" else "development"}
 
       ${stdenv.lib.optionalString (dependencies != [])
         ''
           if [ -d node_modules ]
           then
               cd node_modules
               ${stdenv.lib.concatMapStrings (dependency: pinpointDependenciesOfPackage dependency) dependencies}
               cd ..
           fi
         ''}
     '';
 
   # Recursively traverses all dependencies of a package and pinpoints all
   # dependencies in the package.json file to the versions that are actually
   # being used.
 
   pinpointDependenciesOfPackage = { packageName, dependencies ? [], production ? true, ... }@args:
     ''
       if [ -d "${packageName}" ]
       then
           cd "${packageName}"
           ${pinpointDependencies { inherit dependencies production; }}
           cd ..
           ${stdenv.lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."}
       fi
     '';
 
   # Extract the Node.js source code which is used to compile packages with
   # native bindings
   nodeSources = runCommand "node-sources" {} ''
     tar --no-same-owner --no-same-permissions -xf ${nodejs.src}
     mv node-* $out
   '';
 
   # Script that adds _integrity fields to all package.json files to prevent NPM from consulting the cache (that is empty)
   addIntegrityFieldsScript = writeTextFile {
     name = "addintegrityfields.js";
     text = ''
       var fs = require('fs');
       var path = require('path');
 
       function augmentDependencies(baseDir, dependencies) {
           for(var dependencyName in dependencies) {
               var dependency = dependencies[dependencyName];
 
               // Open package.json and augment metadata fields
               var packageJSONDir = path.join(baseDir, "node_modules", dependencyName);
               var packageJSONPath = path.join(packageJSONDir, "package.json");
 
               if(fs.existsSync(packageJSONPath)) { // Only augment packages that exist. Sometimes we may have production installs in which development dependencies can be ignored
                   console.log("Adding metadata fields to: "+packageJSONPath);
                   var packageObj = JSON.parse(fs.readFileSync(packageJSONPath));
 
                   if(dependency.integrity) {
                       packageObj["_integrity"] = dependency.integrity;
                   } else {
                       packageObj["_integrity"] = "sha1-000000000000000000000000000="; // When no _integrity string has been provided (e.g. by Git dependencies), add a dummy one. It does not seem to harm and it bypasses downloads.
                   }
 
-                  packageObj["_resolved"] = dependency.version; // Set the resolved version to the version identifier. This prevents NPM from cloning Git repositories.
+                  if(dependency.resolved) {
+                      packageObj["_resolved"] = dependency.resolved; // Adopt the resolved property if one has been provided
+                  } else {
+                      packageObj["_resolved"] = dependency.version; // Set the resolved version to the version identifier. This prevents NPM from cloning Git repositories.
+                  }
+
+                  if(dependency.from !== undefined) { // Adopt from property if one has been provided
+                      packageObj["_from"] = dependency.from;
+                  }
+
                   fs.writeFileSync(packageJSONPath, JSON.stringify(packageObj, null, 2));
               }
 
               // Augment transitive dependencies
               if(dependency.dependencies !== undefined) {
                   augmentDependencies(packageJSONDir, dependency.dependencies);
               }
           }
       }
 
       if(fs.existsSync("./package-lock.json")) {
           var packageLock = JSON.parse(fs.readFileSync("./package-lock.json"));
 
           if(packageLock.lockfileVersion !== 1) {
              process.stderr.write("Sorry, I only understand lock file version 1!\n");
              process.exit(1);
           }
 
           if(packageLock.dependencies !== undefined) {
               augmentDependencies(".", packageLock.dependencies);
           }
       }
     '';
   };
 
   # Reconstructs a package-lock file from the node_modules/ folder structure and package.json files with dummy sha1 hashes
   reconstructPackageLock = writeTextFile {
     name = "addintegrityfields.js";
     text = ''
       var fs = require('fs');
       var path = require('path');
 
       var packageObj = JSON.parse(fs.readFileSync("package.json"));
 
       var lockObj = {
           name: packageObj.name,
           version: packageObj.version,
           lockfileVersion: 1,
           requires: true,
           dependencies: {}
       };
 
       function augmentPackageJSON(filePath, dependencies) {
           var packageJSON = path.join(filePath, "package.json");
           if(fs.existsSync(packageJSON)) {
               var packageObj = JSON.parse(fs.readFileSync(packageJSON));
               dependencies[packageObj.name] = {
                   version: packageObj.version,
                   integrity: "sha1-000000000000000000000000000=",
                   dependencies: {}
               };
               processDependencies(path.join(filePath, "node_modules"), dependencies[packageObj.name].dependencies);
           }
       }
 
       function processDependencies(dir, dependencies) {
           if(fs.existsSync(dir)) {
               var files = fs.readdirSync(dir);
 
               files.forEach(function(entry) {
                   var filePath = path.join(dir, entry);
                   var stats = fs.statSync(filePath);
 
                   if(stats.isDirectory()) {
                       if(entry.substr(0, 1) == "@") {
                           // When we encounter a namespace folder, augment all packages belonging to the scope
                           var pkgFiles = fs.readdirSync(filePath);
 
                           pkgFiles.forEach(function(entry) {
                               if(stats.isDirectory()) {
                                   var pkgFilePath = path.join(filePath, entry);
                                   augmentPackageJSON(pkgFilePath, dependencies);
                               }
                           });
                       } else {
                           augmentPackageJSON(filePath, dependencies);
                       }
                   }
               });
           }
       }
 
       processDependencies("node_modules", lockObj.dependencies);
 
       fs.writeFileSync("package-lock.json", JSON.stringify(lockObj, null, 2));
     '';
   };
 
-  # Builds and composes an NPM package including all its dependencies
-  buildNodePackage =
-    { name
-    , packageName
-    , version
-    , dependencies ? []
-    , buildInputs ? []
-    , production ? true
-    , npmFlags ? ""
-    , dontNpmInstall ? false
-    , bypassCache ? false
-    , preRebuild ? ""
-    , dontStrip ? true
-    , unpackPhase ? "true"
-    , buildPhase ? "true"
-    , ... }@args:
-
+  prepareAndInvokeNPM = {packageName, bypassCache, reconstructLock, npmFlags, production}:
     let
       forceOfflineFlag = if bypassCache then "--offline" else "--registry http://www.example.com";
-      extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" "dontStrip" "dontNpmInstall" "preRebuild" "unpackPhase" "buildPhase" ];
     in
-    stdenv.mkDerivation ({
-      name = "node-${name}-${version}";
-      buildInputs = [ tarWrapper python nodejs ]
-        ++ stdenv.lib.optional (stdenv.isLinux) utillinux
-        ++ stdenv.lib.optional (stdenv.isDarwin) libtool
-        ++ buildInputs;
-
-      inherit dontStrip; # Stripping may fail a build for some package deployments
-      inherit dontNpmInstall preRebuild unpackPhase buildPhase;
-
-      compositionScript = composePackage args;
-      pinpointDependenciesScript = pinpointDependenciesOfPackage args;
-
-      passAsFile = [ "compositionScript" "pinpointDependenciesScript" ];
-
-      installPhase = ''
-        # Create and enter a root node_modules/ folder
-        mkdir -p $out/lib/node_modules
-        cd $out/lib/node_modules
-
-        # Compose the package and all its dependencies
-        source $compositionScriptPath
-
+    ''
         # Pinpoint the versions of all dependencies to the ones that are actually being used
         echo "pinpointing versions of dependencies..."
         source $pinpointDependenciesScriptPath
 
         # Patch the shebangs of the bundled modules to prevent them from
         # calling executables outside the Nix store as much as possible
         patchShebangs .
 
         # Deploy the Node.js package by running npm install. Since the
         # dependencies have been provided already by ourselves, it should not
         # attempt to install them again, which is good, because we want to make
         # it Nix's responsibility. If it needs to install any dependencies
         # anyway (e.g. because the dependency parameters are
         # incomplete/incorrect), it fails.
         #
         # The other responsibilities of NPM are kept -- version checks, build
         # steps, postprocessing etc.
 
         export HOME=$TMPDIR
         cd "${packageName}"
         runHook preRebuild
 
         ${stdenv.lib.optionalString bypassCache ''
-          if [ ! -f package-lock.json ]
-          then
-              echo "No package-lock.json file found, reconstructing..."
-              node ${reconstructPackageLock}
-          fi
+          ${stdenv.lib.optionalString reconstructLock ''
+            if [ -f package-lock.json ]
+            then
+                echo "WARNING: Reconstruct lock option enabled, but a lock file already exists!"
+                echo "This will most likely result in version mismatches! We will remove the lock file and regenerate it!"
+                rm package-lock.json
+            else
+                echo "No package-lock.json file found, reconstructing..."
+            fi
+
+            node ${reconstructPackageLock}
+          ''}
 
           node ${addIntegrityFieldsScript}
         ''}
 
         npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} rebuild
 
-        if [ "$dontNpmInstall" != "1" ]
+        if [ "''${dontNpmInstall-}" != "1" ]
         then
             # NPM tries to download packages even when they already exist if npm-shrinkwrap is used.
             rm -f npm-shrinkwrap.json
 
             npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} install
         fi
+    '';
+
+  # Builds and composes an NPM package including all its dependencies
+  buildNodePackage =
+    { name
+    , packageName
+    , version
+    , dependencies ? []
+    , buildInputs ? []
+    , production ? true
+    , npmFlags ? ""
+    , dontNpmInstall ? false
+    , bypassCache ? false
+    , reconstructLock ? false
+    , preRebuild ? ""
+    , dontStrip ? true
+    , unpackPhase ? "true"
+    , buildPhase ? "true"
+    , ... }@args:
+
+    let
+      extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" "dontStrip" "dontNpmInstall" "preRebuild" "unpackPhase" "buildPhase" ];
+    in
+    stdenv.mkDerivation ({
+      name = "node_${name}-${version}";
+      buildInputs = [ tarWrapper python nodejs ]
+        ++ stdenv.lib.optional (stdenv.isLinux) utillinux
+        ++ stdenv.lib.optional (stdenv.isDarwin) libtool
+        ++ buildInputs;
+
+      inherit nodejs;
+
+      inherit dontStrip; # Stripping may fail a build for some package deployments
+      inherit dontNpmInstall preRebuild unpackPhase buildPhase;
+
+      compositionScript = composePackage args;
+      pinpointDependenciesScript = pinpointDependenciesOfPackage args;
+
+      passAsFile = [ "compositionScript" "pinpointDependenciesScript" ];
+
+      installPhase = ''
+        # Create and enter a root node_modules/ folder
+        mkdir -p $out/lib/node_modules
+        cd $out/lib/node_modules
+
+        # Compose the package and all its dependencies
+        source $compositionScriptPath
+
+        ${prepareAndInvokeNPM { inherit packageName bypassCache reconstructLock npmFlags production; }}
 
         # Create symlink to the deployed executable folder, if applicable
         if [ -d "$out/lib/node_modules/.bin" ]
         then
             ln -s $out/lib/node_modules/.bin $out/bin
         fi
 
         # Create symlinks to the deployed manual page folders, if applicable
         if [ -d "$out/lib/node_modules/${packageName}/man" ]
         then
             mkdir -p $out/share
             for dir in "$out/lib/node_modules/${packageName}/man/"*
             do
                 mkdir -p $out/share/man/$(basename "$dir")
                 for page in "$dir"/*
                 do
                     ln -s $page $out/share/man/$(basename "$dir")
                 done
             done
         fi
 
         # Run post install hook, if provided
         runHook postInstall
       '';
     } // extraArgs);
 
   # Builds a development shell
   buildNodeShell =
     { name
     , packageName
     , version
     , src
     , dependencies ? []
     , buildInputs ? []
     , production ? true
     , npmFlags ? ""
     , dontNpmInstall ? false
     , bypassCache ? false
+    , reconstructLock ? false
     , dontStrip ? true
     , unpackPhase ? "true"
     , buildPhase ? "true"
     , ... }@args:
 
     let
-      forceOfflineFlag = if bypassCache then "--offline" else "--registry http://www.example.com";
-
       extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" ];
 
       nodeDependencies = stdenv.mkDerivation ({
         name = "node-dependencies-${name}-${version}";
 
         buildInputs = [ tarWrapper python nodejs ]
           ++ stdenv.lib.optional (stdenv.isLinux) utillinux
           ++ stdenv.lib.optional (stdenv.isDarwin) libtool
           ++ buildInputs;
 
         inherit dontStrip; # Stripping may fail a build for some package deployments
         inherit dontNpmInstall unpackPhase buildPhase;
 
         includeScript = includeDependencies { inherit dependencies; };
         pinpointDependenciesScript = pinpointDependenciesOfPackage args;
 
         passAsFile = [ "includeScript" "pinpointDependenciesScript" ];
 
         installPhase = ''
           mkdir -p $out/${packageName}
           cd $out/${packageName}
 
           source $includeScriptPath
 
           # Create fake package.json to make the npm commands work properly
           cp ${src}/package.json .
           chmod 644 package.json
           ${stdenv.lib.optionalString bypassCache ''
             if [ -f ${src}/package-lock.json ]
             then
                 cp ${src}/package-lock.json .
             fi
           ''}
 
-          # Pinpoint the versions of all dependencies to the ones that are actually being used
-          echo "pinpointing versions of dependencies..."
+          # Go to the parent folder to make sure that all packages are pinpointed
           cd ..
           ${stdenv.lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."}
 
-          source $pinpointDependenciesScriptPath
-          cd ${packageName}
-
-          # Patch the shebangs of the bundled modules to prevent them from
-          # calling executables outside the Nix store as much as possible
-          patchShebangs .
-
-          export HOME=$PWD
+          ${prepareAndInvokeNPM { inherit packageName bypassCache reconstructLock npmFlags production; }}
 
-          ${stdenv.lib.optionalString bypassCache ''
-            if [ ! -f package-lock.json ]
-            then
-                echo "No package-lock.json file found, reconstructing..."
-                node ${reconstructPackageLock}
-            fi
-
-            node ${addIntegrityFieldsScript}
-          ''}
-
-          npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} rebuild
-
-          ${stdenv.lib.optionalString (!dontNpmInstall) ''
-            # NPM tries to download packages even when they already exist if npm-shrinkwrap is used.
-            rm -f npm-shrinkwrap.json
-
-            npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} install
-          ''}
-
+          # Expose the executables that were installed
           cd ..
           ${stdenv.lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."}
 
           mv ${packageName} lib
           ln -s $out/lib/node_modules/.bin $out/bin
         '';
       } // extraArgs);
     in
     stdenv.mkDerivation {
       name = "node-shell-${name}-${version}";
 
       buildInputs = [ python nodejs ] ++ stdenv.lib.optional (stdenv.isLinux) utillinux ++ buildInputs;
       buildCommand = ''
         mkdir -p $out/bin
         cat > $out/bin/shell <<EOF
         #! ${stdenv.shell} -e
         $shellHook
         exec ${stdenv.shell}
         EOF
         chmod +x $out/bin/shell
       '';
 
       # Provide the dependencies in a development shell through the NODE_PATH environment variable
       inherit nodeDependencies;
       shellHook = stdenv.lib.optionalString (dependencies != []) ''
-        export NODE_PATH=$nodeDependencies/lib/node_modules
+        export NODE_PATH=${nodeDependencies}/lib/node_modules
+        export PATH="${nodeDependencies}/bin:$PATH"
       '';
     };
 in
 {
   buildNodeSourceDist = stdenv.lib.makeOverridable buildNodeSourceDist;
   buildNodePackage = stdenv.lib.makeOverridable buildNodePackage;
   buildNodeShell = stdenv.lib.makeOverridable buildNodeShell;
 }