diff --git a/bower.json b/bower.json --- a/bower.json +++ b/bower.json @@ -10,7 +10,7 @@ "paper-tooltip": "PolymerElements/paper-tooltip#^1.1.2", "paper-toast": "PolymerElements/paper-toast#^1.3.0", "paper-toggle-button": "PolymerElements/paper-toggle-button#^1.2.0", - "iron-ajax": "PolymerElements/iron-ajax#^1.4.3", + "iron-ajax": "PolymerElements/iron-ajax#^1.4.4", "iron-autogrow-textarea": "PolymerElements/iron-autogrow-textarea#^1.0.13", "iron-a11y-keys": "PolymerElements/iron-a11y-keys#^1.0.6" } diff --git a/default.nix b/default.nix --- a/default.nix +++ b/default.nix @@ -1,41 +1,40 @@ # Nix environment for the community edition # -# This shall be as lean as possible, just producing the Enterprise +# This shall be as lean as possible, just producing the enterprise-ce # derivation. For advanced tweaks to pimp up the development environment we use # "shell.nix" so that it does not have to clutter this file. +# +# Configuration, set values in "~/.nixpkgs/config.nix". +# example +# { +# # Thoughts on how to configure the dev environment +# rc = { +# codeInternalUrl = "https://usr:token@internal-code.rhodecode.com"; +# sources = { +# rhodecode-vcsserver = "/home/user/work/rhodecode-vcsserver"; +# rhodecode-enterprise-ce = "/home/user/work/rhodecode-enterprise-ce"; +# rhodecode-enterprise-ee = "/home/user/work/rhodecode-enterprise-ee"; +# }; +# }; +# } args@ { pythonPackages ? "python27Packages" , pythonExternalOverrides ? self: super: {} -, doCheck ? true +, doCheck ? false , ... }: let - # Use nixpkgs from args or import them. We use this indirect approach # through args to be able to use the name `pkgs` for our customized packages. # Otherwise we will end up with an infinite recursion. - nixpkgs = args.pkgs or (import { }); + pkgs = args.pkgs or (import { }); - # johbo: Interim bridge which allows us to build with the upcoming - # nixos.16.09 branch (unstable at the moment of writing this note) and the - # current stable nixos-16.03. - backwardsCompatibleFetchgit = { ... }@args: - let - origSources = nixpkgs.fetchgit args; - in - nixpkgs.lib.overrideDerivation origSources (oldAttrs: { - NIX_PREFETCH_GIT_CHECKOUT_HOOK = '' - find $out -name '.git*' -print0 | xargs -0 rm -rf - ''; - }); - - # Create a customized version of nixpkgs which should be used throughout the - # rest of this file. - pkgs = nixpkgs.overridePackages (self: super: { - fetchgit = backwardsCompatibleFetchgit; - }); + # Works with the new python-packages, still can fallback to the old + # variant. + basePythonPackagesUnfix = basePythonPackages.__unfix__ or ( + self: basePythonPackages.override (a: { inherit self; })); # Evaluates to the last segment of a file system path. basename = path: with pkgs.lib; last (splitString "/" path); @@ -46,7 +45,7 @@ let ext = last (splitString "." path); in !builtins.elem (basename path) [ - ".git" ".hg" "__pycache__" ".eggs" + ".git" ".hg" "__pycache__" ".eggs" ".idea" ".dev" "bower_components" "node_modules" "build" "data" "result" "tmp"] && !builtins.elem ext ["egg-info" "pyc"] && @@ -54,18 +53,20 @@ let # it would still be good to restore it since we want to ignore "result-*". !hasPrefix "result" path; - basePythonPackages = with builtins; if isAttrs pythonPackages - then pythonPackages - else getAttr pythonPackages pkgs; + sources = + let + inherit (pkgs.lib) all isString attrValues; + sourcesConfig = pkgs.config.rc.sources or {}; + in + # Ensure that sources are configured as strings. Using a path + # would result in a copy into the nix store. + assert all isString (attrValues sourcesConfig); + sourcesConfig; - buildBowerComponents = - pkgs.buildBowerComponents or - (import ./pkgs/backport-16.03-build-bower-components.nix { inherit pkgs; }); - - sources = pkgs.config.rc.sources or {}; - version = builtins.readFile ./rhodecode/VERSION; + version = builtins.readFile "${rhodecode-enterprise-ce-src}/rhodecode/VERSION"; rhodecode-enterprise-ce-src = builtins.filterSource src-filter ./.; + buildBowerComponents = pkgs.buildBowerComponents; nodeEnv = import ./pkgs/node-default.nix { inherit pkgs; }; @@ -77,133 +78,145 @@ let src = rhodecode-enterprise-ce-src; }; - pythonGeneratedPackages = self: basePythonPackages.override (a: { - inherit self; - }) - // (scopedImport { - self = self; - super = basePythonPackages; - inherit pkgs; - inherit (pkgs) fetchurl fetchgit; - } ./pkgs/python-packages.nix); + rhodecode-testdata-src = sources.rhodecode-testdata or ( + pkgs.fetchhg { + url = "https://code.rhodecode.com/upstream/rc_testdata"; + rev = "v0.10.0"; + sha256 = "0zn9swwvx4vgw4qn8q3ri26vvzgrxn15x6xnjrysi1bwmz01qjl0"; + }); - pythonOverrides = import ./pkgs/python-packages-overrides.nix { - inherit - basePythonPackages - pkgs; + rhodecode-testdata = import "${rhodecode-testdata-src}/default.nix" { + inherit + doCheck + pkgs + pythonPackages; }; pythonLocalOverrides = self: super: { rhodecode-enterprise-ce = let linkNodeAndBowerPackages = '' - echo "Export RhodeCode CE path" export RHODECODE_CE_PATH=${rhodecode-enterprise-ce-src} - echo "Link node packages" + + echo "[BEGIN]: Link node packages" rm -fr node_modules mkdir node_modules # johbo: Linking individual packages allows us to run "npm install" # inside of a shell to try things out. Re-entering the shell will # restore a clean environment. ln -s ${nodeDependencies}/lib/node_modules/* node_modules/ + echo "[DONE]: Link node packages" - echo "DONE: Link node packages" - - echo "Link bower packages" + echo "[BEGIN]: Link bower packages" rm -fr bower_components mkdir bower_components + ln -s ${bowerComponents}/bower_components/* bower_components/ + echo "[DONE]: Link bower packages" + ''; - ln -s ${bowerComponents}/bower_components/* bower_components/ - echo "DONE: Link bower packages" - ''; + releaseName = "RhodeCodeEnterpriseCE-${version}"; in super.rhodecode-enterprise-ce.override (attrs: { - inherit doCheck version; + name = "rhodecode-enterprise-ce-${version}"; - releaseName = "RhodeCodeEnterpriseCE-${version}"; + releaseName = releaseName; src = rhodecode-enterprise-ce-src; dontStrip = true; # prevent strip, we don't need it. - buildInputs = - attrs.buildInputs ++ - (with self; [ - pkgs.nodePackages.bower - pkgs.nodePackages.grunt-cli - pkgs.subversion - rhodecode-testdata - ]); - - #TODO: either move this into overrides, OR use the new machanics from - # pip2nix and requiremtn.txt file - propagatedBuildInputs = attrs.propagatedBuildInputs ++ (with self; [ - rhodecode-tools - ]); - - # TODO: johbo: Make a nicer way to expose the parts. Maybe - # pkgs/default.nix? + # expose following attributed outside passthru = { inherit + rhodecode-testdata bowerComponents linkNodeAndBowerPackages myPythonPackagesUnfix - pythonLocalOverrides; + pythonLocalOverrides + pythonCommunityOverrides; + pythonPackages = self; }; + buildInputs = + attrs.buildInputs or [] ++ [ + rhodecode-testdata + pkgs.nodePackages.bower + pkgs.nodePackages.grunt-cli + ]; + + #NOTE: option to inject additional propagatedBuildInputs + propagatedBuildInputs = + attrs.propagatedBuildInputs or [] ++ [ + + ]; + LC_ALL = "en_US.UTF-8"; LOCALE_ARCHIVE = - if pkgs.stdenv ? glibc + if pkgs.stdenv.isLinux then "${pkgs.glibcLocales}/lib/locale/locale-archive" else ""; + # Add bin directory to path so that tests can find 'rhodecode'. preCheck = '' export PATH="$out/bin:$PATH" ''; + # custom check phase for testing + checkPhase = '' + runHook preCheck + PYTHONHASHSEED=random py.test -vv -p no:sugar -r xw --cov-config=.coveragerc --cov=rhodecode --cov-report=term-missing rhodecode + runHook postCheck + ''; + postCheck = '' - rm -rf $out/lib/${self.python.libPrefix}/site-packages/pytest_pylons + echo "Cleanup of rhodecode/tests" rm -rf $out/lib/${self.python.libPrefix}/site-packages/rhodecode/tests ''; - preBuild = linkNodeAndBowerPackages + '' + preBuild = '' + + echo "Building frontend assets" + ${linkNodeAndBowerPackages} grunt rm -fr node_modules ''; postInstall = '' - echo "Writing meta information for rccontrol to nix-support/rccontrol" + echo "Writing enterprise-ce meta information for rccontrol to nix-support/rccontrol" mkdir -p $out/nix-support/rccontrol cp -v rhodecode/VERSION $out/nix-support/rccontrol/version - echo "DONE: Meta information for rccontrol written" + echo "[DONE]: enterprise-ce meta information for rccontrol written" + + mkdir -p $out/etc + cp configs/production.ini $out/etc + echo "[DONE]: saved enterprise-ce production.ini into $out/etc" # python based programs need to be wrapped - #ln -s ${self.python}/bin/* $out/bin/ + mkdir -p $out/bin + # rhodecode-tools + ln -s ${self.rhodecode-tools}/bin/rhodecode-* $out/bin/ + + # required binaries from dependencies + #ln -s ${self.python}/bin/python $out/bin ln -s ${self.pyramid}/bin/* $out/bin/ ln -s ${self.gunicorn}/bin/gunicorn $out/bin/ ln -s ${self.supervisor}/bin/supervisor* $out/bin/ ln -s ${self.pastescript}/bin/paster $out/bin/ ln -s ${self.channelstream}/bin/channelstream $out/bin/ ln -s ${self.celery}/bin/celery $out/bin/ + echo "[DONE]: created symlinks into $out/bin" - # rhodecode-tools - ln -s ${self.rhodecode-tools}/bin/rhodecode-* $out/bin/ - - # note that condition should be restricted when adding further tools for file in $out/bin/*; do wrapProgram $file \ - --prefix PATH : $PATH \ - --prefix PYTHONPATH : $PYTHONPATH \ - --set PYTHONHASHSEED random + --prefix PATH : $PATH \ + --prefix PYTHONPATH : $PYTHONPATH \ + --set PYTHONHASHSEED random done - mkdir $out/etc - cp configs/production.ini $out/etc + echo "[DONE]: enterprise-ce binary wrapping" - - # TODO: johbo: Make part of ac-tests if [ ! -f rhodecode/public/js/scripts.js ]; then echo "Missing scripts.js" exit 1 @@ -213,31 +226,33 @@ let exit 1 fi ''; - }); - rhodecode-testdata = import "${rhodecode-testdata-src}/default.nix" { - inherit - doCheck - pkgs - pythonPackages; - }; - }; - rhodecode-testdata-src = sources.rhodecode-testdata or ( - pkgs.fetchhg { - url = "https://code.rhodecode.com/upstream/rc_testdata"; - rev = "v0.10.0"; - sha256 = "0zn9swwvx4vgw4qn8q3ri26vvzgrxn15x6xnjrysi1bwmz01qjl0"; - }); + basePythonPackages = with builtins; + if isAttrs pythonPackages then + pythonPackages + else + getAttr pythonPackages pkgs; + + pythonGeneratedPackages = import ./pkgs/python-packages.nix { + inherit pkgs; + inherit (pkgs) fetchurl fetchgit fetchhg; + }; + + pythonCommunityOverrides = import ./pkgs/python-packages-overrides.nix { + inherit pkgs basePythonPackages; + }; # Apply all overrides and fix the final package set myPythonPackagesUnfix = with pkgs.lib; (extends pythonExternalOverrides (extends pythonLocalOverrides - (extends pythonOverrides - pythonGeneratedPackages))); + (extends pythonCommunityOverrides + (extends pythonGeneratedPackages + basePythonPackagesUnfix)))); + myPythonPackages = (pkgs.lib.fix myPythonPackagesUnfix); in myPythonPackages.rhodecode-enterprise-ce diff --git a/docs/contributing/dev-setup.rst b/docs/contributing/dev-setup.rst --- a/docs/contributing/dev-setup.rst +++ b/docs/contributing/dev-setup.rst @@ -47,7 +47,7 @@ Switch nix to the latest STABLE channel run:: - nix-channel --add https://nixos.org/channels/nixos-16.03 nixpkgs + nix-channel --add https://nixos.org/channels/nixos-18.03 nixpkgs Followed by:: diff --git a/package.json b/package.json --- a/package.json +++ b/package.json @@ -1,6 +1,13 @@ { "name": "rhodecode-enterprise", - "version": "0.0.1", + "version": "1.0.0", + "private": true, + "description" : "RhodeCode JS packaged", + "license": "SEE LICENSE IN LICENSE.txt", + "repository" : { + "type" : "hg", + "url" : "https://code.rhodecode.com/rhodecode-enterprise-ce" + }, "devDependencies": { "grunt": "^0.4.5", "grunt-contrib-copy": "^1.0.0", @@ -14,7 +21,7 @@ "grunt-vulcanize": "^1.0.0", "node2nix": "^1.0.0", "jshint": "^2.9.1-rc3", - "bower": "^1.7.9", + "bower": "^1.8.4", "jquery": "1.11.3", "favico.js": "^0.3.10", "clipboard": "^1.7.1", diff --git a/pkgs/README.rst b/pkgs/README.rst new file mode 100644 --- /dev/null +++ b/pkgs/README.rst @@ -0,0 +1,55 @@ + +============================== + Generate the Nix expressions +============================== + +Details can be found in the repository of `RhodeCode Enterprise CE`_ inside of +the file `docs/contributing/dependencies.rst`. + +Start the environment as follows: + +.. code:: shell + + nix-shell pkgs/shell-generate.nix + + + +Python dependencies +=================== + +.. code:: shell + + pip2nix generate --licenses + # or + nix-shell pkgs/shell-generate.nix --command "pip2nix generate --licenses" + + +NodeJS dependencies +=================== + +.. code:: shell + + # switch to pkgs dir + pushd pkgs + node2nix --input ../package.json \ + -o node-packages.nix \ + -e node-env.nix \ + -c node-default.nix \ + -d --flatten --nodejs-6 + popd + + + +Bower dependencies +================== + +.. code:: shell + + bower2nix bower.json pkgs/bower-packages.nix + # or + nix-shell pkgs/shell-generate.nix --command "bower2nix bower.json pkgs/bower-packages.nix" + + +.. Links + +.. _RhodeCode Enterprise CE: https://code.rhodecode.com/rhodecode-enterprise-ce diff --git a/pkgs/backport-16.03-build-bower-components.nix b/pkgs/backport-16.03-build-bower-components.nix deleted file mode 100644 --- a/pkgs/backport-16.03-build-bower-components.nix +++ /dev/null @@ -1,67 +0,0 @@ -# Backported buildBowerComponents so that we can also use it with the version -# 16.03 which is the current stable at the time of this writing. -# -# This file can be removed once building with 16.03 is not needed anymore. - -{ pkgs }: - -{ buildInputs ? [], generated, ... } @ attrs: - -let - bower2nix-src = pkgs.fetchzip { - url = "https://github.com/rvl/bower2nix/archive/v3.0.1.tar.gz"; - sha256 = "1zbvz96k2j6g0r4lvm5cgh41a73k9dgayk7x63cmg538dzznxvyb"; - }; - - bower2nix = import "${bower2nix-src}/default.nix" { inherit pkgs; }; - - fetchbower = import ./backport-16.03-fetchbower.nix { - inherit (pkgs) stdenv lib; - inherit bower2nix; - }; - - # Fetches the bower packages. `generated` should be the result of a - # `bower2nix` command. - bowerPackages = import generated { - inherit (pkgs) buildEnv; - inherit fetchbower; - }; - -in pkgs.stdenv.mkDerivation ( - attrs - // - { - name = "bower_components-" + attrs.name; - - inherit bowerPackages; - - builder = builtins.toFile "builder.sh" '' - source $stdenv/setup - - # The project's bower.json is required - cp $src/bower.json . - - # Dereference symlinks -- bower doesn't like them - cp --recursive --reflink=auto \ - --dereference --no-preserve=mode \ - $bowerPackages bc - - # Bower install in offline mode -- links together the fetched - # bower packages. - HOME=$PWD bower \ - --config.storage.packages=bc/packages \ - --config.storage.registry=bc/registry \ - --offline install - - # Sets up a single bower_components directory within - # the output derivation. - mkdir -p $out - mv bower_components $out - ''; - - buildInputs = buildInputs ++ [ - pkgs.git - pkgs.nodePackages.bower - ]; - } -) diff --git a/pkgs/backport-16.03-fetchbower.nix b/pkgs/backport-16.03-fetchbower.nix deleted file mode 100644 --- a/pkgs/backport-16.03-fetchbower.nix +++ /dev/null @@ -1,26 +0,0 @@ -{ stdenv, lib, bower2nix }: -let - bowerVersion = version: - let - components = lib.splitString "#" version; - hash = lib.last components; - ver = if builtins.length components == 1 then version else hash; - in ver; - - fetchbower = name: version: target: outputHash: stdenv.mkDerivation { - name = "${name}-${bowerVersion version}"; - buildCommand = '' - fetch-bower --quiet --out=$PWD/out "${name}" "${target}" "${version}" - # In some cases, the result of fetchBower is different depending - # on the output directory (e.g. if the bower package contains - # symlinks). So use a local output directory before copying to - # $out. - cp -R out $out - ''; - outputHashMode = "recursive"; - outputHashAlgo = "sha256"; - inherit outputHash; - buildInputs = [ bower2nix ]; - }; - -in fetchbower diff --git a/pkgs/bower-packages.nix b/pkgs/bower-packages.nix --- a/pkgs/bower-packages.nix +++ b/pkgs/bower-packages.nix @@ -1,33 +1,34 @@ +# Generated by bower2nix v3.2.0 (https://github.com/rvl/bower2nix) { fetchbower, buildEnv }: buildEnv { name = "bower-env"; ignoreCollisions = true; paths = [ - (fetchbower "webcomponentsjs" "0.7.22" "^0.7.22" "0ggh3k8ssafd056ib1m5bvzi7cpz3ry7gr5176d79na1w0c3i7dz") - (fetchbower "polymer" "Polymer/polymer#1.6.1" "Polymer/polymer#^1.6.1" "09mm0jgk457gvwqlc155swch7gjr6fs3g7spnvhi6vh5b6518540") - (fetchbower "paper-button" "PolymerElements/paper-button#1.0.13" "PolymerElements/paper-button#^1.0.13" "0i3y153nqk06pn0gk282vyybnl3g1w3w41d5i9z659cgn27g3fvm") - (fetchbower "paper-spinner" "PolymerElements/paper-spinner#1.2.0" "PolymerElements/paper-spinner#^1.2.0" "1av1m6y81jw3hjhz1yqy3rwcgxarjzl58ldfn4q6sn51pgzngfqb") - (fetchbower "paper-tooltip" "PolymerElements/paper-tooltip#1.1.3" "PolymerElements/paper-tooltip#^1.1.2" "0vmrm1n8k9sk9nvqy03q177axy22pia6i3j1gxbk72j3pqiqvg6k") - (fetchbower "paper-toast" "PolymerElements/paper-toast#1.3.0" "PolymerElements/paper-toast#^1.3.0" "0x9rqxsks5455s8pk4aikpp99ijdn6kxr9gvhwh99nbcqdzcxq1m") - (fetchbower "paper-toggle-button" "PolymerElements/paper-toggle-button#1.2.0" "PolymerElements/paper-toggle-button#^1.2.0" "0mphcng3ngspbpg4jjn0mb91nvr4xc1phq3qswib15h6sfww1b2w") - (fetchbower "iron-ajax" "PolymerElements/iron-ajax#1.4.4" "PolymerElements/iron-ajax#^1.4.4" "0jpi7ik3zljw8yh2ccc85r26lcpzmkc2nl1kn6fqdx57zkzk9v5b") - (fetchbower "iron-autogrow-textarea" "PolymerElements/iron-autogrow-textarea#1.0.13" "PolymerElements/iron-autogrow-textarea#^1.0.13" "0zwhpl97vii1s8k0lgain8i9dnw29b0mxc5ixdscx9las13n2lqq") - (fetchbower "iron-a11y-keys" "PolymerElements/iron-a11y-keys#1.0.6" "PolymerElements/iron-a11y-keys#^1.0.6" "1xz3mgghfcxixq28sdb654iaxj4nyi1bzcwf77ydkms6fviqs9mv") - (fetchbower "iron-flex-layout" "PolymerElements/iron-flex-layout#1.3.1" "PolymerElements/iron-flex-layout#^1.0.0" "0nswv3ih3bhflgcd2wjfmddqswzgqxb2xbq65jk9w3rkj26hplbl") - (fetchbower "paper-behaviors" "PolymerElements/paper-behaviors#1.0.12" "PolymerElements/paper-behaviors#^1.0.0" "012bqk97awgz55cn7rm9g7cckrdhkqhls3zvp8l6nd4rdwcrdzq8") - (fetchbower "paper-material" "PolymerElements/paper-material#1.0.6" "PolymerElements/paper-material#^1.0.0" "0rljmknfdbm5aabvx9pk77754zckj3l127c3mvnmwkpkkr353xnh") - (fetchbower "paper-styles" "PolymerElements/paper-styles#1.1.4" "PolymerElements/paper-styles#^1.0.0" "0j8vg74xrcxlni8i93dsab3y80f34kk30lv4yblqpkp9c3nrilf7") - (fetchbower "neon-animation" "PolymerElements/neon-animation#1.2.4" "PolymerElements/neon-animation#^1.0.0" "16mz9i2n5w0k5j8d6gha23cnbdgm5syz3fawyh89gdbq97bi2q5j") - (fetchbower "iron-a11y-announcer" "PolymerElements/iron-a11y-announcer#1.0.5" "PolymerElements/iron-a11y-announcer#^1.0.0" "0n7c7j1pwk3835s7s2jd9125wdcsqf216yi5gj07wn5s8h8p7m9d") - (fetchbower "iron-overlay-behavior" "PolymerElements/iron-overlay-behavior#1.8.6" "PolymerElements/iron-overlay-behavior#^1.0.9" "14brn9gz6qqskarg3fxk91xs7vg02vgcsz9a9743kidxr0l0413m") - (fetchbower "iron-fit-behavior" "PolymerElements/iron-fit-behavior#1.2.5" "PolymerElements/iron-fit-behavior#^1.1.0" "1msnlh8lp1xg6v4h6dkjwj9kzac5q5q208ayla3x9hi483ki6rlf") - (fetchbower "iron-checked-element-behavior" "PolymerElements/iron-checked-element-behavior#1.0.5" "PolymerElements/iron-checked-element-behavior#^1.0.0" "0l0yy4ah454s8bzfv076s8by7h67zy9ni6xb932qwyhx8br6c1m7") - (fetchbower "promise-polyfill" "polymerlabs/promise-polyfill#1.0.1" "polymerlabs/promise-polyfill#^1.0.0" "045bj2caav3famr5hhxgs1dx7n08r4s46mlzwb313vdy17is38xb") - (fetchbower "iron-behaviors" "PolymerElements/iron-behaviors#1.0.17" "PolymerElements/iron-behaviors#^1.0.0" "021qvkmbk32jrrmmphpmwgby4bzi5jyf47rh1bxmq2ip07ly4bpr") - (fetchbower "iron-validatable-behavior" "PolymerElements/iron-validatable-behavior#1.1.1" "PolymerElements/iron-validatable-behavior#^1.0.0" "1yhxlvywhw2klbbgm3f3cmanxfxggagph4ii635zv0c13707wslv") - (fetchbower "iron-form-element-behavior" "PolymerElements/iron-form-element-behavior#1.0.6" "PolymerElements/iron-form-element-behavior#^1.0.0" "0rdhxivgkdhhz2yadgdbjfc70l555p3y83vjh8rfj5hr0asyn6q1") - (fetchbower "iron-a11y-keys-behavior" "polymerelements/iron-a11y-keys-behavior#1.1.9" "polymerelements/iron-a11y-keys-behavior#^1.0.0" "1imm4gc84qizihhbyhfa8lwjh3myhj837f79i5m04xjgwrjmkaf6") - (fetchbower "paper-ripple" "PolymerElements/paper-ripple#1.0.8" "PolymerElements/paper-ripple#^1.0.0" "0r9sq8ik7wwrw0qb82c3rw0c030ljwd3s466c9y4qbcrsbvfjnns") - (fetchbower "font-roboto" "PolymerElements/font-roboto#1.0.1" "PolymerElements/font-roboto#^1.0.1" "02jz43r0wkyr3yp7rq2rc08l5cwnsgca9fr54sr4rhsnl7cjpxrj") - (fetchbower "iron-meta" "PolymerElements/iron-meta#1.1.2" "PolymerElements/iron-meta#^1.0.0" "1wl4dx8fnsknw9z9xi8bpc4cy9x70c11x4zxwxnj73hf3smifppl") - (fetchbower "iron-resizable-behavior" "PolymerElements/iron-resizable-behavior#1.0.5" "PolymerElements/iron-resizable-behavior#^1.0.0" "1fd5zmbr2hax42vmcasncvk7lzi38fmb1kyii26nn8pnnjak7zkn") - (fetchbower "iron-selector" "PolymerElements/iron-selector#1.5.2" "PolymerElements/iron-selector#^1.0.0" "1ajv46llqzvahm5g6g75w7nfyjcslp53ji0wm96l2k94j87spv3r") - (fetchbower "web-animations-js" "web-animations/web-animations-js#2.2.2" "web-animations/web-animations-js#^2.2.0" "1izfvm3l67vwys0bqbhidi9rqziw2f8wv289386sc6jsxzgkzhga") + (fetchbower "webcomponentsjs" "0.7.24" "^0.7.22" "0d6hfsd51n7qykzci9cw5vlsrajvffaf5ic3azlp2rfz76m651qj") + (fetchbower "polymer" "Polymer/polymer#1.11.3" "Polymer/polymer#^1.6.1" "0n11ag2pmczw5yv3m76bh0a7hvicqvcaiv7knixx1r704pw107s6") + (fetchbower "paper-button" "PolymerElements/paper-button#1.0.15" "PolymerElements/paper-button#^1.0.13" "0zabrp8p4s9md1hlwg0rqmbx0k87a41lsg9pzk747hcb349gblg0") + (fetchbower "paper-spinner" "PolymerElements/paper-spinner#1.2.1" "PolymerElements/paper-spinner#^1.2.0" "0d6xc9fd2ipcli7w77yrn1k0z9j373c9y1f16db2840cyb4rvii8") + (fetchbower "paper-tooltip" "PolymerElements/paper-tooltip#1.1.4" "PolymerElements/paper-tooltip#^1.1.2" "0j8s09dxqql8mgnvb7x382scq98xk2vjgylk06bsd1gphp3d3qzm") + (fetchbower "paper-toast" "PolymerElements/paper-toast#1.3.1" "PolymerElements/paper-toast#^1.3.0" "1s0csv8dwgdyg4psq1zrd6vivlpsgzi4sjqllwqmlwhfnxfl5ql4") + (fetchbower "paper-toggle-button" "PolymerElements/paper-toggle-button#1.3.0" "PolymerElements/paper-toggle-button#^1.2.0" "0hvv2y406lzlrkkcmv9nnd99bmcgcrhcx86q3axxv8k3580gqq97") + (fetchbower "iron-ajax" "PolymerElements/iron-ajax#1.4.4" "PolymerElements/iron-ajax#^1.4.4" "0vs4dqcw5y02kj11ivzs901s5nwn97fk01xz2jmpy2fgh6l9q5yr") + (fetchbower "iron-autogrow-textarea" "PolymerElements/iron-autogrow-textarea#1.0.15" "PolymerElements/iron-autogrow-textarea#^1.0.13" "1jw40ki5w21il0i9pwjywk4y6mk9lrj8fm57vfg9nlpbiqm2vswb") + (fetchbower "iron-a11y-keys" "PolymerElements/iron-a11y-keys#1.0.9" "PolymerElements/iron-a11y-keys#^1.0.6" "07c2wm1p9g52qidl67a43yb7pzd88ygycgghlwzjbh2vkwrs40kp") + (fetchbower "iron-flex-layout" "PolymerElements/iron-flex-layout#1.3.9" "PolymerElements/iron-flex-layout#^1.0.0" "1r54la4n8n0lq97vdxnlpdrarxsiwp2b3vfvby9il3j4y4s8vi4h") + (fetchbower "paper-behaviors" "PolymerElements/paper-behaviors#1.0.13" "PolymerElements/paper-behaviors#^1.0.0" "0yljykkdg9p67dinplmp6hc5ma6sp95ykah8kz6id5z8gjmsd05b") + (fetchbower "paper-material" "PolymerElements/paper-material#1.0.7" "PolymerElements/paper-material#^1.0.0" "1q9r3i5f61y6hmd18h3fcmn7y29yznraz83f9256z8cc0vglfjdb") + (fetchbower "paper-styles" "PolymerElements/paper-styles#1.3.1" "PolymerElements/paper-styles#^1.0.0" "11fcxp9kx6sqp2yq0883psn8xyw5d3i753mimqbx8aqa5abvrk4q") + (fetchbower "neon-animation" "PolymerElements/neon-animation#1.2.5" "PolymerElements/neon-animation#^1.0.0" "144sq9ijw1nnp2jagpa1ammrc018kp1y6nlmgq1v1iishv4ylsl5") + (fetchbower "iron-a11y-announcer" "PolymerElements/iron-a11y-announcer#1.0.6" "PolymerElements/iron-a11y-announcer#^1.0.0" "1az02v91s17v9bir868pifv0s2lwxchm0i4l20176f98366813zk") + (fetchbower "iron-overlay-behavior" "PolymerElements/iron-overlay-behavior#1.10.4" "PolymerElements/iron-overlay-behavior#^1.0.9" "0px6s756cgqzxzq53fgk1297j07gyfykqkhdzmj9fwyyrwiv1g8z") + (fetchbower "iron-fit-behavior" "PolymerElements/iron-fit-behavior#1.2.7" "PolymerElements/iron-fit-behavior#^1.1.0" "0b864x9cdxadvzkdcn1d3yvi32kqccfv8j467337rgcq193l60jb") + (fetchbower "iron-checked-element-behavior" "PolymerElements/iron-checked-element-behavior#1.0.6" "PolymerElements/iron-checked-element-behavior#^1.0.0" "165nh5vkdsr4d5vkq4sj1sz0igfy6vgx844ys164mqg8gkjncvxm") + (fetchbower "promise-polyfill" "polymerlabs/promise-polyfill#1.0.1" "polymerlabs/promise-polyfill#^1.0.0" "0warxr9fk2d3cvgiq81djxwiky73gxxr5s3xa97d3c83q1lidlzy") + (fetchbower "iron-behaviors" "PolymerElements/iron-behaviors#1.0.18" "PolymerElements/iron-behaviors#^1.0.0" "1icx212kh1cfdvk3v7pdqic9c48pdhnknq8ajx0dlps1l5sm69xh") + (fetchbower "iron-validatable-behavior" "PolymerElements/iron-validatable-behavior#1.1.2" "PolymerElements/iron-validatable-behavior#^1.0.0" "1gjjn08y5s43p7aizif24i2yvkd1sasy77dix62irkwzig3favqr") + (fetchbower "iron-form-element-behavior" "PolymerElements/iron-form-element-behavior#1.0.7" "PolymerElements/iron-form-element-behavior#^1.0.0" "02qbxqsqxjzy086cqbv6pgibmi0888q757p95ig6x3nc62xk81an") + (fetchbower "iron-a11y-keys-behavior" "PolymerElements/iron-a11y-keys-behavior#1.1.9" "PolymerElements/iron-a11y-keys-behavior#^1.0.0" "0dy1ca1nb9v9y968q998vgs147fkmn4irnnrdfl40ln1bln45qx9") + (fetchbower "paper-ripple" "PolymerElements/paper-ripple#1.0.10" "PolymerElements/paper-ripple#^1.0.0" "0d5gjf7cw7hk520h6xnav1xrpd948qc8mzvqgqycqkad4j4vdck7") + (fetchbower "font-roboto" "PolymerElements/font-roboto#1.1.0" "PolymerElements/font-roboto#^1.0.1" "0z4msvaa5pnr84j2r957g313fmdbdbrknhdw1axy5g48845yv04s") + (fetchbower "iron-meta" "PolymerElements/iron-meta#1.1.3" "PolymerElements/iron-meta#^1.0.0" "13lsj648ibkyw3lha6g6r7afmk4yxvgdi63bkpy54wx63gfx7xir") + (fetchbower "iron-resizable-behavior" "PolymerElements/iron-resizable-behavior#1.0.6" "PolymerElements/iron-resizable-behavior#^1.0.0" "1r8qk670nigqpw50x1m5yvbx7p9sahiwlf0f9z71v508d63vrbi1") + (fetchbower "iron-selector" "PolymerElements/iron-selector#1.5.3" "PolymerElements/iron-selector#^1.0.0" "1362pq6vy113h4y6hn31hhp52hh8g269s5aj7vsq266v7y59igf6") + (fetchbower "web-animations-js" "web-animations/web-animations-js#2.3.1" "web-animations/web-animations-js#^2.2.0" "16haz886711qrcf9h9wrjwf5hrz2c69l4jxlq0iyzar823c51qkq") ]; } diff --git a/pkgs/nix-common/pip2nix.nix b/pkgs/nix-common/pip2nix.nix new file mode 100755 --- /dev/null +++ b/pkgs/nix-common/pip2nix.nix @@ -0,0 +1,17 @@ +{ pkgs +, pythonPackages +}: + +rec { + pip2nix-src = pkgs.fetchzip { + url = https://github.com/johbo/pip2nix/archive/51e6fdae34d0e8ded9efeef7a8601730249687a6.tar.gz; + sha256 = "02a4jjgi7lsvf8mhrxsd56s9a3yg20081rl9bgc2m84w60v2gbz2"; + }; + + pip2nix = import pip2nix-src { + inherit + pkgs + pythonPackages; + }; + +} diff --git a/pkgs/node-default.nix b/pkgs/node-default.nix --- a/pkgs/node-default.nix +++ b/pkgs/node-default.nix @@ -1,12 +1,14 @@ -# This file has been generated by node2nix 1.0.0. Do not edit! +# This file has been generated by node2nix 1.5.3. Do not edit! {pkgs ? import { inherit system; - }, system ? builtins.currentSystem}: + }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-6_x"}: let nodeEnv = import ./node-env.nix { - inherit (pkgs) stdenv python utillinux runCommand writeTextFile nodejs; + inherit (pkgs) stdenv python2 utillinux runCommand writeTextFile; + inherit nodejs; + libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null; }; in import ./node-packages.nix { diff --git a/pkgs/node-env.nix b/pkgs/node-env.nix --- a/pkgs/node-env.nix +++ b/pkgs/node-env.nix @@ -1,24 +1,26 @@ # This file originates from node2nix -{stdenv, python, nodejs, utillinux, runCommand, writeTextFile}: +{stdenv, nodejs, python2, utillinux, libtool, runCommand, writeTextFile}: let + python = if nodejs ? python then nodejs.python else python2; + # Create a tar wrapper that filters all the 'Ignoring unknown extended header keyword' noise tarWrapper = runCommand "tarWrapper" {} '' mkdir -p $out/bin - + cat > $out/bin/tar < package.json < {}) +, pythonPackages ? "python27Packages" +}: + +with pkgs.lib; + +let _pythonPackages = pythonPackages; in +let + pythonPackages = getAttr _pythonPackages pkgs; + + pip2nix = import ./nix-common/pip2nix.nix { + inherit + pkgs + pythonPackages; + }; + +in + +pkgs.stdenv.mkDerivation { + name = "pip2nix-generated"; + buildInputs = [ + # Allows to generate python packages + pip2nix.pip2nix + pythonPackages.pip-tools + + # Allows to generate bower dependencies + pkgs.nodePackages.bower2nix + + # Allows to generate node dependencies + pkgs.nodePackages.node2nix + + # We need mysql_config to be around + pkgs.mysql + + # We need postgresql to be around + pkgs.postgresql + + # Curl is needed for pycurl + pkgs.curl + ]; + + shellHook = '' + runHook preShellHook + runHook postShellHook + ''; + + preShellHook = '' + echo "Starting Generate Shell" + # Custom prompt to distinguish from other dev envs. + export PS1="\n\[\033[1;32m\][Generate-shell:\w]$\[\033[0m\] " + ''; +} diff --git a/release.nix b/release.nix --- a/release.nix +++ b/release.nix @@ -1,232 +1,16 @@ -# -# About -# ===== -# -# This file defines jobs for our CI system and the attribute "build" is used -# as the input for packaging. -# -# -# CI details -# ========== -# -# This file defines an attribute set of derivations. Each of these attributes is -# then used in our CI system as one job to run. This way we keep the -# configuration for the CI jobs as well under version control. -# -# Run CI jobs locally -# ------------------- -# -# Since it is all based on normal Nix derivations, the jobs can be tested -# locally with a run of "nix-build" like the following example: -# -# nix-build release.nix -A test-api -I vcsserver=~/rhodecode-vcsserver -# -# Note: Replace "~/rhodecode-vcsserver" with a path where a clone of the -# vcsserver resides. +# This file defines how to "build" for packaging. { pkgs ? import {} , doCheck ? true }: let - - inherit (pkgs) - stdenv - system; - - testing = import { - inherit system; - }; - - runInMachine = testing.runInMachine; - - sphinx = import ./docs/default.nix {}; - - mkDocs = kind: stdenv.mkDerivation { - name = kind; - srcs = [ - (./. + (builtins.toPath "/${kind}")) - (builtins.filterSource - (path: type: baseNameOf path == "VERSION") - ./rhodecode) - ]; - sourceRoot = kind; - buildInputs = [ sphinx ]; - configurePhase = null; - buildPhase = '' - make SPHINXBUILD=sphinx-build html - ''; - installPhase = '' - mkdir -p $out - mv _build/html $out/ - - mkdir -p $out/nix-support - echo "doc manual $out/html index.html" >> \ - "$out/nix-support/hydra-build-products" - ''; - }; - - enterprise = import ./default.nix { + enterprise_ce = import ./default.nix { inherit + doCheck pkgs; - - # TODO: for quick local testing - doCheck = false; - }; - - test-cfg = stdenv.mkDerivation { - name = "test-cfg"; - unpackPhase = "true"; - buildInputs = [ - enterprise.src - ]; - installPhase = '' - mkdir -p $out/etc - cp ${enterprise.src}/test.ini $out/etc/enterprise.ini - # TODO: johbo: Needed, so that the login works, this causes - # probably some side effects - substituteInPlace $out/etc/enterprise.ini --replace "is_test = True" "" - - # Gevent configuration - cp $out/etc/enterprise.ini $out/etc/enterprise-gevent.ini; - cat >> $out/etc/enterprise-gevent.ini < /dev/null - - echo "Starting rc-server" - vcsserver --config ${vcsserverCfg} >vcsserver.log 2>&1 & - rc-server enterprise.ini >rc-server.log 2>&1 & - - while ! curl -f -s http://localhost:5000 > /dev/null - do - echo "Waiting for server to be ready..." - sleep 3 - done - echo "Webserver is ready." - - echo "Starting the test run" - py.test -c example.ini -vs --maxfail=5 tests - - echo "Kill rc-server" - kill %2 - kill %1 - ''; - - # TODO: johbo: Use the install phase again once the normal mkDerivation - # can be used again. - postInstall = '' - mkdir -p $out - cp enterprise.ini $out - cp ${vcsserverCfg} $out/vcsserver.ini - cp rc-server.log $out - cp vcsserver.log $out - - mkdir -p $out/nix-support - echo "report config $out enterprise.ini" >> $out/nix-support/hydra-build-products - echo "report config $out vcsserver.ini" >> $out/nix-support/hydra-build-products - echo "report rc-server $out rc-server.log" >> $out/nix-support/hydra-build-products - echo "report vcsserver $out vcsserver.log" >> $out/nix-support/hydra-build-products - ''; - }; - - vcsserver = import { - inherit pkgs; - - # TODO: johbo: Think of a more elegant solution to this problem - pythonExternalOverrides = self: super: (enterprise.myPythonPackagesUnfix self); - }; - - runTests = optionString: (enterprise.override (attrs: { - doCheck = true; - name = "test-run"; - buildInputs = attrs.buildInputs ++ [ - vcsserver - ]; - checkPhase = '' - py.test ${optionString} -vv -ra - ''; - buildPhase = attrs.shellHook; - installPhase = '' - echo "Intentionally not installing anything" - ''; - meta.description = "Enterprise test run ${optionString}"; - })); - - jobs = { - - build = enterprise; - - # johbo: Currently this is simply running the tests against the sources. Nicer - # would be to run xdist and against the installed application, so that we also - # cover the impact of installing the application. - test-api = runTests "rhodecode/api"; - test-functional = runTests "rhodecode/tests/functional"; - test-rest = runTests "rhodecode/tests --ignore=rhodecode/tests/functional"; - test-full = runTests "rhodecode"; - - docs = mkDocs "docs"; - - aggregate = pkgs.releaseTools.aggregate { - name = "aggregated-jobs"; - constituents = [ - jobs.build - jobs.test-api - jobs.test-rest - jobs.docs - ]; - }; - }; - -in jobs +in { + build = enterprise_ce; +} diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ -## core -setuptools==39.2.0 -setuptools-scm==1.15.6 +## dependencies +setuptools-scm==2.1.0 amqp==2.3.1 authomatic==0.1.0.post1 babel==1.3 @@ -59,7 +58,7 @@ pyramid-mako==1.0.2 pyramid==1.9.2 pysqlite==2.8.3 python-dateutil -python-ldap==2.4.45 +python-ldap==3.1.0 python-memcached==1.59 python-pam==1.8.2 pytz==2018.4 diff --git a/rhodecode/config/licenses.json b/rhodecode/config/licenses.json --- a/rhodecode/config/licenses.json +++ b/rhodecode/config/licenses.json @@ -310,11 +310,7 @@ }, "python2.7-requests-2.9.1": { "Apache License 2.0": "http://spdx.org/licenses/Apache-2.0" - }, - "python2.7-setuptools-19.4": { - "Python Software Foundation License version 2": "http://spdx.org/licenses/Python-2.0", - "Zope Public License 2.0": "http://spdx.org/licenses/ZPL-2.0" - }, + }, "python2.7-setuptools-scm-1.15.6": { "MIT License": "http://spdx.org/licenses/MIT" }, diff --git a/shell.nix b/shell.nix --- a/shell.nix +++ b/shell.nix @@ -1,6 +1,9 @@ +# This file contains the adjustments which are desired for a development +# environment. + { pkgs ? (import {}) , pythonPackages ? "python27Packages" -, doCheck ? true +, doCheck ? false , sourcesOverrides ? {} , doDevelopInstall ? true }: @@ -10,7 +13,10 @@ let sources = (pkgs.config.rc.sources or {}) // sourcesOverrides; enterprise-ce = import ./default.nix { - inherit pkgs pythonPackages doCheck; + inherit + pkgs + pythonPackages + doCheck; }; ce-pythonPackages = enterprise-ce.pythonPackages; @@ -22,14 +28,17 @@ let let path = pkgs.lib.attrByPath [attributeName] null sources; doIt = doDevelopInstall && path != null; + in - pkgs.lib.optionalString doIt ( - builtins.trace "Develop install of ${attributeName} from ${path}" '' - echo "Develop install of '${attributeName}' from '${path}' [BEGIN]" + # do develop installation with empty hosts to skip any package duplicates to + # be replaced. This only pushes the package to be locally available + pkgs.lib.optionalString doIt ('' + echo "[BEGIN] Develop install of '${attributeName}' from '${path}'" pushd ${path} python setup.py develop --prefix $tmp_path --allow-hosts "" popd - echo "Develop install of '${attributeName}' from '${path}' [DONE]" + echo "[DONE] Develop install of '${attributeName}' from '${path}'" + echo "" ''); # This method looks up a path from `pkgs.config.rc.sources` and imports the @@ -38,13 +47,16 @@ let optionalDevelopInstallBuildInputs = attributeName: let path = pkgs.lib.attrByPath [attributeName] null sources; + doIt = doDevelopInstall && path != null && pkgs.lib.pathExists "${nixFile}"; nixFile = "${path}/default.nix"; - doIt = doDevelopInstall && path != null && pkgs.lib.pathExists "${nixFile}"; + derivate = import "${nixFile}" { inherit doCheck pkgs pythonPackages; }; in - pkgs.lib.lists.optionals doIt derivate.propagatedNativeBuildInputs; + pkgs.lib.lists.optionals doIt ( + derivate.propagatedBuildInputs + ); developInstalls = [ "rhodecode-vcsserver" ]; @@ -53,31 +65,52 @@ in enterprise-ce.override (attrs: { # make development a little bit more convenient. src = null; + # Add dependencies which are useful for the development environment. buildInputs = attrs.buildInputs ++ - pkgs.lib.lists.concatMap optionalDevelopInstallBuildInputs developInstalls ++ (with ce-pythonPackages; [ bumpversion invoke ipdb ]); - # Somewhat snappier setup of the development environment - # TODO: think of supporting a stable path again, so that multiple shells - # can share it. - preShellHook = enterprise-ce.linkNodeAndBowerPackages + '' + # place to inject some required libs from develop installs + propagatedBuildInputs = + attrs.propagatedBuildInputs ++ + pkgs.lib.lists.concatMap optionalDevelopInstallBuildInputs developInstalls; + + + # Make sure we execute both hooks + shellHook = '' + runHook preShellHook + runHook postShellHook + ''; + + preShellHook = '' + echo "Entering CE-Shell" + # Custom prompt to distinguish from other dev envs. export PS1="\n\[\033[1;32m\][CE-shell:\w]$\[\033[0m\] " + echo "Building frontend assets" + ${enterprise-ce.linkNodeAndBowerPackages} + # Setup a temporary directory. tmp_path=$(mktemp -d) export PATH="$tmp_path/bin:$PATH" export PYTHONPATH="$tmp_path/${ce-pythonPackages.python.sitePackages}:$PYTHONPATH" mkdir -p $tmp_path/${ce-pythonPackages.python.sitePackages} - # Develop installations + # Develop installation + echo "[BEGIN]: develop install of rhodecode-enterprise-ce" python setup.py develop --prefix $tmp_path --allow-hosts "" - echo "Additional develop installs" - '' + pkgs.lib.strings.concatMapStrings optionalDevelopInstall developInstalls; + ''; + + postShellHook = '' + echo "** Additional develop installs **" + '' + + pkgs.lib.strings.concatMapStrings optionalDevelopInstall developInstalls + + '' + ''; })