release: Merge default into stable for release preparation

docs/pip2nix.ini

@@ -0,0 +1,3 b''
+[pip2nix]
+requirements = -r ./requirements_docs.txt
+output = ./python-packages-generated.nix

docs/python-packages-generated.nix

@@ -0,0 +1,221 b''
+# Generated by pip2nix 0.8.0.dev1
+# See https://github.com/johbo/pip2nix
+
+{ pkgs, fetchurl, fetchgit, fetchhg }:
+
+self: super: {
+  "alabaster" = super.buildPythonPackage {
+    name = "alabaster-0.7.11";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/3f/46/9346ea429931d80244ab7f11c4fce83671df0b7ae5a60247a2b588592c46/alabaster-0.7.11.tar.gz";
+      sha256 = "1mvm69xsn5xf1jc45kdq1mn0yq0pfn54mv2jcww4s1vwqx6iyfxn";
+    };
+  };
+  "babel" = super.buildPythonPackage {
+    name = "babel-2.6.0";
+    doCheck = false;
+    propagatedBuildInputs = [
+      self."pytz"
+    ];
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/be/cc/9c981b249a455fa0c76338966325fc70b7265521bad641bf2932f77712f4/Babel-2.6.0.tar.gz";
+      sha256 = "08rxmbx2s4irp0w0gmn498vns5xy0fagm0fg33xa772jiks51flc";
+    };
+  };
+  "certifi" = super.buildPythonPackage {
+    name = "certifi-2018.8.24";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/e1/0f/f8d5e939184547b3bdc6128551b831a62832713aa98c2ccdf8c47ecc7f17/certifi-2018.8.24.tar.gz";
+      sha256 = "0f0nhrj9mlrf79iway4578wrsgmjh0fmacl9zv8zjckdy7b90rip";
+    };
+  };
+  "chardet" = super.buildPythonPackage {
+    name = "chardet-3.0.4";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/fc/bb/a5768c230f9ddb03acc9ef3f0d4a3cf93462473795d18e9535498c8f929d/chardet-3.0.4.tar.gz";
+      sha256 = "1bpalpia6r5x1kknbk11p1fzph56fmmnp405ds8icksd3knr5aw4";
+    };
+  };
+  "docutils" = super.buildPythonPackage {
+    name = "docutils-0.14";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/84/f4/5771e41fdf52aabebbadecc9381d11dea0fa34e4759b4071244fa094804c/docutils-0.14.tar.gz";
+      sha256 = "0x22fs3pdmr42kvz6c654756wja305qv6cx1zbhwlagvxgr4xrji";
+    };
+  };
+  "idna" = super.buildPythonPackage {
+    name = "idna-2.7";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/65/c4/80f97e9c9628f3cac9b98bfca0402ede54e0563b56482e3e6e45c43c4935/idna-2.7.tar.gz";
+      sha256 = "05jam7d31767dr12x0rbvvs8lxnpb1mhdb2zdlfxgh83z6k3hjk8";
+    };
+  };
+  "imagesize" = super.buildPythonPackage {
+    name = "imagesize-1.1.0";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/41/f5/3cf63735d54aa9974e544aa25858d8f9670ac5b4da51020bbfc6aaade741/imagesize-1.1.0.tar.gz";
+      sha256 = "1dg3wn7qpwmhgqc0r9na2ding1wif9q5spz3j9zn2riwphc2k0zk";
+    };
+  };
+  "jinja2" = super.buildPythonPackage {
+    name = "jinja2-2.9.6";
+    doCheck = false;
+    propagatedBuildInputs = [
+      self."markupsafe"
+    ];
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/90/61/f820ff0076a2599dd39406dcb858ecb239438c02ce706c8e91131ab9c7f1/Jinja2-2.9.6.tar.gz";
+      sha256 = "1zzrkywhziqffrzks14kzixz7nd4yh2vc0fb04a68vfd2ai03anx";
+    };
+  };
+  "markupsafe" = super.buildPythonPackage {
+    name = "markupsafe-1.0";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/4d/de/32d741db316d8fdb7680822dd37001ef7a448255de9699ab4bfcbdf4172b/MarkupSafe-1.0.tar.gz";
+      sha256 = "0rdn1s8x9ni7ss8rfiacj7x1085lx8mh2zdwqslnw8xc3l4nkgm6";
+    };
+  };
+  "packaging" = super.buildPythonPackage {
+    name = "packaging-17.1";
+    doCheck = false;
+    propagatedBuildInputs = [
+      self."pyparsing"
+      self."six"
+    ];
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/77/32/439f47be99809c12ef2da8b60a2c47987786d2c6c9205549dd6ef95df8bd/packaging-17.1.tar.gz";
+      sha256 = "0nrpayk8kij1zm9sjnk38ldz3a6705ggvw8ljylqbrb4vmqbf6gh";
+    };
+  };
+  "pygments" = super.buildPythonPackage {
+    name = "pygments-2.2.0";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/71/2a/2e4e77803a8bd6408a2903340ac498cb0a2181811af7c9ec92cb70b0308a/Pygments-2.2.0.tar.gz";
+      sha256 = "1k78qdvir1yb1c634nkv6rbga8wv4289xarghmsbbvzhvr311bnv";
+    };
+  };
+  "pyparsing" = super.buildPythonPackage {
+    name = "pyparsing-2.2.0";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/3c/ec/a94f8cf7274ea60b5413df054f82a8980523efd712ec55a59e7c3357cf7c/pyparsing-2.2.0.tar.gz";
+      sha256 = "016b9gh606aa44sq92jslm89bg874ia0yyiyb643fa6dgbsbqch8";
+    };
+  };
+  "pytz" = super.buildPythonPackage {
+    name = "pytz-2018.4";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/10/76/52efda4ef98e7544321fd8d5d512e11739c1df18b0649551aeccfb1c8376/pytz-2018.4.tar.gz";
+      sha256 = "0jgpqx3kk2rhv81j1izjxvmx8d0x7hzs1857pgqnixic5wq2ar60";
+    };
+  };
+  "requests" = super.buildPythonPackage {
+    name = "requests-2.19.1";
+    doCheck = false;
+    propagatedBuildInputs = [
+      self."chardet"
+      self."idna"
+      self."urllib3"
+      self."certifi"
+    ];
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/54/1f/782a5734931ddf2e1494e4cd615a51ff98e1879cbe9eecbdfeaf09aa75e9/requests-2.19.1.tar.gz";
+      sha256 = "0snf8xxdzsgh1x2zv3vilvbrv9jbpmnfagzzb1rjmmvflckdh8pc";
+    };
+  };
+  "setuptools" = super.buildPythonPackage {
+    name = "setuptools-40.2.0";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/ef/1d/201c13e353956a1c840f5d0fbf0461bd45bbd678ea4843ebf25924e8984c/setuptools-40.2.0.zip";
+      sha256 = "19ng5m7kigllg3x96c91y3a2k28g6kwnbb1v4warrnp4xma1v227";
+    };
+  };
+  "six" = super.buildPythonPackage {
+    name = "six-1.11.0";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/16/d8/bc6316cf98419719bd59c91742194c111b6f2e85abac88e496adefaf7afe/six-1.11.0.tar.gz";
+      sha256 = "1scqzwc51c875z23phj48gircqjgnn3af8zy2izjwmnlxrxsgs3h";
+    };
+  };
+  "snowballstemmer" = super.buildPythonPackage {
+    name = "snowballstemmer-1.2.1";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/20/6b/d2a7cb176d4d664d94a6debf52cd8dbae1f7203c8e42426daa077051d59c/snowballstemmer-1.2.1.tar.gz";
+      sha256 = "0a0idq4y5frv7qsg2x62jd7rd272749xk4x99misf5rcifk2d7wi";
+    };
+  };
+  "sphinx" = super.buildPythonPackage {
+    name = "sphinx-1.7.8";
+    doCheck = false;
+    propagatedBuildInputs = [
+      self."six"
+      self."jinja2"
+      self."pygments"
+      self."docutils"
+      self."snowballstemmer"
+      self."babel"
+      self."alabaster"
+      self."imagesize"
+      self."requests"
+      self."setuptools"
+      self."packaging"
+      self."sphinxcontrib-websupport"
+      self."typing"
+    ];
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/ac/54/4ef326d0c654da1ed91341a7a1f43efc18a8c770ddd2b8e45df97cb79d82/Sphinx-1.7.8.tar.gz";
+      sha256 = "1ryz0w4c31930f1br2sjwrxwx9cmsy7cqdb0d81g98n9bj250w50";
+    };
+  };
+  "sphinx-rtd-theme" = super.buildPythonPackage {
+    name = "sphinx-rtd-theme-0.4.1";
+    doCheck = false;
+    propagatedBuildInputs = [
+      self."sphinx"
+    ];
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/f2/b0/a1933d792b806118ddbca6699f2e2c844d9b1b16e84a89d7effd5cd2a800/sphinx_rtd_theme-0.4.1.tar.gz";
+      sha256 = "1xkyqam8dzbjaymdyvkiif85m4y3jf8crdiwlgcfp8gqcj57aj9v";
+    };
+  };
+  "sphinxcontrib-websupport" = super.buildPythonPackage {
+    name = "sphinxcontrib-websupport-1.1.0";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/07/7a/e74b06dce85555ffee33e1d6b7381314169ebf7e31b62c18fcb2815626b7/sphinxcontrib-websupport-1.1.0.tar.gz";
+      sha256 = "1ff3ix76xi1y6m99qxhaq5161ix9swwzydilvdya07mgbcvpzr4x";
+    };
+  };
+  "typing" = super.buildPythonPackage {
+    name = "typing-3.6.6";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/bf/9b/2bf84e841575b633d8d91ad923e198a415e3901f228715524689495b4317/typing-3.6.6.tar.gz";
+      sha256 = "0ba9acs4awx15bf9v3nrs781msbd2nx826906nj6fqks2bvca9s0";
+    };
+  };
+  "urllib3" = super.buildPythonPackage {
+    name = "urllib3-1.23";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/3c/d2/dc5471622bd200db1cd9319e02e71bc655e9ea27b8e0ce65fc69de0dac15/urllib3-1.23.tar.gz";
+      sha256 = "1bvbd35q3zdcd7gsv38fwpizy7p06dr0154g5gfybrvnbvhwb2m6";
+    };
+  };
+
+### Test requirements
+
+  
+}

docs/release-notes/release-notes-4.13.0.rst

@@ -0,0 +1,133 b''
+|RCE| 4.13.0 |RNS|
+------------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2018-09-05
+
+
+New Features
+^^^^^^^^^^^^
+
+- Branch permissions: new set of permissions were added to control branch modification.
+  There are 4 basic permissions that can be set for branch names/branch patterns:
+  * no-access (any modification for given branch is forbidden)
+  * web-merge (modify branch by web based PR merge)
+  * push (only non-forced modification on branch are allowed)
+  * forced push (all modification to branch are allowed)
+  Available in EE edition only.
+- Unified search and repo-switcher: a new persistent search box is now present allowing
+  to search for repositories, repository groups, commits (using full text search),
+  users, user-groups. Replaces redundant quick-filters/repo switcher.
+- Default Reviewers: added possibility to match against regex based pattern as
+  alternative syntax to glob which didn't handle all the cases.
+- Built-in Error tracker: added new exception tracking capability. All errors are now
+  tracked and stored. This allows instance admins to see potential problems without
+  access to the machine and logs.
+- User Groups: each user group which users have access to expose public profile link.
+  It's possible to view the members of a group before attaching it to any resource.
+- New caching framework: existing Beaker cache was completely replaced by dogpile.cache
+  library. This new cache framework in addition to solving multiple
+  performance/reliability problems of Beaker is used to cache permissions tree.
+  This gives huge performance boosts for very large and complex permission trees.
+- Pull Requests: description field is now allowed to use a RST/Markdown syntax.
+- SVN: added support for SVN 1.10 release line.
+
+
+General
+^^^^^^^
+
+- Google: updated google auth plugin with latest API changes.
+- Frontend: Switched to Polymer 2.0.
+- Events: added a default timeout for operation calling the endpoint url, so
+  they won't block forever.
+- SQLAlchemy: allow DB connection ping/refresh using dedicated flag from .ini file.
+  `sqlalchemy.db1.ping_connection = true`
+- Pull Requests: added option to force-refresh merge workspace in case of problems.
+  Adding GET param `?force_refresh=1` into PR page triggers the refresh.
+- Pull Requests: show more info about version of comment vs latest version.
+- Diffs: skip line numbers during copy from a diff view.
+- License: use simple cache to read license info.
+  Due to the complex and expensive encryption, this reduces requests time by ~10ms.
+- Debug: add new custom logging to track unique requests across systems.
+  Allows tracking single requests in very busy system by unique ID added into logging system.
+- Configuration: .ini files now can replace a special placeholders e.g "{ENV_NAME}"
+  into a value from the ENVIRONMENT. Allows easier setup in Docker and similar.
+- Backend: don't support vcsserver.scm_app anymore, now it uses http even if scm_app
+  is specified.
+- Repositories: re-order creation/fork forms for better UX and consistency.
+- UI: Add the number of inactive users in _admin/users and _admin/user_groups
+- UX: updated registration form to better indicate what is the process of binding a
+  RhodeCode account with external one like Google.
+- API: pull-requests allow automatic title generation via API
+- VCSServer: errors: use a better interface to track exceptions and tracebacks.
+- VCSServer: caches: replaced beaker with dogpile cache.
+- GIT: use GIT_DISCOVERY_ACROSS_FILESYSTEM for better compatibility on NFS servers.
+- Dependencies: bumped mercurial to 4.6.2
+- Dependencies: bumped evolve to 8.0.1
+- Dependencies: bumped hgsubversion to 1.9.2
+- Dependencies: bumped git version to 2.16.4
+- Dependencies: bumped SVN to 1.10.2
+- Dependencies: added alternative pymysql drivers for mysql
+- NIX: updated to 18.03 nix packages, now shipped with python 2.7.15
+  release and multiple other new libraries.
+
+
+Security
+^^^^^^^^
+
+- Mercurial: general protocol security updates.
+  * Fixes Mercurial's CVE for lack of permissions checking on mercurial batch commands.
+  * Introduced more strict checks for permissions, now they default to push instead of pull.
+  * Decypher batch commands and pick top-most permission to be required.
+  * This follows changes in Mercurial CORE after 4.6.1 release.
+- Fixed bug in bleach sanitizer allowing certain custom payload to bypass it. Now
+  we always fails if sanitizing fails. This could lead to stored XSS
+- Fixed stored XSS in binary file rendering.
+- Fixed stored XSS in repo forks datagrid.
+
+
+Performance
+^^^^^^^^^^^
+
+- Permissions: Permission trees for users and now cached, after calculation.
+  This reduces response time for some pages dramatically.
+  In case of any permission changes caches are invalidated.
+- Core: new dogpile.cache based cache framework was introduced, which is faster than
+  previously used Beaker.
+
+
+Fixes
+^^^^^
+
+- Audit Logs: store properly IP for certain events.
+- External Auth: pass along came_from into the url so we get properly
+  redirected back after logging using external auth provider.
+- Pull Requests: lock submit on pull request to prevent double submission on a fast click.
+- Pull Requests: fixed a case of unresolved comments attached to removed file in pull request.
+  That prevented from closing it.
+- Pull Requests: use numeric repo id for creation of shadow repos. Fixes a problem
+  when repository is renamed during PR lifetime.
+- API: fixed creation of a pull request with default reviewer rules.
+- Default Reviewers: fixed voting rule calculation on user group.
+- Pull Requests: in GIT use force fetch and update for target ref.
+  This solves a case when in PR a target repository is force updated (by push force)
+  and is out of sync.
+- VCSServer: detect early potential locale problem, and fallback to LC_ALL=C,
+  instead of crashing vcsserver.
+- Pull Requests: use a safer way of destroying shadow repositories.
+  Fixes some problems in NFS storage and big repositories
+
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- The direct backend `vcsserver.scm_app` is not supported anymore. This backed was
+  already deprecated some time ago. Now it will use `http` mode even if scm_app is
+  specified. Please contact us in case you still use it, and not sure how to upgrade.
+- New dogpile cache settings are not ported to converted .ini. If users want to do
+  adjustments please copy the settings over dogpile cache section from a newly
+  generated rhodecode.template.ini file. This file is stored next to rhodecode.ini
+- SVN 1.10.2 was introduced in this release. Please make sure to update your
+  mod_dav to the same version for best compatibility.

pkgs/README.rst

@@ -0,0 +1,52 b''
+
+==============================
+ Generate the Nix expressions
+==============================
+
+Details can be found in the repository of `RhodeCode Enterprise CE`_ inside of
+the file `docs/contributing/dependencies.rst`.
+
+Start the environment as follows:
+
+.. code:: shell
+
+   nix-shell pkgs/shell-generate.nix
+
+
+
+Python dependencies
+===================
+
+.. code:: shell
+
+   pip2nix generate --licenses
+   # or
+   nix-shell pkgs/shell-generate.nix --command "pip2nix generate --licenses"
+
+
+NodeJS dependencies
+===================
+
+.. code:: shell
+
+   rm -rf node_modules
+   nix-shell pkgs/shell-generate.nix --command "
+       node2nix --input package.json \
+                -o pkgs/node-packages.nix \
+                -e pkgs/node-env.nix \
+                -c pkgs/node-default.nix \
+                -d --flatten --nodejs-6 "
+
+
+
+Bower dependencies
+==================
+
+.. code:: shell
+
+   nix-shell pkgs/shell-generate.nix --command "bower2nix bower.json pkgs/bower-packages.nix"
+
+
+.. Links
+
+.. _RhodeCode Enterprise CE: https://code.rhodecode.com/rhodecode-enterprise-ce

pkgs/nix-common/pip2nix.nix

@@ -0,0 +1,17 b''
+{ pkgs
+, pythonPackages
+}:
+
+rec {
+  pip2nix-src = pkgs.fetchzip {
+    url = https://github.com/johbo/pip2nix/archive/51e6fdae34d0e8ded9efeef7a8601730249687a6.tar.gz;
+    sha256 = "02a4jjgi7lsvf8mhrxsd56s9a3yg20081rl9bgc2m84w60v2gbz2";
+  };
+
+  pip2nix = import pip2nix-src {
+    inherit
+      pkgs
+      pythonPackages;
+  };
+
+}

pkgs/patch_beaker/patch-beaker-metadata-reuse.diff

@@ -0,0 +1,13 b''
+diff -rup Beaker-1.9.1-orig/beaker/ext/database.py Beaker-1.9.1/beaker/ext/database.py
+--- Beaker-1.9.1-orig/beaker/ext/database.py	2018-05-22 18:22:34.802619619 +0200
++++ Beaker-1.9.1/beaker/ext/database.py	2018-05-22 17:07:14.048335196 +0200
+@@ -91,7 +91,8 @@ class DatabaseNamespaceManager(OpenResou
+                              sa.Column('created', types.DateTime, nullable=False),
+                              sa.Column('data', types.PickleType, nullable=False),
+                              sa.UniqueConstraint('namespace'),
+-                             schema=schema_name if schema_name else meta.schema
++                             schema=schema_name if schema_name else meta.schema,
++                             extend_existing=True
+             )
+             cache.create(checkfirst=True)
+             return cache

pkgs/shell-generate.nix

@@ -0,0 +1,53 b''
+{ pkgs ? (import <nixpkgs> {})
+, pythonPackages ? "python27Packages"
+}:
+
+with pkgs.lib;
+
+let _pythonPackages = pythonPackages; in
+let
+  pythonPackages = getAttr _pythonPackages pkgs;
+
+  pip2nix = import ./nix-common/pip2nix.nix {
+    inherit
+      pkgs
+      pythonPackages;
+  };
+
+in
+
+pkgs.stdenv.mkDerivation {
+  name = "pip2nix-generated";
+  buildInputs = [
+    # Allows to generate python packages
+    pip2nix.pip2nix
+    pythonPackages.pip-tools
+
+    # Allows to generate bower dependencies
+    pkgs.nodePackages.bower2nix
+
+    # Allows to generate node dependencies
+    pkgs.nodePackages.node2nix
+
+    # We need mysql_config to be around
+    pkgs.mysql
+
+    # We need postgresql to be around
+    pkgs.postgresql
+
+    # Curl is needed for pycurl
+    pkgs.curl
+  ];
+
+  shellHook = ''
+    runHook preShellHook
+    runHook postShellHook
+  '';
+
+  preShellHook = ''
+    echo "Starting Generate Shell"
+    # Custom prompt to distinguish from other dev envs.
+    export PS1="\n\[\033[1;32m\][Generate-shell:\w]$\[\033[0m\] "
+    export PYCURL_SSL_LIBRARY=openssl
+  '';
+}

.bumpversion.cfg

@@ -1,5 +1,5 b''
 [bumpversion]
-current_version = 4.12.4
+current_version = 4.13.0
 message = release: Bump version {current_version} to {new_version}
 
 [bumpversion:file:rhodecode/VERSION]

.release.cfg

@@ -5,25 +5,20 b' done = false'
 done = true
 
 [task:rc_tools_pinned]
-done = true
 
 [task:fixes_on_stable]
-done = true
 
 [task:pip2nix_generated]
-done = true
 
 [task:changelog_updated]
-done = true
 
 [task:generate_api_docs]
-done = true
+
+[task:updated_translation]
 
 [release]
-state = prepared
-version = 4.12.4
-
-[task:updated_translation]
+state = in_progress
+version = 4.13.0
 
 [task:generate_js_routes]
 

MANIFEST.in

@@ -26,6 +26,8 b' recursive-include rhodecode *.mako'
 # 502 page
 include rhodecode/public/502.html
 
+# robots
+include rhodecode/public/robots.txt
 
 # images, css
 include rhodecode/public/css/*.css

README.rst

@@ -7,8 +7,9 b' About'
 
 ``RhodeCode`` is a fast and powerful management tool for Mercurial_ and GIT_
 and Subversion_ with a built in push/pull server, full text search,
-pull requests and powerfull code-review system. It works on http/https and
+pull requests and powerful code-review system. It works on http/https and
 has a few unique features like:
+
  - plugable architecture
  - advanced permission system with IP restrictions
  - rich set of authentication plugins including LDAP,
@@ -78,10 +79,9 b' Listed bellow are various support resour'
 
 - Official issue tracker `RhodeCode Issue tracker <https://issues.rhodecode.com>`_
 
-- Search our community portal `Community portal <https://community.rhodecode.com>`_
+- Official Slack Community Channel `RhodeCode Slack <https://rhodecode.com/join>`_
 
-- Join #rhodecode on FreeNode (irc.freenode.net)
-  or use http://webchat.freenode.net/?channels=rhodecode for web access to irc.
+- Search our community portal `Community portal <https://community.rhodecode.com>`_
 
 - You can also follow RhodeCode on twitter **@RhodeCode** where we often post
   news and other interesting stuff about RhodeCode.

bower.json

@@ -3,15 +3,15 b''
   "description": "User interface for elements for rhodecode",
   "main": "index.html",
   "dependencies": {
-    "webcomponentsjs": "^0.7.22",
-    "polymer": "Polymer/polymer#^1.6.1",
-    "paper-button": "PolymerElements/paper-button#^1.0.13",
-    "paper-spinner": "PolymerElements/paper-spinner#^1.2.0",
-    "paper-tooltip": "PolymerElements/paper-tooltip#^1.1.2",
-    "paper-toast": "PolymerElements/paper-toast#^1.3.0",
-    "paper-toggle-button": "PolymerElements/paper-toggle-button#^1.2.0",
-    "iron-ajax": "PolymerElements/iron-ajax#^1.4.3",
-    "iron-autogrow-textarea": "PolymerElements/iron-autogrow-textarea#^1.0.13",
-    "iron-a11y-keys": "PolymerElements/iron-a11y-keys#^1.0.6"
+    "webcomponentsjs": "^1.2.3",
+    "polymer": "Polymer/polymer#^2.6.0",
+    "paper-button": "PolymerElements/paper-button#^2.1.2",
+    "paper-spinner": "PolymerElements/paper-spinner#^2.1.0",
+    "paper-tooltip": "PolymerElements/paper-tooltip#^2.1.1",
+    "paper-toast": "PolymerElements/paper-toast#^2.1.2",
+    "paper-toggle-button": "PolymerElements/paper-toggle-button#^2.1.1",
+    "iron-ajax": "PolymerElements/iron-ajax#^2.1.3",
+    "iron-autogrow-textarea": "PolymerElements/iron-autogrow-textarea#^2.2.0",
+    "iron-a11y-keys": "PolymerElements/iron-a11y-keys#^2.1.2"
   }
 }

configs/development.ini

@@ -2,7 +2,6 b''
 
 ################################################################################
 ##                 RHODECODE COMMUNITY EDITION CONFIGURATION                  ##
-# The %(here)s variable will be replaced with the parent directory of this file#
 ################################################################################
 
 [DEFAULT]
@@ -46,10 +45,9 b' debug = true'
 host = 127.0.0.1
 port = 5000
 
-##################################
-##     WAITRESS WSGI SERVER     ##
-## Recommended for Development  ##
-##################################
+###########################################################
+## WAITRESS WSGI SERVER - Recommended for Development  ####
+###########################################################
 
 use = egg:waitress#main
 ## number of worker threads
@@ -67,15 +65,15 b' asyncore_use_poll = true'
 ## run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
 
 #use = egg:gunicorn#main
-## Sets the number of process workers. You must set `instance_id = *`
-## when this option is set to more than one worker, recommended
-## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
-## The `instance_id = *` must be set in the [app:main] section below
+## Sets the number of process workers. More workers means more concurent connections
+## RhodeCode can handle at the same time. Each additional worker also it increases
+## memory usage as each has it's own set of caches.
+## Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more
+## than 8-10 unless for really big deployments .e.g 700-1000 users.
+## `instance_id = *` must be set in the [app:main] section below (which is the default)
+## when using more than 1 worker.
 #workers = 2
-## number of threads for each of the worker, must be set to 1 for gevent
-## generally recommended to be at 1
-#threads = 1
-## process name
+## process name visible in process list
 #proc_name = rhodecode
 ## type of worker class, one of sync, gevent
 ## recommended for bigger setup is using of of other than sync one
@@ -103,6 +101,11 b' use = egg:PasteDeploy#prefix'
 prefix = /
 
 [app:main]
+## The %(here)s variable will be replaced with the absolute path of parent directory
+## of this file
+## In addition ENVIRONMENT variables usage is possible, e.g
+## sqlalchemy.db1.url = {ENV_RC_DB_URL}
+
 use = egg:rhodecode-enterprise-ce
 
 ## enable proxy prefix middleware, defined above
@@ -148,12 +151,12 b' gzip_responses = false'
 ## autogenerate javascript routes file on startup
 generate_js_files = false
 
-## Optional Languages
-## en(default), be, de, es, fr, it, ja, pl, pt, ru, zh
+## System global default language.
+## All available languages: en(default), be, de, es, fr, it, ja, pl, pt, ru, zh
 lang = en
 
-## perform a full repository scan on each server start, this should be
-## set to false after first startup, to allow faster server restarts.
+## Perform a full repository scan and import on each server start.
+## Settings this to true could lead to very long startup time.
 startup.import_repos = false
 
 ## Uncomment and set this path to use archive download cache.
@@ -167,23 +170,23 b' startup.import_repos = false'
 ## SSH calls. Set this for events to receive proper url for SSH calls.
 app.base_url = http://rhodecode.local
 
-## change this to unique ID for security
+## Unique application ID. Should be a random unique string for security.
 app_instance_uuid = rc-production
 
-## cut off limit for large diffs (size in bytes). If overall diff size on
+## Cut off limit for large diffs (size in bytes). If overall diff size on
 ## commit, or pull request exceeds this limit this diff will be displayed
 ## partially. E.g 512000 == 512Kb
 cut_off_limit_diff = 512000
 
-## cut off limit for large files inside diffs (size in bytes). Each individual
+## Cut off limit for large files inside diffs (size in bytes). Each individual
 ## file inside diff which exceeds this limit will be displayed partially.
 ##  E.g 128000 == 128Kb
 cut_off_limit_file = 128000
 
-## use cache version of scm repo everywhere
+## use cached version of vcs repositories everywhere. Recommended to be `true`
 vcs_full_cache = true
 
-## force https in RhodeCode, fixes https redirects, assumes it's always https
+## Force https in RhodeCode, fixes https redirects, assumes it's always https.
 ## Normally this is controlled by proper http flags sent from http server
 force_https = false
 
@@ -230,7 +233,7 b' gist_alias_url ='
 #    GistView:*
 api_access_controllers_whitelist =
 
-## default encoding used to convert from and to unicode
+## Default encoding used to convert from and to unicode
 ## can be also a comma separated list of encoding in case of mixed encodings
 default_encoding = UTF-8
 
@@ -277,8 +280,8 b' allow_repo_location_change = true'
 ## allows to setup custom hooks in settings page
 allow_custom_hooks_settings = true
 
-## generated license token, goto license page in RhodeCode settings to obtain
-## new token
+## Generated license token required for EE edition license.
+## New generated token value can be found in Admin > settings > license page.
 license_token =
 
 ## supervisor connection uri, for managing supervisor and logs.
@@ -308,65 +311,66 b' celery.max_tasks_per_child = 100'
 ## tasks will never be sent to the queue, but executed locally instead.
 celery.task_always_eager = false
 
-####################################
-###         BEAKER CACHE        ####
-####################################
-# default cache dir for templates.  Putting this into a ramdisk
-## can boost performance, eg. %(here)s/data_ramdisk
+#####################################
+###         DOGPILE CACHE        ####
+#####################################
+## Default cache dir for caches.  Putting this into a ramdisk
+## can boost performance, eg. /tmpfs/data_ramdisk, however this directory might require
+## large amount of space
 cache_dir = %(here)s/data
 
-## locking and default file storage for Beaker. Putting this into a ramdisk
-## can boost performance, eg. %(here)s/data_ramdisk/cache/beaker_data
-beaker.cache.data_dir = %(here)s/data/cache/beaker_data
-beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock
-
-beaker.cache.regions = super_short_term, short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long
+## `cache_perms` cache settings for permission tree, auth TTL.
+rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
+rc_cache.cache_perms.expiration_time = 300
 
-beaker.cache.super_short_term.type = memory
-beaker.cache.super_short_term.expire = 10
-beaker.cache.super_short_term.key_length = 256
+## alternative `cache_perms` redis backend with distributed lock
+#rc_cache.cache_perms.backend = dogpile.cache.rc.redis
+#rc_cache.cache_perms.expiration_time = 300
+## redis_expiration_time needs to be greater then expiration_time
+#rc_cache.cache_perms.arguments.redis_expiration_time = 7200
+#rc_cache.cache_perms.arguments.socket_timeout = 30
+#rc_cache.cache_perms.arguments.host = localhost
+#rc_cache.cache_perms.arguments.port = 6379
+#rc_cache.cache_perms.arguments.db = 0
+#rc_cache.cache_perms.arguments.distributed_lock = true
 
-beaker.cache.short_term.type = memory
-beaker.cache.short_term.expire = 60
-beaker.cache.short_term.key_length = 256
-
-beaker.cache.long_term.type = memory
-beaker.cache.long_term.expire = 36000
-beaker.cache.long_term.key_length = 256
+## `cache_repo` cache settings for FileTree, Readme, RSS FEEDS
+rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace
+rc_cache.cache_repo.expiration_time = 2592000
 
-beaker.cache.sql_cache_short.type = memory
-beaker.cache.sql_cache_short.expire = 10
-beaker.cache.sql_cache_short.key_length = 256
+## alternative `cache_repo` redis backend with distributed lock
+#rc_cache.cache_repo.backend = dogpile.cache.rc.redis
+#rc_cache.cache_repo.expiration_time = 2592000
+## redis_expiration_time needs to be greater then expiration_time
+#rc_cache.cache_repo.arguments.redis_expiration_time = 2678400
+#rc_cache.cache_repo.arguments.socket_timeout = 30
+#rc_cache.cache_repo.arguments.host = localhost
+#rc_cache.cache_repo.arguments.port = 6379
+#rc_cache.cache_repo.arguments.db = 1
+#rc_cache.cache_repo.arguments.distributed_lock = true
 
-## default is memory cache, configure only if required
-## using multi-node or multi-worker setup
-#beaker.cache.auth_plugins.type = ext:database
-#beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock
-#beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode
-#beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode
-#beaker.cache.auth_plugins.sa.pool_recycle = 3600
-#beaker.cache.auth_plugins.sa.pool_size = 10
-#beaker.cache.auth_plugins.sa.max_overflow = 0
+## cache settings for SQL queries, this needs to use memory type backend
+rc_cache.sql_cache_short.backend = dogpile.cache.rc.memory_lru
+rc_cache.sql_cache_short.expiration_time = 30
 
-beaker.cache.repo_cache_long.type = memorylru_base
-beaker.cache.repo_cache_long.max_items = 4096
-beaker.cache.repo_cache_long.expire = 2592000
+## `cache_repo_longterm` cache for repo object instances, this needs to use memory
+## type backend as the objects kept are not pickle serializable
+rc_cache.cache_repo_longterm.backend = dogpile.cache.rc.memory_lru
+## by default we use 96H, this is using invalidation on push anyway
+rc_cache.cache_repo_longterm.expiration_time = 345600
+## max items in LRU cache, reduce this number to save memory, and expire last used
+## cached objects
+rc_cache.cache_repo_longterm.max_size = 10000
 
-## default is memorylru_base cache, configure only if required
-## using multi-node or multi-worker setup
-#beaker.cache.repo_cache_long.type = ext:memcached
-#beaker.cache.repo_cache_long.url = localhost:11211
-#beaker.cache.repo_cache_long.expire = 1209600
-#beaker.cache.repo_cache_long.key_length = 256
 
 ####################################
 ###       BEAKER SESSION        ####
 ####################################
 
 ## .session.type is type of storage options for the session, current allowed
-## types are file, ext:memcached, ext:database, and memory (default).
+## types are file, ext:memcached, ext:redis, ext:database, and memory (default).
 beaker.session.type = file
-beaker.session.data_dir = %(here)s/data/sessions/data
+beaker.session.data_dir = %(here)s/data/sessions
 
 ## db based session, fast, and allows easy management over logged in users
 #beaker.session.type = ext:database
@@ -502,10 +506,7 b' appenlight.log_namespace_blacklist ='
 ################################################################################
 #set debug = false
 
-
-##############
-## STYLING  ##
-##############
+# enable debug style page
 debug_style = true
 
 ###########################################
@@ -514,6 +515,9 b' debug_style = true'
 #sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
 #sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 #sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode
+# pymysql is an alternative driver for MySQL, use in case of problems with default one
+#sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode
+
 sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
 
 # see sqlalchemy docs for other advanced settings
@@ -533,6 +537,9 b' sqlalchemy.db1.convert_unicode = true'
 ## which defaults to five.
 #sqlalchemy.db1.max_overflow = 10
 
+## Connection check ping, used to detect broken database connections
+## could be enabled to better handle cases if MySQL has gone away errors
+#sqlalchemy.db1.ping_connection = true
 
 ##################
 ### VCS CONFIG ###
@@ -547,15 +554,18 b' vcs.server.protocol = http'
 
 ## Push/Pull operations protocol, available options are:
 ## `http` - use http-rpc backend (default)
-##
 vcs.scm_app_implementation = http
 
 ## Push/Pull operations hooks protocol, available options are:
 ## `http` - use http-rpc backend (default)
 vcs.hooks.protocol = http
 
+## Host on which this instance is listening for hooks. If vcsserver is in other location
+## this should be adjusted.
+vcs.hooks.host = 127.0.0.1
+
 vcs.server.log_level = debug
-## Start VCSServer with this instance as a subprocess, usefull for development
+## Start VCSServer with this instance as a subprocess, useful for development
 vcs.start_server = false
 
 ## List of enabled VCS backends, available options are:
@@ -638,7 +648,7 b' custom.conf = 1'
 ### LOGGING CONFIGURATION   ####
 ################################
 [loggers]
-keys = root, sqlalchemy, beaker, rhodecode, ssh_wrapper, celery
+keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
 
 [handlers]
 keys = console, console_sql
@@ -694,9 +704,12 b' level = DEBUG'
 formatter = color_formatter
 
 [handler_console_sql]
+# "level = DEBUG" logs SQL queries and results.
+# "level = INFO" logs SQL queries.
+# "level = WARN" logs neither.  (Recommended for production systems.)
 class = StreamHandler
 args = (sys.stderr, )
-level = DEBUG
+level = WARN
 formatter = color_formatter_sql
 
 ################
@@ -715,5 +728,5 b' datefmt = %Y-%m-%d %H:%M:%S'
 
 [formatter_color_formatter_sql]
 class = rhodecode.lib.logging_formatter.ColorFormatterSql
-format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
+format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S

configs/production.ini

@@ -2,7 +2,6 b''
 
 ################################################################################
 ##                 RHODECODE COMMUNITY EDITION CONFIGURATION                  ##
-# The %(here)s variable will be replaced with the parent directory of this file#
 ################################################################################
 
 [DEFAULT]
@@ -46,10 +45,9 b' debug = true'
 host = 127.0.0.1
 port = 5000
 
-##################################
-##     WAITRESS WSGI SERVER     ##
-## Recommended for Development  ##
-##################################
+###########################################################
+## WAITRESS WSGI SERVER - Recommended for Development  ####
+###########################################################
 
 #use = egg:waitress#main
 ## number of worker threads
@@ -67,21 +65,21 b' port = 5000'
 ## run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
 
 use = egg:gunicorn#main
-## Sets the number of process workers. You must set `instance_id = *`
-## when this option is set to more than one worker, recommended
-## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
-## The `instance_id = *` must be set in the [app:main] section below
+## Sets the number of process workers. More workers means more concurent connections
+## RhodeCode can handle at the same time. Each additional worker also it increases
+## memory usage as each has it's own set of caches.
+## Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more
+## than 8-10 unless for really big deployments .e.g 700-1000 users.
+## `instance_id = *` must be set in the [app:main] section below (which is the default)
+## when using more than 1 worker.
 workers = 2
-## number of threads for each of the worker, must be set to 1 for gevent
-## generally recommended to be at 1
-#threads = 1
-## process name
+## process name visible in process list
 proc_name = rhodecode
 ## type of worker class, one of sync, gevent
 ## recommended for bigger setup is using of of other than sync one
 worker_class = gevent
 ## The maximum number of simultaneous clients. Valid only for Gevent
-#worker_connections = 10
+worker_connections = 10
 ## max number of requests that worker will handle before being gracefully
 ## restarted, could prevent memory leaks
 max_requests = 1000
@@ -103,6 +101,11 b' use = egg:PasteDeploy#prefix'
 prefix = /
 
 [app:main]
+## The %(here)s variable will be replaced with the absolute path of parent directory
+## of this file
+## In addition ENVIRONMENT variables usage is possible, e.g
+## sqlalchemy.db1.url = {ENV_RC_DB_URL}
+
 use = egg:rhodecode-enterprise-ce
 
 ## enable proxy prefix middleware, defined above
@@ -123,12 +126,12 b' gzip_responses = false'
 ## autogenerate javascript routes file on startup
 generate_js_files = false
 
-## Optional Languages
-## en(default), be, de, es, fr, it, ja, pl, pt, ru, zh
+## System global default language.
+## All available languages: en(default), be, de, es, fr, it, ja, pl, pt, ru, zh
 lang = en
 
-## perform a full repository scan on each server start, this should be
-## set to false after first startup, to allow faster server restarts.
+## Perform a full repository scan and import on each server start.
+## Settings this to true could lead to very long startup time.
 startup.import_repos = false
 
 ## Uncomment and set this path to use archive download cache.
@@ -142,23 +145,23 b' startup.import_repos = false'
 ## SSH calls. Set this for events to receive proper url for SSH calls.
 app.base_url = http://rhodecode.local
 
-## change this to unique ID for security
+## Unique application ID. Should be a random unique string for security.
 app_instance_uuid = rc-production
 
-## cut off limit for large diffs (size in bytes). If overall diff size on
+## Cut off limit for large diffs (size in bytes). If overall diff size on
 ## commit, or pull request exceeds this limit this diff will be displayed
 ## partially. E.g 512000 == 512Kb
 cut_off_limit_diff = 512000
 
-## cut off limit for large files inside diffs (size in bytes). Each individual
+## Cut off limit for large files inside diffs (size in bytes). Each individual
 ## file inside diff which exceeds this limit will be displayed partially.
 ##  E.g 128000 == 128Kb
 cut_off_limit_file = 128000
 
-## use cache version of scm repo everywhere
+## use cached version of vcs repositories everywhere. Recommended to be `true`
 vcs_full_cache = true
 
-## force https in RhodeCode, fixes https redirects, assumes it's always https
+## Force https in RhodeCode, fixes https redirects, assumes it's always https.
 ## Normally this is controlled by proper http flags sent from http server
 force_https = false
 
@@ -205,7 +208,7 b' gist_alias_url ='
 #    GistView:*
 api_access_controllers_whitelist =
 
-## default encoding used to convert from and to unicode
+## Default encoding used to convert from and to unicode
 ## can be also a comma separated list of encoding in case of mixed encodings
 default_encoding = UTF-8
 
@@ -252,8 +255,8 b' allow_repo_location_change = true'
 ## allows to setup custom hooks in settings page
 allow_custom_hooks_settings = true
 
-## generated license token, goto license page in RhodeCode settings to obtain
-## new token
+## Generated license token required for EE edition license.
+## New generated token value can be found in Admin > settings > license page.
 license_token =
 
 ## supervisor connection uri, for managing supervisor and logs.
@@ -283,65 +286,66 b' celery.max_tasks_per_child = 100'
 ## tasks will never be sent to the queue, but executed locally instead.
 celery.task_always_eager = false
 
-####################################
-###         BEAKER CACHE        ####
-####################################
-# default cache dir for templates.  Putting this into a ramdisk
-## can boost performance, eg. %(here)s/data_ramdisk
+#####################################
+###         DOGPILE CACHE        ####
+#####################################
+## Default cache dir for caches.  Putting this into a ramdisk
+## can boost performance, eg. /tmpfs/data_ramdisk, however this directory might require
+## large amount of space
 cache_dir = %(here)s/data
 
-## locking and default file storage for Beaker. Putting this into a ramdisk
-## can boost performance, eg. %(here)s/data_ramdisk/cache/beaker_data
-beaker.cache.data_dir = %(here)s/data/cache/beaker_data
-beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock
-
-beaker.cache.regions = super_short_term, short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long
+## `cache_perms` cache settings for permission tree, auth TTL.
+rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
+rc_cache.cache_perms.expiration_time = 300
 
-beaker.cache.super_short_term.type = memory
-beaker.cache.super_short_term.expire = 10
-beaker.cache.super_short_term.key_length = 256
+## alternative `cache_perms` redis backend with distributed lock
+#rc_cache.cache_perms.backend = dogpile.cache.rc.redis
+#rc_cache.cache_perms.expiration_time = 300
+## redis_expiration_time needs to be greater then expiration_time
+#rc_cache.cache_perms.arguments.redis_expiration_time = 7200
+#rc_cache.cache_perms.arguments.socket_timeout = 30
+#rc_cache.cache_perms.arguments.host = localhost
+#rc_cache.cache_perms.arguments.port = 6379
+#rc_cache.cache_perms.arguments.db = 0
+#rc_cache.cache_perms.arguments.distributed_lock = true
 
-beaker.cache.short_term.type = memory
-beaker.cache.short_term.expire = 60
-beaker.cache.short_term.key_length = 256
-
-beaker.cache.long_term.type = memory
-beaker.cache.long_term.expire = 36000
-beaker.cache.long_term.key_length = 256
+## `cache_repo` cache settings for FileTree, Readme, RSS FEEDS
+rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace
+rc_cache.cache_repo.expiration_time = 2592000
 
-beaker.cache.sql_cache_short.type = memory
-beaker.cache.sql_cache_short.expire = 10
-beaker.cache.sql_cache_short.key_length = 256
+## alternative `cache_repo` redis backend with distributed lock
+#rc_cache.cache_repo.backend = dogpile.cache.rc.redis
+#rc_cache.cache_repo.expiration_time = 2592000
+## redis_expiration_time needs to be greater then expiration_time
+#rc_cache.cache_repo.arguments.redis_expiration_time = 2678400
+#rc_cache.cache_repo.arguments.socket_timeout = 30
+#rc_cache.cache_repo.arguments.host = localhost
+#rc_cache.cache_repo.arguments.port = 6379
+#rc_cache.cache_repo.arguments.db = 1
+#rc_cache.cache_repo.arguments.distributed_lock = true
 
-## default is memory cache, configure only if required
-## using multi-node or multi-worker setup
-#beaker.cache.auth_plugins.type = ext:database
-#beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock
-#beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode
-#beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode
-#beaker.cache.auth_plugins.sa.pool_recycle = 3600
-#beaker.cache.auth_plugins.sa.pool_size = 10
-#beaker.cache.auth_plugins.sa.max_overflow = 0
+## cache settings for SQL queries, this needs to use memory type backend
+rc_cache.sql_cache_short.backend = dogpile.cache.rc.memory_lru
+rc_cache.sql_cache_short.expiration_time = 30
 
-beaker.cache.repo_cache_long.type = memorylru_base
-beaker.cache.repo_cache_long.max_items = 4096
-beaker.cache.repo_cache_long.expire = 2592000
+## `cache_repo_longterm` cache for repo object instances, this needs to use memory
+## type backend as the objects kept are not pickle serializable
+rc_cache.cache_repo_longterm.backend = dogpile.cache.rc.memory_lru
+## by default we use 96H, this is using invalidation on push anyway
+rc_cache.cache_repo_longterm.expiration_time = 345600
+## max items in LRU cache, reduce this number to save memory, and expire last used
+## cached objects
+rc_cache.cache_repo_longterm.max_size = 10000
 
-## default is memorylru_base cache, configure only if required
-## using multi-node or multi-worker setup
-#beaker.cache.repo_cache_long.type = ext:memcached
-#beaker.cache.repo_cache_long.url = localhost:11211
-#beaker.cache.repo_cache_long.expire = 1209600
-#beaker.cache.repo_cache_long.key_length = 256
 
 ####################################
 ###       BEAKER SESSION        ####
 ####################################
 
 ## .session.type is type of storage options for the session, current allowed
-## types are file, ext:memcached, ext:database, and memory (default).
+## types are file, ext:memcached, ext:redis, ext:database, and memory (default).
 beaker.session.type = file
-beaker.session.data_dir = %(here)s/data/sessions/data
+beaker.session.data_dir = %(here)s/data/sessions
 
 ## db based session, fast, and allows easy management over logged in users
 #beaker.session.type = ext:database
@@ -484,6 +488,9 b' set debug = false'
 #sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
 #sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 #sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode
+# pymysql is an alternative driver for MySQL, use in case of problems with default one
+#sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode
+
 sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 
 # see sqlalchemy docs for other advanced settings
@@ -503,6 +510,9 b' sqlalchemy.db1.convert_unicode = true'
 ## which defaults to five.
 #sqlalchemy.db1.max_overflow = 10
 
+## Connection check ping, used to detect broken database connections
+## could be enabled to better handle cases if MySQL has gone away errors
+#sqlalchemy.db1.ping_connection = true
 
 ##################
 ### VCS CONFIG ###
@@ -517,15 +527,17 b' vcs.server.protocol = http'
 
 ## Push/Pull operations protocol, available options are:
 ## `http` - use http-rpc backend (default)
-##
 vcs.scm_app_implementation = http
 
 ## Push/Pull operations hooks protocol, available options are:
 ## `http` - use http-rpc backend (default)
 vcs.hooks.protocol = http
+## Host on which this instance is listening for hooks. If vcsserver is in other location
+## this should be adjusted.
+vcs.hooks.host = 127.0.0.1
 
 vcs.server.log_level = info
-## Start VCSServer with this instance as a subprocess, usefull for development
+## Start VCSServer with this instance as a subprocess, useful for development
 vcs.start_server = false
 
 ## List of enabled VCS backends, available options are:
@@ -608,7 +620,7 b' custom.conf = 1'
 ### LOGGING CONFIGURATION   ####
 ################################
 [loggers]
-keys = root, sqlalchemy, beaker, rhodecode, ssh_wrapper, celery
+keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
 
 [handlers]
 keys = console, console_sql
@@ -664,6 +676,9 b' level = INFO'
 formatter = generic
 
 [handler_console_sql]
+# "level = DEBUG" logs SQL queries and results.
+# "level = INFO" logs SQL queries.
+# "level = WARN" logs neither.  (Recommended for production systems.)
 class = StreamHandler
 args = (sys.stderr, )
 level = WARN
@@ -685,5 +700,5 b' datefmt = %Y-%m-%d %H:%M:%S'
 
 [formatter_color_formatter_sql]
 class = rhodecode.lib.logging_formatter.ColorFormatterSql
-format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
+format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S

default.nix

@@ -1,41 +1,40 b''
 # Nix environment for the community edition
 #
-# This shall be as lean as possible, just producing the Enterprise
+# This shall be as lean as possible, just producing the enterprise-ce
 # derivation. For advanced tweaks to pimp up the development environment we use
 # "shell.nix" so that it does not have to clutter this file.
+#
+# Configuration, set values in "~/.nixpkgs/config.nix".
+# example
+#  {
+#    # Thoughts on how to configure the dev environment
+#    rc = {
+#      codeInternalUrl = "https://usr:[e-mail]";
+#      sources = {
+#        rhodecode-vcsserver = "/home/user/work/rhodecode-vcsserver";
+#        rhodecode-enterprise-ce = "/home/user/work/rhodecode-enterprise-ce";
+#        rhodecode-enterprise-ee = "/home/user/work/rhodecode-enterprise-ee";
+#      };
+#    };
+#  }
 
 args@
 { pythonPackages ? "python27Packages"
 , pythonExternalOverrides ? self: super: {}
-, doCheck ? true
+, doCheck ? false
 , ...
 }:
 
 let
-
   # Use nixpkgs from args or import them. We use this indirect approach
   # through args to be able to use the name `pkgs` for our customized packages.
   # Otherwise we will end up with an infinite recursion.
-  nixpkgs = args.pkgs or (import <nixpkgs> { });
+  pkgs = args.pkgs or (import <nixpkgs> { });
 
-  # johbo: Interim bridge which allows us to build with the upcoming
-  # nixos.16.09 branch (unstable at the moment of writing this note) and the
-  # current stable nixos-16.03.
-  backwardsCompatibleFetchgit = { ... }@args:
-    let
-      origSources = nixpkgs.fetchgit args;
-    in
-    nixpkgs.lib.overrideDerivation origSources (oldAttrs: {
-      NIX_PREFETCH_GIT_CHECKOUT_HOOK = ''
-        find $out -name '.git*' -print0 | xargs -0 rm -rf
-      '';
-    });
-
-  # Create a customized version of nixpkgs which should be used throughout the
-  # rest of this file.
-  pkgs = nixpkgs.overridePackages (self: super: {
-    fetchgit = backwardsCompatibleFetchgit;
-  });
+  # Works with the new python-packages, still can fallback to the old
+  # variant.
+  basePythonPackagesUnfix = basePythonPackages.__unfix__ or (
+    self: basePythonPackages.override (a: { inherit self; }));
 
   # Evaluates to the last segment of a file system path.
   basename = path: with pkgs.lib; last (splitString "/" path);
@@ -46,7 +45,7 b' let'
       ext = last (splitString "." path);
     in
       !builtins.elem (basename path) [
-        ".git" ".hg" "__pycache__" ".eggs"
+        ".git" ".hg" "__pycache__" ".eggs" ".idea" ".dev"
         "bower_components" "node_modules"
         "build" "data" "result" "tmp"] &&
       !builtins.elem ext ["egg-info" "pyc"] &&
@@ -54,18 +53,20 b' let'
       # it would still be good to restore it since we want to ignore "result-*".
       !hasPrefix "result" path;
 
-  basePythonPackages = with builtins; if isAttrs pythonPackages
-    then pythonPackages
-    else getAttr pythonPackages pkgs;
+  sources =
+    let
+      inherit (pkgs.lib) all isString attrValues;
+      sourcesConfig = pkgs.config.rc.sources or {};
+    in
+      # Ensure that sources are configured as strings. Using a path
+      # would result in a copy into the nix store.
+      assert all isString (attrValues sourcesConfig);
+      sourcesConfig;
 
-  buildBowerComponents =
-    pkgs.buildBowerComponents or
-    (import ./pkgs/backport-16.03-build-bower-components.nix { inherit pkgs; });
-
-  sources = pkgs.config.rc.sources or {};
-  version = builtins.readFile ./rhodecode/VERSION;
+  version = builtins.readFile "${rhodecode-enterprise-ce-src}/rhodecode/VERSION";
   rhodecode-enterprise-ce-src = builtins.filterSource src-filter ./.;
 
+  buildBowerComponents = pkgs.buildBowerComponents;
   nodeEnv = import ./pkgs/node-default.nix {
     inherit pkgs;
   };
@@ -77,133 +78,145 b' let'
     src = rhodecode-enterprise-ce-src;
   };
 
-  pythonGeneratedPackages = self: basePythonPackages.override (a: {
-    inherit self;
-  })
-  // (scopedImport {
-    self = self;
-    super = basePythonPackages;
-    inherit pkgs;
-    inherit (pkgs) fetchurl fetchgit;
-  } ./pkgs/python-packages.nix);
+  rhodecode-testdata-src = sources.rhodecode-testdata or (
+    pkgs.fetchhg {
+      url = "https://code.rhodecode.com/upstream/rc_testdata";
+      rev = "v0.10.0";
+      sha256 = "0zn9swwvx4vgw4qn8q3ri26vvzgrxn15x6xnjrysi1bwmz01qjl0";
+  });
 
-  pythonOverrides = import ./pkgs/python-packages-overrides.nix {
-    inherit
-      basePythonPackages
-      pkgs;
+  rhodecode-testdata = import "${rhodecode-testdata-src}/default.nix" {
+  inherit
+    doCheck
+    pkgs
+    pythonPackages;
   };
 
   pythonLocalOverrides = self: super: {
     rhodecode-enterprise-ce =
       let
         linkNodeAndBowerPackages = ''
-          echo "Export RhodeCode CE path"
           export RHODECODE_CE_PATH=${rhodecode-enterprise-ce-src}
-          echo "Link node packages"
+
+          echo "[BEGIN]: Link node packages"
           rm -fr node_modules
           mkdir node_modules
           # johbo: Linking individual packages allows us to run "npm install"
           # inside of a shell to try things out. Re-entering the shell will
           # restore a clean environment.
           ln -s ${nodeDependencies}/lib/node_modules/* node_modules/
+          echo "[DONE]: Link node packages"
 
-          echo "DONE: Link node packages"
-
-          echo "Link bower packages"
+          echo "[BEGIN]: Link bower packages"
           rm -fr bower_components
           mkdir bower_components
+          ln -s ${bowerComponents}/bower_components/* bower_components/
+          echo "[DONE]: Link bower packages"
+        '';
 
-          ln -s ${bowerComponents}/bower_components/* bower_components/
-          echo "DONE: Link bower packages"
-        '';
+        releaseName = "RhodeCodeEnterpriseCE-${version}";
       in super.rhodecode-enterprise-ce.override (attrs: {
-
       inherit
         doCheck
         version;
+
       name = "rhodecode-enterprise-ce-${version}";
-      releaseName = "RhodeCodeEnterpriseCE-${version}";
+      releaseName = releaseName;
       src = rhodecode-enterprise-ce-src;
       dontStrip = true; # prevent strip, we don't need it.
 
-      buildInputs =
-        attrs.buildInputs ++
-        (with self; [
-          pkgs.nodePackages.bower
-          pkgs.nodePackages.grunt-cli
-          pkgs.subversion
-          pytest-catchlog
-          rhodecode-testdata
-        ]);
-
-      #TODO: either move this into overrides, OR use the new machanics from
-      # pip2nix and requiremtn.txt file
-      propagatedBuildInputs = attrs.propagatedBuildInputs ++ (with self; [
-        rhodecode-tools
-      ]);
-
-      # TODO: johbo: Make a nicer way to expose the parts. Maybe
-      # pkgs/default.nix?
+      # expose following attributed outside
       passthru = {
         inherit
+          rhodecode-testdata
           bowerComponents
           linkNodeAndBowerPackages
           myPythonPackagesUnfix
-          pythonLocalOverrides;
+          pythonLocalOverrides
+          pythonCommunityOverrides;
+
         pythonPackages = self;
       };
 
+      buildInputs =
+        attrs.buildInputs or [] ++ [
+          rhodecode-testdata
+          pkgs.nodePackages.bower
+          pkgs.nodePackages.grunt-cli
+        ];
+
+      #NOTE: option to inject additional propagatedBuildInputs
+      propagatedBuildInputs =
+        attrs.propagatedBuildInputs or [] ++ [
+
+        ];
+
       LC_ALL = "en_US.UTF-8";
       LOCALE_ARCHIVE =
-        if pkgs.stdenv ? glibc
+        if pkgs.stdenv.isLinux
         then "${pkgs.glibcLocales}/lib/locale/locale-archive"
         else "";
 
+      # Add bin directory to path so that tests can find 'rhodecode'.
       preCheck = ''
         export PATH="$out/bin:$PATH"
       '';
 
+      # custom check phase for testing
+      checkPhase = ''
+        runHook preCheck
+        PYTHONHASHSEED=random py.test -vv -p no:sugar -r xw --cov-config=.coveragerc --cov=rhodecode --cov-report=term-missing rhodecode
+        runHook postCheck
+      '';
+
       postCheck = ''
-        rm -rf $out/lib/${self.python.libPrefix}/site-packages/pytest_pylons
+        echo "Cleanup of rhodecode/tests"
         rm -rf $out/lib/${self.python.libPrefix}/site-packages/rhodecode/tests
       '';
 
-      preBuild = linkNodeAndBowerPackages + ''
+      preBuild = ''
+
+        echo "Building frontend assets"
+        ${linkNodeAndBowerPackages}
         grunt
         rm -fr node_modules
       '';
 
       postInstall = ''
-        echo "Writing meta information for rccontrol to nix-support/rccontrol"
+        echo "Writing enterprise-ce meta information for rccontrol to nix-support/rccontrol"
         mkdir -p $out/nix-support/rccontrol
         cp -v rhodecode/VERSION $out/nix-support/rccontrol/version
-        echo "DONE: Meta information for rccontrol written"
+        echo "[DONE]: enterprise-ce meta information for rccontrol written"
+
+        mkdir -p $out/etc
+        cp configs/production.ini $out/etc
+        echo "[DONE]: saved enterprise-ce production.ini into $out/etc"
 
         # python based programs need to be wrapped
+        mkdir -p $out/bin
+        # rhodecode-tools
+        ln -s ${self.rhodecode-tools}/bin/rhodecode-* $out/bin/
+
+        # required binaries from dependencies
+        #ln -s ${self.python}/bin/python $out/bin
         ln -s ${self.pyramid}/bin/* $out/bin/
         ln -s ${self.gunicorn}/bin/gunicorn $out/bin/
         ln -s ${self.supervisor}/bin/supervisor* $out/bin/
-        ln -s ${self.PasteScript}/bin/paster $out/bin/
+        ln -s ${self.pastescript}/bin/paster $out/bin/
         ln -s ${self.channelstream}/bin/channelstream $out/bin/
         ln -s ${self.celery}/bin/celery $out/bin/
+        echo "[DONE]: created symlinks into $out/bin"
 
-        # rhodecode-tools
-        ln -s ${self.rhodecode-tools}/bin/rhodecode-* $out/bin/
-
-        # note that condition should be restricted when adding further tools
         for file in $out/bin/*;
         do
           wrapProgram $file \
-              --prefix PATH : $PATH \
-              --prefix PYTHONPATH : $PYTHONPATH \
-              --set PYTHONHASHSEED random
+            --prefix PATH : $PATH \
+            --prefix PYTHONPATH : $PYTHONPATH \
+            --set PYTHONHASHSEED random
         done
 
-        mkdir $out/etc
-        cp configs/production.ini $out/etc
+        echo "[DONE]: enterprise-ce binary wrapping"
 
-
-        # TODO: johbo: Make part of ac-tests
         if [ ! -f rhodecode/public/js/scripts.js ]; then
           echo "Missing scripts.js"
           exit 1
@@ -213,31 +226,33 b' let'
           exit 1
         fi
       '';
-
     });
 
-    rhodecode-testdata = import "${rhodecode-testdata-src}/default.nix" {
-    inherit
-      doCheck
-      pkgs
-      pythonPackages;
-    };
-
   };
 
-  rhodecode-testdata-src = sources.rhodecode-testdata or (
-    pkgs.fetchhg {
-      url = "https://code.rhodecode.com/upstream/rc_testdata";
-      rev = "v0.10.0";
-      sha256 = "0zn9swwvx4vgw4qn8q3ri26vvzgrxn15x6xnjrysi1bwmz01qjl0";
-  });
+  basePythonPackages = with builtins;
+    if isAttrs pythonPackages then
+      pythonPackages
+    else
+      getAttr pythonPackages pkgs;
+
+  pythonGeneratedPackages = import ./pkgs/python-packages.nix {
+    inherit pkgs;
+    inherit (pkgs) fetchurl fetchgit fetchhg;
+  };
+
+  pythonCommunityOverrides = import ./pkgs/python-packages-overrides.nix {
+    inherit pkgs basePythonPackages;
+  };
 
   # Apply all overrides and fix the final package set
   myPythonPackagesUnfix = with pkgs.lib;
     (extends pythonExternalOverrides
     (extends pythonLocalOverrides
-    (extends pythonOverrides
-             pythonGeneratedPackages)));
+    (extends pythonCommunityOverrides
+    (extends pythonGeneratedPackages
+             basePythonPackagesUnfix))));
+
   myPythonPackages = (pkgs.lib.fix myPythonPackagesUnfix);
 
 in myPythonPackages.rhodecode-enterprise-ce

docs/.howto

@@ -1,4 +1,9 b''
+# generating packages
+nix-shell pkgs/shell-generate.nix
+cd docs
+pip2nix generate
+
 # building the docs
 cd docs
 nix-build default.nix -o result
-make clean html
  No newline at end of file
+make clean html

docs/admin/indexing.rst

@@ -88,17 +88,21 b' want to index:'
    (venv)$ rhodecode-index --instance-name=enterprise-1
 
    # Using default installation
-   $ /home/user/.rccontrol/enterprise-4/profile/bin/rhodecode-index \
-       --instance-name=enterprise-4
+   $ /home/user/.rccontrol/enterprise-1/profile/bin/rhodecode-index \
+       --instance-name=enterprise-1
 
    # Using a custom mapping file
-   $ /home/user/.rccontrol/enterprise-4/profile/bin/rhodecode-index \
-       --instance-name=enterprise-4 \
-       --mapping=/home/user/.rccontrol/enterprise-4/mapping.ini
+   $ /home/user/.rccontrol/enterprise-1/profile/bin/rhodecode-index \
+       --instance-name=enterprise-1 \
+       --mapping=/home/user/.rccontrol/enterprise-1/mapping.ini
 
 .. note::
 
-   |RCT| require |PY| 2.7 to run.
+   In case of often indexing the index may become fragmented. Most often a result of that
+   is error about `too many open files`. To fix this indexer needs to be executed with
+   --optimize flag. E.g `rhodecode-index --instance-name=enterprise-1 --optimize`
+   This should be executed regularly, once a week is recommended.
+
 
 .. _set-index:
 

docs/admin/tuning-user-sessions-performance.rst

@@ -12,7 +12,7 b' if there are lots of session files.'
 
 Therefore, in a large scale deployment, to give better performance,
 scalability, and maintainability we recommend switching from file-based
-sessions to database-based user sessions or memcached sessions.
+sessions to database-based user sessions or Redis based sessions.
 
 To switch to database-based user sessions uncomment the following section in
 your :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file.
@@ -41,14 +41,20 b' and make sure you comment out the file b'
       #beaker.session.data_dir = %(here)s/data/sessions/data
 
 
-To switch to memcached-based user sessions uncomment the following section in
+The `table_name` will be automatically created on specified database if it isn't yet existing.
+Database specified in the `beaker.session.sa.url` can be the same that RhodeCode
+uses, or if required it can be a different one. We recommend to use the same database.
+
+
+
+To switch to reds-based user sessions uncomment the following section in
 your :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file.
 
 .. code-block:: ini
 
-      ## memcached sessions
-      beaker.session.type = ext:memcached
-      beaker.session.url = localhost:11211
+      ## redis sessions
+      beaker.session.type = ext:redis
+      beaker.session.url = localhost:6379
 
 
 and make sure you comment out the file based sessions.

docs/api/methods/pull-request-methods.rst

@@ -83,7 +83,7 b' comment_pull_request'
 create_pull_request 
 -------------------
 
-.. py:function:: create_pull_request(apiuser, source_repo, target_repo, source_ref, target_ref, title, description=<Optional:''>, reviewers=<Optional:None>)
+.. py:function:: create_pull_request(apiuser, source_repo, target_repo, source_ref, target_ref, title=<Optional:''>, description=<Optional:''>, description_renderer=<Optional:''>, reviewers=<Optional:None>)
 
    Creates a new pull request.
 
@@ -104,10 +104,14 b' create_pull_request'
    :type source_ref: str
    :param target_ref: Set the target ref name.
    :type target_ref: str
-   :param title: Set the pull request title.
+   :param title: Optionally Set the pull request title, it's generated otherwise
    :type title: str
    :param description: Set the pull request description.
    :type description: Optional(str)
+   :type description_renderer: Optional(str)
+   :param description_renderer: Set pull request renderer for the description.
+       It should be 'rst', 'markdown' or 'plain'. If not give default
+       system renderer will be used
    :param reviewers: Set the new pull request reviewers list.
        Reviewer defined by review rules will be added automatically to the
        defined list.
@@ -368,7 +372,7 b' merge_pull_request'
 update_pull_request 
 -------------------
 
-.. py:function:: update_pull_request(apiuser, pullrequestid, repoid=<Optional:None>, title=<Optional:''>, description=<Optional:''>, reviewers=<Optional:None>, update_commits=<Optional:None>)
+.. py:function:: update_pull_request(apiuser, pullrequestid, repoid=<Optional:None>, title=<Optional:''>, description=<Optional:''>, description_renderer=<Optional:''>, reviewers=<Optional:None>, update_commits=<Optional:None>)
 
    Updates a pull request.
 
@@ -382,6 +386,9 b' update_pull_request'
    :type title: str
    :param description: Update pull request description.
    :type description: Optional(str)
+   :type description_renderer: Optional(str)
+   :param description_renderer: Update pull request renderer for the description.
+       It should be 'rst', 'markdown' or 'plain'
    :param reviewers: Update pull request reviewers list with new value.
    :type reviewers: Optional(list)
        Accepts username strings or objects of the format:

docs/conf.py

@@ -32,6 +32,7 b' import common'
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
+    'sphinx.ext.autodoc',
     'sphinx.ext.intersphinx',
     'sphinx.ext.todo',
     'sphinx.ext.imgmath'
@@ -42,13 +43,6 b' intersphinx_mapping = {'
     'control': ('https://docs.rhodecode.com/RhodeCode-Control/', None),
 }
 
-if tags.has('dev'):
-    intersphinx_mapping.update({
-        'enterprise': ('https://ci.rhodecode.com/documentation/Momentum/', None),
-        'control': ('https://ci.rhodecode.com/documentation/Control/', None),
-    })
-
-
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
@@ -65,11 +59,12 b" master_doc = 'index'"
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 
-# TODO: johbo: Move into common package for documentation utilities
+
 def _get_version():
     with open('../rhodecode/VERSION') as f:
         return f.read().strip()
 
+
 # The full version, including alpha/beta/rc tags.
 release = _get_version()
 # The short X.Y version.

docs/contributing/dev-setup.rst

@@ -47,7 +47,7 b' Switch nix to the latest STABLE channel'
 
 run::
 
-   nix-channel --add https://nixos.org/channels/nixos-16.03 nixpkgs
+   nix-channel --add https://nixos.org/channels/nixos-18.03 nixpkgs
 
 Followed by::
 

docs/default.nix

@@ -5,249 +5,215 b' let'
 
   pkgs = import <nixpkgs> { inherit system; };
 
-  inherit (pkgs) fetchurl fetchgit;
+  inherit (pkgs) fetchurl;
 
   buildPythonPackage = pkgs.python27Packages.buildPythonPackage;
   python = pkgs.python27Packages.python;
 
-  Jinja2 = buildPythonPackage rec {
-    name = "Jinja2-2.9.6";
-    buildInputs = [];
+
+  alabaster = buildPythonPackage {
+    name = "alabaster-0.7.11";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/3f/46/9346ea429931d80244ab7f11c4fce83671df0b7ae5a60247a2b588592c46/alabaster-0.7.11.tar.gz";
+      sha256 = "1mvm69xsn5xf1jc45kdq1mn0yq0pfn54mv2jcww4s1vwqx6iyfxn";
+    };
+  };
+  babel = buildPythonPackage {
+    name = "babel-2.6.0";
     doCheck = false;
-    propagatedBuildInputs = [MarkupSafe];
+    propagatedBuildInputs = [
+      pytz
+    ];
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/be/cc/9c981b249a455fa0c76338966325fc70b7265521bad641bf2932f77712f4/Babel-2.6.0.tar.gz";
+      sha256 = "08rxmbx2s4irp0w0gmn498vns5xy0fagm0fg33xa772jiks51flc";
+    };
+  };
+  certifi = buildPythonPackage {
+    name = "certifi-2018.8.24";
+    doCheck = false;
     src = fetchurl {
-      url = "https://pypi.python.org/packages/90/61/f820ff0076a2599dd39406dcb858ecb239438c02ce706c8e91131ab9c7f1/Jinja2-2.9.6.tar.gz";
-      md5 = "6411537324b4dba0956aaa8109f3c77b";
+      url = "https://files.pythonhosted.org/packages/e1/0f/f8d5e939184547b3bdc6128551b831a62832713aa98c2ccdf8c47ecc7f17/certifi-2018.8.24.tar.gz";
+      sha256 = "0f0nhrj9mlrf79iway4578wrsgmjh0fmacl9zv8zjckdy7b90rip";
+    };
+  };
+  chardet = buildPythonPackage {
+    name = "chardet-3.0.4";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/fc/bb/a5768c230f9ddb03acc9ef3f0d4a3cf93462473795d18e9535498c8f929d/chardet-3.0.4.tar.gz";
+      sha256 = "1bpalpia6r5x1kknbk11p1fzph56fmmnp405ds8icksd3knr5aw4";
+    };
+  };
+  docutils = buildPythonPackage {
+    name = "docutils-0.14";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/84/f4/5771e41fdf52aabebbadecc9381d11dea0fa34e4759b4071244fa094804c/docutils-0.14.tar.gz";
+      sha256 = "0x22fs3pdmr42kvz6c654756wja305qv6cx1zbhwlagvxgr4xrji";
     };
   };
-
-  MarkupSafe = buildPythonPackage rec {
-    name = "MarkupSafe-1.0";
-    buildInputs = [];
+  idna = buildPythonPackage {
+    name = "idna-2.7";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/65/c4/80f97e9c9628f3cac9b98bfca0402ede54e0563b56482e3e6e45c43c4935/idna-2.7.tar.gz";
+      sha256 = "05jam7d31767dr12x0rbvvs8lxnpb1mhdb2zdlfxgh83z6k3hjk8";
+    };
+  };
+  imagesize = buildPythonPackage {
+    name = "imagesize-1.1.0";
     doCheck = false;
-    propagatedBuildInputs = [];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/4d/de/32d741db316d8fdb7680822dd37001ef7a448255de9699ab4bfcbdf4172b/MarkupSafe-1.0.tar.gz";
-      md5 = "2fcedc9284d50e577b5192e8e3578355";
+      url = "https://files.pythonhosted.org/packages/41/f5/3cf63735d54aa9974e544aa25858d8f9670ac5b4da51020bbfc6aaade741/imagesize-1.1.0.tar.gz";
+      sha256 = "1dg3wn7qpwmhgqc0r9na2ding1wif9q5spz3j9zn2riwphc2k0zk";
+    };
+  };
+  jinja2 = buildPythonPackage {
+    name = "jinja2-2.9.6";
+    doCheck = false;
+    propagatedBuildInputs = [
+      markupsafe
+    ];
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/90/61/f820ff0076a2599dd39406dcb858ecb239438c02ce706c8e91131ab9c7f1/Jinja2-2.9.6.tar.gz";
+      sha256 = "1zzrkywhziqffrzks14kzixz7nd4yh2vc0fb04a68vfd2ai03anx";
     };
   };
-
-  Pygments = buildPythonPackage {
-    name = "Pygments-2.2.0";
-    buildInputs = [];
+  markupsafe = buildPythonPackage {
+    name = "markupsafe-1.0";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/4d/de/32d741db316d8fdb7680822dd37001ef7a448255de9699ab4bfcbdf4172b/MarkupSafe-1.0.tar.gz";
+      sha256 = "0rdn1s8x9ni7ss8rfiacj7x1085lx8mh2zdwqslnw8xc3l4nkgm6";
+    };
+  };
+  packaging = buildPythonPackage {
+    name = "packaging-17.1";
     doCheck = false;
-    propagatedBuildInputs = [];
+    propagatedBuildInputs = [
+      pyparsing
+      six
+    ];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/71/2a/2e4e77803a8bd6408a2903340ac498cb0a2181811af7c9ec92cb70b0308a/Pygments-2.2.0.tar.gz";
-      md5 = "13037baca42f16917cbd5ad2fab50844";
+      url = "https://files.pythonhosted.org/packages/77/32/439f47be99809c12ef2da8b60a2c47987786d2c6c9205549dd6ef95df8bd/packaging-17.1.tar.gz";
+      sha256 = "0nrpayk8kij1zm9sjnk38ldz3a6705ggvw8ljylqbrb4vmqbf6gh";
+    };
+  };
+  pygments = buildPythonPackage {
+    name = "pygments-2.2.0";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/71/2a/2e4e77803a8bd6408a2903340ac498cb0a2181811af7c9ec92cb70b0308a/Pygments-2.2.0.tar.gz";
+      sha256 = "1k78qdvir1yb1c634nkv6rbga8wv4289xarghmsbbvzhvr311bnv";
     };
   };
-
-  Sphinx = buildPythonPackage (rec {
-    name = "Sphinx-1.6.5";
+  pyparsing = buildPythonPackage {
+    name = "pyparsing-2.2.0";
+    doCheck = false;
     src = fetchurl {
-      url = "https://pypi.python.org/packages/8b/7e/b188d9a3b9c938e736e02a74c1363c2888e095d770df2c72b4c312f9fdcb/Sphinx-1.6.5.tar.gz";
-      md5 = "cd73118c21ec610432e63e6421ec54f1";
+      url = "https://files.pythonhosted.org/packages/3c/ec/a94f8cf7274ea60b5413df054f82a8980523efd712ec55a59e7c3357cf7c/pyparsing-2.2.0.tar.gz";
+      sha256 = "016b9gh606aa44sq92jslm89bg874ia0yyiyb643fa6dgbsbqch8";
     };
+  };
+  pytz = buildPythonPackage {
+    name = "pytz-2018.4";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/10/76/52efda4ef98e7544321fd8d5d512e11739c1df18b0649551aeccfb1c8376/pytz-2018.4.tar.gz";
+      sha256 = "0jgpqx3kk2rhv81j1izjxvmx8d0x7hzs1857pgqnixic5wq2ar60";
+    };
+  };
+  requests = buildPythonPackage {
+    name = "requests-2.19.1";
+    doCheck = false;
     propagatedBuildInputs = [
-        six
-        Jinja2
-        Pygments
-        docutils
-        snowballstemmer
-        babel
-        alabaster
-        imagesize
-        requests
-        setuptools
-        sphinxcontrib-websupport
-        typing
-
-        # special cases
-        pytz
-        sphinx_rtd_theme
-
+      chardet
+      idna
+      urllib3
+      certifi
     ];
-  });
-
-  alabaster = buildPythonPackage rec {
-    name = "alabaster-0.7.10";
-    buildInputs = [];
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/54/1f/782a5734931ddf2e1494e4cd615a51ff98e1879cbe9eecbdfeaf09aa75e9/requests-2.19.1.tar.gz";
+      sha256 = "0snf8xxdzsgh1x2zv3vilvbrv9jbpmnfagzzb1rjmmvflckdh8pc";
+    };
+  };
+  six = buildPythonPackage {
+    name = "six-1.11.0";
     doCheck = false;
-    propagatedBuildInputs = [];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/d0/a5/e3a9ad3ee86aceeff71908ae562580643b955ea1b1d4f08ed6f7e8396bd7/alabaster-0.7.10.tar.gz";
-      md5 = "7934dccf38801faa105f6e7b4784f493";
+      url = "https://files.pythonhosted.org/packages/16/d8/bc6316cf98419719bd59c91742194c111b6f2e85abac88e496adefaf7afe/six-1.11.0.tar.gz";
+      sha256 = "1scqzwc51c875z23phj48gircqjgnn3af8zy2izjwmnlxrxsgs3h";
+    };
+  };
+  snowballstemmer = buildPythonPackage {
+    name = "snowballstemmer-1.2.1";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/20/6b/d2a7cb176d4d664d94a6debf52cd8dbae1f7203c8e42426daa077051d59c/snowballstemmer-1.2.1.tar.gz";
+      sha256 = "0a0idq4y5frv7qsg2x62jd7rd272749xk4x99misf5rcifk2d7wi";
     };
   };
-
-  babel = buildPythonPackage {
-    name = "babel-2.5.1";
-    buildInputs = [];
+  sphinx = buildPythonPackage {
+    name = "sphinx-1.7.8";
     doCheck = false;
-    propagatedBuildInputs = [pytz];
+    propagatedBuildInputs = [
+      six
+      jinja2
+      pygments
+      docutils
+      snowballstemmer
+      babel
+      alabaster
+      imagesize
+      requests
+      setuptools
+      packaging
+      sphinxcontrib-websupport
+      typing
+    ];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/5a/22/63f1dbb8514bb7e0d0c8a85cc9b14506599a075e231985f98afd70430e1f/Babel-2.5.1.tar.gz";
-      md5 = "60228b3ce93a203357158b909afe8ae1";
+      url = "https://files.pythonhosted.org/packages/ac/54/4ef326d0c654da1ed91341a7a1f43efc18a8c770ddd2b8e45df97cb79d82/Sphinx-1.7.8.tar.gz";
+      sha256 = "1ryz0w4c31930f1br2sjwrxwx9cmsy7cqdb0d81g98n9bj250w50";
     };
   };
-
-  certifi = buildPythonPackage {
-    name = "certifi-2017.11.5";
-    buildInputs = [];
+  sphinx-rtd-theme = buildPythonPackage {
+    name = "sphinx-rtd-theme-0.4.1";
     doCheck = false;
-    propagatedBuildInputs = [];
+    propagatedBuildInputs = [
+      sphinx
+    ];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/23/3f/8be01c50ed24a4bd6b8da799839066ce0288f66f5e11f0367323467f0cbc/certifi-2017.11.5.tar.gz";
-      md5 = "c15ac46ed1fe4b607ff3405928f9a992";
+      url = "https://files.pythonhosted.org/packages/f2/b0/a1933d792b806118ddbca6699f2e2c844d9b1b16e84a89d7effd5cd2a800/sphinx_rtd_theme-0.4.1.tar.gz";
+      sha256 = "1xkyqam8dzbjaymdyvkiif85m4y3jf8crdiwlgcfp8gqcj57aj9v";
     };
   };
-
-  chardet = buildPythonPackage {
-    name = "chardet-3.0.4";
-    buildInputs = [];
+  sphinxcontrib-websupport = buildPythonPackage {
+    name = "sphinxcontrib-websupport-1.1.0";
     doCheck = false;
-    propagatedBuildInputs = [];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/fc/bb/a5768c230f9ddb03acc9ef3f0d4a3cf93462473795d18e9535498c8f929d/chardet-3.0.4.tar.gz";
-      md5 = "7dd1ba7f9c77e32351b0a0cfacf4055c";
+      url = "https://files.pythonhosted.org/packages/07/7a/e74b06dce85555ffee33e1d6b7381314169ebf7e31b62c18fcb2815626b7/sphinxcontrib-websupport-1.1.0.tar.gz";
+      sha256 = "1ff3ix76xi1y6m99qxhaq5161ix9swwzydilvdya07mgbcvpzr4x";
     };
   };
-
-  docutils = buildPythonPackage {
-    name = "docutils-0.14";
-    buildInputs = [];
+  typing = buildPythonPackage {
+    name = "typing-3.6.6";
     doCheck = false;
-    propagatedBuildInputs = [];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/84/f4/5771e41fdf52aabebbadecc9381d11dea0fa34e4759b4071244fa094804c/docutils-0.14.tar.gz";
-      md5 = "c53768d63db3873b7d452833553469de";
+      url = "https://files.pythonhosted.org/packages/bf/9b/2bf84e841575b633d8d91ad923e198a415e3901f228715524689495b4317/typing-3.6.6.tar.gz";
+      sha256 = "0ba9acs4awx15bf9v3nrs781msbd2nx826906nj6fqks2bvca9s0";
+    };
+  };
+  urllib3 = buildPythonPackage {
+    name = "urllib3-1.23";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/3c/d2/dc5471622bd200db1cd9319e02e71bc655e9ea27b8e0ce65fc69de0dac15/urllib3-1.23.tar.gz";
+      sha256 = "1bvbd35q3zdcd7gsv38fwpizy7p06dr0154g5gfybrvnbvhwb2m6";
     };
   };
 
-  idna = buildPythonPackage {
-    name = "idna-2.6";
-    buildInputs = [];
-    doCheck = false;
-    propagatedBuildInputs = [];
-    src = fetchurl {
-      url = "https://pypi.python.org/packages/f4/bd/0467d62790828c23c47fc1dfa1b1f052b24efdf5290f071c7a91d0d82fd3/idna-2.6.tar.gz";
-      md5 = "c706e2790b016bd0ed4edd2d4ba4d147";
-    };
-  };
-
-  imagesize = buildPythonPackage {
-    name = "imagesize-0.7.1";
-    buildInputs = [];
-    doCheck = false;
-    propagatedBuildInputs = [];
-    src = fetchurl {
-      url = "https://pypi.python.org/packages/53/72/6c6f1e787d9cab2cc733cf042f125abec07209a58308831c9f292504e826/imagesize-0.7.1.tar.gz";
-      md5 = "976148283286a6ba5f69b0f81aef8052";
-    };
-  };
-
-  pytz = buildPythonPackage {
-    name = "pytz-2017.3";
-    buildInputs = [];
-    doCheck = false;
-    propagatedBuildInputs = [];
-    src = fetchurl {
-      url = "https://pypi.python.org/packages/60/88/d3152c234da4b2a1f7a989f89609ea488225eaea015bc16fbde2b3fdfefa/pytz-2017.3.zip";
-      md5 = "7006b56c0d68a162d9fe57d4249c3171";
-    };
-  };
-
-  requests = buildPythonPackage {
-    name = "requests-2.18.4";
-    buildInputs = [];
-    doCheck = false;
-    propagatedBuildInputs = [chardet idna urllib3 certifi];
-    src = fetchurl {
-      url = "https://pypi.python.org/packages/b0/e1/eab4fc3752e3d240468a8c0b284607899d2fbfb236a56b7377a329aa8d09/requests-2.18.4.tar.gz";
-      md5 = "081412b2ef79bdc48229891af13f4d82";
-    };
-  };
-
-  six = buildPythonPackage {
-    name = "six-1.11.0";
-    buildInputs = [];
-    doCheck = false;
-    propagatedBuildInputs = [];
-    src = fetchurl {
-      url = "https://pypi.python.org/packages/16/d8/bc6316cf98419719bd59c91742194c111b6f2e85abac88e496adefaf7afe/six-1.11.0.tar.gz";
-      md5 = "d12789f9baf7e9fb2524c0c64f1773f8";
-    };
-  };
-
-  snowballstemmer = buildPythonPackage {
-    name = "snowballstemmer-1.2.1";
-    buildInputs = [];
-    doCheck = false;
-    propagatedBuildInputs = [];
-    src = fetchurl {
-      url = "https://pypi.python.org/packages/20/6b/d2a7cb176d4d664d94a6debf52cd8dbae1f7203c8e42426daa077051d59c/snowballstemmer-1.2.1.tar.gz";
-      md5 = "643b019667a708a922172e33a99bf2fa";
-    };
-  };
-
-  sphinx-rtd-theme = buildPythonPackage {
-    name = "sphinx-rtd-theme-0.2.5b1";
-    buildInputs = [];
-    doCheck = false;
-    propagatedBuildInputs = [];
-    src = fetchurl {
-      url = "https://pypi.python.org/packages/59/e4/9e3a74a3271e6734911d3f549e8439db53b8ac29adf10c8f698e6c86246b/sphinx_rtd_theme-0.2.5b1.tar.gz";
-      md5 = "0923473a43bd2527f32151f195f2a521";
-    };
-  };
-
-  sphinxcontrib-websupport = buildPythonPackage {
-    name = "sphinxcontrib-websupport-1.0.1";
-    buildInputs = [];
-    doCheck = false;
-    propagatedBuildInputs = [];
-    src = fetchurl {
-      url = "https://pypi.python.org/packages/c5/6b/f0630436b931ad4f8331a9399ca18a7d447f0fcc0c7178fb56b1aee68d01/sphinxcontrib-websupport-1.0.1.tar.gz";
-      md5 = "84df26463b1ba65b07f926dbe2055665";
-    };
-  };
-
-  typing = buildPythonPackage {
-    name = "typing-3.6.2";
-    buildInputs = [];
-    doCheck = false;
-    propagatedBuildInputs = [];
-    src = fetchurl {
-      url = "https://pypi.python.org/packages/ca/38/16ba8d542e609997fdcd0214628421c971f8c395084085354b11ff4ac9c3/typing-3.6.2.tar.gz";
-      md5 = "143af0bf3afd1887622771f2f1ffe8e1";
-    };
-  };
-
-  urllib3 = buildPythonPackage {
-    name = "urllib3-1.22";
-    buildInputs = [];
-    doCheck = false;
-    propagatedBuildInputs = [];
-    src = fetchurl {
-      url = "https://pypi.python.org/packages/ee/11/7c59620aceedcc1ef65e156cc5ce5a24ef87be4107c2b74458464e437a5d/urllib3-1.22.tar.gz";
-      md5 = "0da7bed3fe94bf7dc59ae37885cc72f7";
-    };
-  };
-
-
-  sphinx_rtd_theme = buildPythonPackage rec {
-    name = "sphinx-rtd-theme-0.2.5b1";
-    buildInputs = [];
-    doCheck = false;
-    propagatedBuildInputs = [];
-    src = fetchurl {
-      url = "https://pypi.python.org/packages/59/e4/9e3a74a3271e6734911d3f549e8439db53b8ac29adf10c8f698e6c86246b/sphinx_rtd_theme-0.2.5b1.tar.gz";
-      md5 = "0923473a43bd2527f32151f195f2a521";
-    };
-
-
-  };
   # Avoid that setuptools is replaced, this leads to trouble
   # with buildPythonPackage.
   setuptools = pkgs.python27Packages.setuptools;
@@ -255,7 +221,7 b' let'
 in python.buildEnv.override {
   inherit python;
   extraLibs = [
-    Sphinx
-    sphinx_rtd_theme
+    sphinx
+    sphinx-rtd-theme
   ];
-}
+}
  No newline at end of file

docs/known-issues/known-issues.rst

@@ -37,6 +37,7 b' To convert the old version into a curren'
 |repo| into a local machine using a recent |git| client, then push it to a new
 |repo| inside |RCE|.
 
+
 VCS Server Memory Consumption
 -----------------------------
 
@@ -48,24 +49,34 b' shipped with the optimal configuration a'
 To fix this issue, upgrade to |RCE| 3.3.2 or greater, and if you discover
 memory consumption issues check the VCS Server settings.
 
-Fedora 23
----------
+Fedora 23 / Ubuntu 18.04
+------------------------
+
+|RCC| has a know problem with locales, due to changes in glibc 2.27+ which affects
+the local-archive format, which is now incompatible with our used glibc 2.26.
+
 
-|RCC| does not run perfectly on Fedora 23 due to a locale issue. This is a
-known issue under investigation due to the Nix packaging of the product, see the
-`Github issue here`_. |RCC| runs fine on Fedora 21.
+To work around this problem, you need set path to ``$LOCAL_ARCHIVE`` to the
+locale package in older pre glibc 2.27 format, or set `LC_ALL=C` in your enviroment.
+
+To use the pre 2.27 locale-archive fix follow these steps:
 
-To work around this problem, you need to point ``$LOCAL_ARCHIVE`` to the
-workaround locale package.
+1. Download the pre 2.27 locale-archive package
 
-1. Download this package:
-   http://lipa.ms.mff.cuni.cz/~cunav5am/nix/locale-archive
+.. code-block:: bash
+
+    wget https://dls.rhodecode.com/assets/locale-archive
+
 
 2. Point ``$LOCAL_ARCHIVE`` to the locale package.
 
 .. code-block:: bash
 
-    $ export LOCALE_ARCHIVE=/home/VERSION/locale-archive # change to your path
+    $ export LOCALE_ARCHIVE=/home/USER/locale-archive  # change to your path
+
+This can either added in `~/.rccontrol/supervisor/supervisord.ini`
+or in user .bashrc/.zshrc etc, or via a startup script that
+runs `rccontrol self-init`
 
 If you happen to be running |RCC| from systemd, use the following
 example to pass the correct locale information on boot.
@@ -85,4 +96,3 b' example to pass the correct locale infor'
     [Install]
     WantedBy=multi-user.target
 
-.. _Github issue here: https://github.com/NixOS/nix/issues/599

docs/release-notes/release-notes.rst

@@ -9,6 +9,7 b' Release Notes'
 .. toctree::
    :maxdepth: 1
 
+   release-notes-4.13.0.rst
    release-notes-4.12.4.rst
    release-notes-4.12.3.rst
    release-notes-4.12.2.rst

docs/requirements_docs.txt

@@ -1,8 +1,8 b''
-Sphinx==1.6.5
+sphinx==1.7.8
 six==1.11.0
-sphinx_rtd_theme==0.2.5b1
+sphinx_rtd_theme==0.4.1
 docutils==0.14.0
-Pygments==2.2.0
-MarkupSafe==1.0.0
-Jinja2==2.9.6
-pytz
  No newline at end of file
+pygments==2.2.0
+markupsafe==1.0.0
+jinja2==2.9.6
+pytz==2018.4

package.json

@@ -1,6 +1,13 b''
 {
   "name": "rhodecode-enterprise",
-  "version": "0.0.1",
+  "version": "1.0.0",
+  "private": true,
+  "description" : "RhodeCode JS packaged",
+  "license": "SEE LICENSE IN LICENSE.txt",
+  "repository" : {
+    "type" : "hg",
+    "url" : "https://code.rhodecode.com/rhodecode-enterprise-ce"
+  },
   "devDependencies": {
     "grunt": "^0.4.5",
     "grunt-contrib-copy": "^1.0.0",
@@ -8,16 +15,17 b''
     "grunt-contrib-jshint": "^0.12.0",
     "grunt-contrib-less": "^1.1.0",
     "grunt-contrib-watch": "^0.6.1",
-    "crisper": "^2.0.2",
-    "vulcanize": "^1.14.8",
+    "crisper": "^2.1.1",
+    "vulcanize": "^1.16.0",
     "grunt-crisper": "^1.0.1",
     "grunt-vulcanize": "^1.0.0",
     "node2nix": "^1.0.0",
     "jshint": "^2.9.1-rc3",
-    "bower": "^1.7.9",
+    "bower": "^1.8.4",
     "jquery": "1.11.3",
     "favico.js": "^0.3.10",
-    "clipboard": "^1.7.1",
+    "clipboard": "^2.0.1",
+    "qrious": "^4.0.2",
     "moment": "^2.18.1",
     "mousetrap": "^1.6.1",
     "appenlight-client": "git+https://[e-mail]/AppEnlight/appenlight-client-js.git#0.5.1"

pkgs/bower-packages.nix

@@ -1,33 +1,30 b''
+# Generated by bower2nix v3.2.0 (https://github.com/rvl/bower2nix)
 { fetchbower, buildEnv }:
 buildEnv { name = "bower-env"; ignoreCollisions = true; paths = [
-  (fetchbower "webcomponentsjs" "0.7.22" "^0.7.22" "178h9j8jq9wi5845f5pxhhhqw6x022nzmpzm4di8fgsdl1f6nr5d")
-  (fetchbower "polymer" "Polymer/polymer#1.6.1" "Polymer/polymer#^1.6.1" "09mm0jgk457gvwqlc155swch7gjr6fs3g7spnvhi6vh5b6518540")
-  (fetchbower "paper-button" "PolymerElements/paper-button#1.0.13" "PolymerElements/paper-button#^1.0.13" "0i3y153nqk06pn0gk282vyybnl3g1w3w41d5i9z659cgn27g3fvm")
-  (fetchbower "paper-spinner" "PolymerElements/paper-spinner#1.2.0" "PolymerElements/paper-spinner#^1.2.0" "1av1m6y81jw3hjhz1yqy3rwcgxarjzl58ldfn4q6sn51pgzngfqb")
-  (fetchbower "paper-tooltip" "PolymerElements/paper-tooltip#1.1.3" "PolymerElements/paper-tooltip#^1.1.2" "0vmrm1n8k9sk9nvqy03q177axy22pia6i3j1gxbk72j3pqiqvg6k")
-  (fetchbower "paper-toast" "PolymerElements/paper-toast#1.3.0" "PolymerElements/paper-toast#^1.3.0" "0x9rqxsks5455s8pk4aikpp99ijdn6kxr9gvhwh99nbcqdzcxq1m")
-  (fetchbower "paper-toggle-button" "PolymerElements/paper-toggle-button#1.2.0" "PolymerElements/paper-toggle-button#^1.2.0" "0mphcng3ngspbpg4jjn0mb91nvr4xc1phq3qswib15h6sfww1b2w")
-  (fetchbower "iron-ajax" "PolymerElements/iron-ajax#1.4.4" "PolymerElements/iron-ajax#^1.4.4" "0jpi7ik3zljw8yh2ccc85r26lcpzmkc2nl1kn6fqdx57zkzk9v5b")
-  (fetchbower "iron-autogrow-textarea" "PolymerElements/iron-autogrow-textarea#1.0.13" "PolymerElements/iron-autogrow-textarea#^1.0.13" "0zwhpl97vii1s8k0lgain8i9dnw29b0mxc5ixdscx9las13n2lqq")
-  (fetchbower "iron-a11y-keys" "PolymerElements/iron-a11y-keys#1.0.6" "PolymerElements/iron-a11y-keys#^1.0.6" "1xz3mgghfcxixq28sdb654iaxj4nyi1bzcwf77ydkms6fviqs9mv")
-  (fetchbower "iron-flex-layout" "PolymerElements/iron-flex-layout#1.3.1" "PolymerElements/iron-flex-layout#^1.0.0" "0nswv3ih3bhflgcd2wjfmddqswzgqxb2xbq65jk9w3rkj26hplbl")
-  (fetchbower "paper-behaviors" "PolymerElements/paper-behaviors#1.0.12" "PolymerElements/paper-behaviors#^1.0.0" "012bqk97awgz55cn7rm9g7cckrdhkqhls3zvp8l6nd4rdwcrdzq8")
-  (fetchbower "paper-material" "PolymerElements/paper-material#1.0.6" "PolymerElements/paper-material#^1.0.0" "0rljmknfdbm5aabvx9pk77754zckj3l127c3mvnmwkpkkr353xnh")
-  (fetchbower "paper-styles" "PolymerElements/paper-styles#1.1.4" "PolymerElements/paper-styles#^1.0.0" "0j8vg74xrcxlni8i93dsab3y80f34kk30lv4yblqpkp9c3nrilf7")
-  (fetchbower "neon-animation" "PolymerElements/neon-animation#1.2.4" "PolymerElements/neon-animation#^1.0.0" "16mz9i2n5w0k5j8d6gha23cnbdgm5syz3fawyh89gdbq97bi2q5j")
-  (fetchbower "iron-a11y-announcer" "PolymerElements/iron-a11y-announcer#1.0.5" "PolymerElements/iron-a11y-announcer#^1.0.0" "0n7c7j1pwk3835s7s2jd9125wdcsqf216yi5gj07wn5s8h8p7m9d")
-  (fetchbower "iron-overlay-behavior" "PolymerElements/iron-overlay-behavior#1.8.6" "PolymerElements/iron-overlay-behavior#^1.0.9" "14brn9gz6qqskarg3fxk91xs7vg02vgcsz9a9743kidxr0l0413m")
-  (fetchbower "iron-fit-behavior" "PolymerElements/iron-fit-behavior#1.2.5" "PolymerElements/iron-fit-behavior#^1.1.0" "1msnlh8lp1xg6v4h6dkjwj9kzac5q5q208ayla3x9hi483ki6rlf")
-  (fetchbower "iron-checked-element-behavior" "PolymerElements/iron-checked-element-behavior#1.0.5" "PolymerElements/iron-checked-element-behavior#^1.0.0" "0l0yy4ah454s8bzfv076s8by7h67zy9ni6xb932qwyhx8br6c1m7")
-  (fetchbower "promise-polyfill" "polymerlabs/promise-polyfill#1.0.1" "polymerlabs/promise-polyfill#^1.0.0" "045bj2caav3famr5hhxgs1dx7n08r4s46mlzwb313vdy17is38xb")
-  (fetchbower "iron-behaviors" "PolymerElements/iron-behaviors#1.0.17" "PolymerElements/iron-behaviors#^1.0.0" "021qvkmbk32jrrmmphpmwgby4bzi5jyf47rh1bxmq2ip07ly4bpr")
-  (fetchbower "iron-validatable-behavior" "PolymerElements/iron-validatable-behavior#1.1.1" "PolymerElements/iron-validatable-behavior#^1.0.0" "1yhxlvywhw2klbbgm3f3cmanxfxggagph4ii635zv0c13707wslv")
-  (fetchbower "iron-form-element-behavior" "PolymerElements/iron-form-element-behavior#1.0.6" "PolymerElements/iron-form-element-behavior#^1.0.0" "0rdhxivgkdhhz2yadgdbjfc70l555p3y83vjh8rfj5hr0asyn6q1")
-  (fetchbower "iron-a11y-keys-behavior" "polymerelements/iron-a11y-keys-behavior#1.1.9" "polymerelements/iron-a11y-keys-behavior#^1.0.0" "1imm4gc84qizihhbyhfa8lwjh3myhj837f79i5m04xjgwrjmkaf6")
-  (fetchbower "paper-ripple" "PolymerElements/paper-ripple#1.0.8" "PolymerElements/paper-ripple#^1.0.0" "0r9sq8ik7wwrw0qb82c3rw0c030ljwd3s466c9y4qbcrsbvfjnns")
-  (fetchbower "font-roboto" "PolymerElements/font-roboto#1.0.1" "PolymerElements/font-roboto#^1.0.1" "02jz43r0wkyr3yp7rq2rc08l5cwnsgca9fr54sr4rhsnl7cjpxrj")
-  (fetchbower "iron-meta" "PolymerElements/iron-meta#1.1.2" "PolymerElements/iron-meta#^1.0.0" "1wl4dx8fnsknw9z9xi8bpc4cy9x70c11x4zxwxnj73hf3smifppl")
-  (fetchbower "iron-resizable-behavior" "PolymerElements/iron-resizable-behavior#1.0.5" "PolymerElements/iron-resizable-behavior#^1.0.0" "1fd5zmbr2hax42vmcasncvk7lzi38fmb1kyii26nn8pnnjak7zkn")
-  (fetchbower "iron-selector" "PolymerElements/iron-selector#1.5.2" "PolymerElements/iron-selector#^1.0.0" "1ajv46llqzvahm5g6g75w7nfyjcslp53ji0wm96l2k94j87spv3r")
-  (fetchbower "web-animations-js" "web-animations/web-animations-js#2.2.2" "web-animations/web-animations-js#^2.2.0" "1izfvm3l67vwys0bqbhidi9rqziw2f8wv289386sc6jsxzgkzhga")
+  (fetchbower "webcomponentsjs" "1.2.3" "^1.2.3" "0db2fjk779rhl9d4ifpk1j9a83vngx6j3hk7si1752dqxj857267")
+  (fetchbower "polymer" "Polymer/polymer#2.6.0" "Polymer/polymer#^2.6.0" "1jw5syjn0naa7dlxzxvsgwb20zbkbpx0pcg0wzjq4wr49c20nw0k")
+  (fetchbower "paper-button" "PolymerElements/paper-button#2.1.2" "PolymerElements/paper-button#^2.1.2" "1lwmjwhvimh02gss39yjp5h7yg9mxl1ig59yrk085v86y5f5ilz1")
+  (fetchbower "paper-spinner" "PolymerElements/paper-spinner#2.1.0" "PolymerElements/paper-spinner#^2.1.0" "07jamgxzvy4r9pgf5ikj1fm8nkrw6sywgcxvmm7ax58hjvmd4i45")
+  (fetchbower "paper-tooltip" "PolymerElements/paper-tooltip#2.1.1" "PolymerElements/paper-tooltip#^2.1.1" "0ja9vz4xps7dxfx6kmrwq0gifbjsyc3pk9l3hd5233ss72ghlvgb")
+  (fetchbower "paper-toast" "PolymerElements/paper-toast#2.1.2" "PolymerElements/paper-toast#^2.1.2" "0mkj9ayjx2l9hvrz4kw3yjbgrs9w873k05ywv94nh92mzpjkxn04")
+  (fetchbower "paper-toggle-button" "PolymerElements/paper-toggle-button#2.1.1" "PolymerElements/paper-toggle-button#^2.1.1" "1rl5ar3ny6r3v9hr8s68v8b3zh5yl52pikhi4b9iv9grv29g5999")
+  (fetchbower "iron-ajax" "PolymerElements/iron-ajax#2.1.3" "PolymerElements/iron-ajax#^2.1.3" "187zb6lkv18yapl292qhn611mbl4i2bqs2dimlwmdjaln285nc1l")
+  (fetchbower "iron-autogrow-textarea" "PolymerElements/iron-autogrow-textarea#2.2.0" "PolymerElements/iron-autogrow-textarea#^2.2.0" "1cx7g49m20z08x25z24g2pvv6m6dwfbkga3nsw0rpi3jk7xhznqw")
+  (fetchbower "iron-a11y-keys" "PolymerElements/iron-a11y-keys#2.1.2" "PolymerElements/iron-a11y-keys#^2.1.2" "0zdn3bzfrk88iwahmmzvcjn3m90zga1l8dx42b365n8if0n0zapj")
+  (fetchbower "shadycss" "webcomponents/shadycss#1.3.5" "webcomponents/shadycss#^v1.1.0" "1xp07d6jlmc3ywg4cdp2nijwi0vrbzw468242xz0s6zx5z2n2wzf")
+  (fetchbower "iron-flex-layout" "PolymerElements/iron-flex-layout#2.0.3" "PolymerElements/iron-flex-layout#1 - 2" "1ir9ili3rkcgbfj6is5s70vyb7m6avz0av0lpjjlhhsplrrl90bk")
+  (fetchbower "paper-behaviors" "PolymerElements/paper-behaviors#2.1.1" "PolymerElements/paper-behaviors#1 - 2" "0wzyznmy1q64jsm6lck0xma92lg2hz9h9m05r68iy20iyrvbjld7")
+  (fetchbower "paper-styles" "PolymerElements/paper-styles#2.1.0" "PolymerElements/paper-styles#1 - 2" "0xzsd7pvjsk7lgcmvx4q0dvps40q3mymkx26zgs8ns91s1ssfqbx")
+  (fetchbower "iron-a11y-announcer" "PolymerElements/iron-a11y-announcer#2.1.0" "PolymerElements/iron-a11y-announcer#1 - 2" "1hvmw76im8y3pxmg1yfnlkzap429q9i4v1z42y7m8lz0996jmxrz")
+  (fetchbower "iron-overlay-behavior" "PolymerElements/iron-overlay-behavior#2.3.4" "PolymerElements/iron-overlay-behavior#1 - 2" "0hk2fxhfz7kg27349qz1c87vq6nfdcnjr9az3q95ak3rigsb71wg")
+  (fetchbower "iron-fit-behavior" "PolymerElements/iron-fit-behavior#2.2.1" "PolymerElements/iron-fit-behavior#1 - 2" "0v89am4r2fncr7m9k0f2xqh5bvgn2dqjjc7p515ix4grk89whbs3")
+  (fetchbower "iron-checked-element-behavior" "PolymerElements/iron-checked-element-behavior#2.1.1" "PolymerElements/iron-checked-element-behavior#1 - 2" "00gpvnb7anspadw59av7lca6p03qlx59dgnhw3lqf5v223dsh0l3")
+  (fetchbower "iron-behaviors" "PolymerElements/iron-behaviors#2.1.1" "PolymerElements/iron-behaviors#1 - 2" "0c71l1il76jg8gvyh8bwlqraif53gfnz5700vpg35qyr1biwbr1w")
+  (fetchbower "iron-validatable-behavior" "PolymerElements/iron-validatable-behavior#2.1.0" "PolymerElements/iron-validatable-behavior#1 - 2" "1j65zv7hcxdcyyanzhp2wap3bnx7cz8ghd44m12fad2206jj6ysi")
+  (fetchbower "iron-a11y-keys-behavior" "PolymerElements/iron-a11y-keys-behavior#2.1.1" "PolymerElements/iron-a11y-keys-behavior#1 - 2" "0jsw7wvh4wzcjqdbvxyha99h1ph61lykyyyvdb6hm8m8w40bhs9b")
+  (fetchbower "paper-ripple" "PolymerElements/paper-ripple#2.1.1" "PolymerElements/paper-ripple#1 - 2" "0sam6w83nxf9mss79j6hjfsx0288pf4hwr1bw5xclzgpddcjmrm5")
+  (fetchbower "font-roboto" "PolymerElements/font-roboto#1.1.0" "PolymerElements/font-roboto#^1.0.1" "0z4msvaa5pnr84j2r957g313fmdbdbrknhdw1axy5g48845yv04s")
+  (fetchbower "iron-resizable-behavior" "PolymerElements/iron-resizable-behavior#2.1.1" "PolymerElements/iron-resizable-behavior#1 - 2" "0169rg6kby0ypfiwbkin46ywaszwj7r91yn081yqd96l1115aqnk")
+  (fetchbower "iron-form-element-behavior" "PolymerElements/iron-form-element-behavior#2.1.3" "PolymerElements/iron-form-element-behavior#1 - 2" "1c83kkfqvb5pdlvrhad3l7b11i244a8lykc8kzfnb9ya0xrln0cn")
+  (fetchbower "iron-meta" "PolymerElements/iron-meta#2.1.1" "PolymerElements/iron-meta#1 - 2" "12w1pcl3w97xpxihqddbf2d9dx8xz5i1jd9bz1xrmfsh25isni40")
 ]; }

pkgs/node-default.nix

@@ -1,12 +1,14 b''
-# This file has been generated by node2nix 1.0.0. Do not edit!
+# This file has been generated by node2nix 1.6.0. Do not edit!
 
 {pkgs ? import <nixpkgs> {
     inherit system;
-  }, system ? builtins.currentSystem}:
+  }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-6_x"}:
 
 let
   nodeEnv = import ./node-env.nix {
-    inherit (pkgs) stdenv python utillinux runCommand writeTextFile nodejs;
+    inherit (pkgs) stdenv python2 utillinux runCommand writeTextFile;
+    inherit nodejs;
+    libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null;
   };
 in
 import ./node-packages.nix {

pkgs/node-env.nix

@@ -1,31 +1,33 b''
 # This file originates from node2nix
 
-{stdenv, python, nodejs, utillinux, runCommand, writeTextFile}:
+{stdenv, nodejs, python2, utillinux, libtool, runCommand, writeTextFile}:
 
 let
+  python = if nodejs ? python then nodejs.python else python2;
+
   # Create a tar wrapper that filters all the 'Ignoring unknown extended header keyword' noise
   tarWrapper = runCommand "tarWrapper" {} ''
     mkdir -p $out/bin
-    
+
     cat > $out/bin/tar <<EOF
     #! ${stdenv.shell} -e
     $(type -p tar) "\$@" --warning=no-unknown-keyword
     EOF
-    
+
     chmod +x $out/bin/tar
   '';
-  
+
   # Function that generates a TGZ file from a NPM project
   buildNodeSourceDist =
     { name, version, src, ... }:
-    
+
     stdenv.mkDerivation {
       name = "node-tarball-${name}-${version}";
       inherit src;
       buildInputs = [ nodejs ];
       buildPhase = ''
         export HOME=$TMPDIR
-        tgzFile=$(npm pack)
+        tgzFile=$(npm pack | tail -n 1) # Hooks to the pack command will add output (https://docs.npmjs.com/misc/scripts)
       '';
       installPhase = ''
         mkdir -p $out/tarballs
@@ -42,151 +44,322 b' let'
           # Bundle the dependencies of the package
           mkdir -p node_modules
           cd node_modules
-          
+
           # Only include dependencies if they don't exist. They may also be bundled in the package.
           if [ ! -e "${dependency.name}" ]
           then
               ${composePackage dependency}
           fi
-          
+
           cd ..
         ''
       ) dependencies);
 
   # Recursively composes the dependencies of a package
   composePackage = { name, packageName, src, dependencies ? [], ... }@args:
-    let
-      fixImpureDependencies = writeTextFile {
-        name = "fixDependencies.js";
-        text = ''
-          var fs = require('fs');
-          var url = require('url');
-          
-          /*
-           * Replaces an impure version specification by *
-           */
-          function replaceImpureVersionSpec(versionSpec) {
-              var parsedUrl = url.parse(versionSpec);
-              
-              if(versionSpec == "latest" || versionSpec == "unstable" ||
-                  versionSpec.substr(0, 2) == ".." || dependency.substr(0, 2) == "./" || dependency.substr(0, 2) == "~/" || dependency.substr(0, 1) == '/')
-                  return '*';
-              else if(parsedUrl.protocol == "git:" || parsedUrl.protocol == "git+ssh:" || parsedUrl.protocol == "git+http:" || parsedUrl.protocol == "git+https:" ||
-                  parsedUrl.protocol == "http:" || parsedUrl.protocol == "https:")
-                  return '*';
-              else
-                  return versionSpec;
-          }
-      
-          var packageObj = JSON.parse(fs.readFileSync('./package.json'));
-      
-          /* Replace dependencies */
-          if(packageObj.dependencies !== undefined) {
-              for(var dependency in packageObj.dependencies) {
-                  var versionSpec = packageObj.dependencies[dependency];
-                  packageObj.dependencies[dependency] = replaceImpureVersionSpec(versionSpec);
-              }
-          }
-          
-          /* Replace development dependencies */
-          if(packageObj.devDependencies !== undefined) {
-              for(var dependency in packageObj.devDependencies) {
-                  var versionSpec = packageObj.devDependencies[dependency];
-                  packageObj.devDependencies[dependency] = replaceImpureVersionSpec(versionSpec);
-              }
-          }
-          
-          /* Replace optional dependencies */
-          if(packageObj.optionalDependencies !== undefined) {
-              for(var dependency in packageObj.optionalDependencies) {
-                  var versionSpec = packageObj.optionalDependencies[dependency];
-                  packageObj.optionalDependencies[dependency] = replaceImpureVersionSpec(versionSpec);
-              }
-          }
-          
-          /* Write the fixed JSON file */
-          fs.writeFileSync("package.json", JSON.stringify(packageObj));
-        '';
-      };
-    in
     ''
       DIR=$(pwd)
       cd $TMPDIR
-      
+
       unpackFile ${src}
-      
+
       # Make the base dir in which the target dependency resides first
       mkdir -p "$(dirname "$DIR/${packageName}")"
 
       if [ -f "${src}" ]
       then
           # Figure out what directory has been unpacked
-          packageDir=$(find . -type d -maxdepth 1 | tail -1)
-          
+          packageDir="$(find . -maxdepth 1 -type d | tail -1)"
+
           # Restore write permissions to make building work
+          find "$packageDir" -type d -print0 | xargs -0 chmod u+x
           chmod -R u+w "$packageDir"
-          
+
           # Move the extracted tarball into the output folder
           mv "$packageDir" "$DIR/${packageName}"
       elif [ -d "${src}" ]
       then
+          # Get a stripped name (without hash) of the source directory.
+          # On old nixpkgs it's already set internally.
+          if [ -z "$strippedName" ]
+          then
+              strippedName="$(stripHash ${src})"
+          fi
+
           # Restore write permissions to make building work
-           chmod -R u+w $strippedName
-           
-           # Move the extracted directory into the output folder
-           mv $strippedName "$DIR/${packageName}"
+          chmod -R u+w "$strippedName"
+
+          # Move the extracted directory into the output folder
+          mv "$strippedName" "$DIR/${packageName}"
       fi
-      
+
       # Unset the stripped name to not confuse the next unpack step
       unset strippedName
-      
-      # Some version specifiers (latest, unstable, URLs, file paths) force NPM to make remote connections or consult paths outside the Nix store.
-      # The following JavaScript replaces these by * to prevent that
+
+      # Include the dependencies of the package
       cd "$DIR/${packageName}"
-      node ${fixImpureDependencies}
-      
-      # Include the dependencies of the package
       ${includeDependencies { inherit dependencies; }}
       cd ..
       ${stdenv.lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."}
     '';
 
+  pinpointDependencies = {dependencies, production}:
+    let
+      pinpointDependenciesFromPackageJSON = writeTextFile {
+        name = "pinpointDependencies.js";
+        text = ''
+          var fs = require('fs');
+          var path = require('path');
+
+          function resolveDependencyVersion(location, name) {
+              if(location == process.env['NIX_STORE']) {
+                  return null;
+              } else {
+                  var dependencyPackageJSON = path.join(location, "node_modules", name, "package.json");
+
+                  if(fs.existsSync(dependencyPackageJSON)) {
+                      var dependencyPackageObj = JSON.parse(fs.readFileSync(dependencyPackageJSON));
+
+                      if(dependencyPackageObj.name == name) {
+                          return dependencyPackageObj.version;
+                      }
+                  } else {
+                      return resolveDependencyVersion(path.resolve(location, ".."), name);
+                  }
+              }
+          }
+
+          function replaceDependencies(dependencies) {
+              if(typeof dependencies == "object" && dependencies !== null) {
+                  for(var dependency in dependencies) {
+                      var resolvedVersion = resolveDependencyVersion(process.cwd(), dependency);
+
+                      if(resolvedVersion === null) {
+                          process.stderr.write("WARNING: cannot pinpoint dependency: "+dependency+", context: "+process.cwd()+"\n");
+                      } else {
+                          dependencies[dependency] = resolvedVersion;
+                      }
+                  }
+              }
+          }
+
+          /* Read the package.json configuration */
+          var packageObj = JSON.parse(fs.readFileSync('./package.json'));
+
+          /* Pinpoint all dependencies */
+          replaceDependencies(packageObj.dependencies);
+          if(process.argv[2] == "development") {
+              replaceDependencies(packageObj.devDependencies);
+          }
+          replaceDependencies(packageObj.optionalDependencies);
+
+          /* Write the fixed package.json file */
+          fs.writeFileSync("package.json", JSON.stringify(packageObj, null, 2));
+        '';
+      };
+    in
+    ''
+      node ${pinpointDependenciesFromPackageJSON} ${if production then "production" else "development"}
+
+      ${stdenv.lib.optionalString (dependencies != [])
+        ''
+          if [ -d node_modules ]
+          then
+              cd node_modules
+              ${stdenv.lib.concatMapStrings (dependency: pinpointDependenciesOfPackage dependency) dependencies}
+              cd ..
+          fi
+        ''}
+    '';
+
+  # Recursively traverses all dependencies of a package and pinpoints all
+  # dependencies in the package.json file to the versions that are actually
+  # being used.
+
+  pinpointDependenciesOfPackage = { packageName, dependencies ? [], production ? true, ... }@args:
+    ''
+      if [ -d "${packageName}" ]
+      then
+          cd "${packageName}"
+          ${pinpointDependencies { inherit dependencies production; }}
+          cd ..
+          ${stdenv.lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."}
+      fi
+    '';
+
   # Extract the Node.js source code which is used to compile packages with
   # native bindings
   nodeSources = runCommand "node-sources" {} ''
     tar --no-same-owner --no-same-permissions -xf ${nodejs.src}
     mv node-* $out
   '';
-  
+
+  # Script that adds _integrity fields to all package.json files to prevent NPM from consulting the cache (that is empty)
+  addIntegrityFieldsScript = writeTextFile {
+    name = "addintegrityfields.js";
+    text = ''
+      var fs = require('fs');
+      var path = require('path');
+
+      function augmentDependencies(baseDir, dependencies) {
+          for(var dependencyName in dependencies) {
+              var dependency = dependencies[dependencyName];
+
+              // Open package.json and augment metadata fields
+              var packageJSONDir = path.join(baseDir, "node_modules", dependencyName);
+              var packageJSONPath = path.join(packageJSONDir, "package.json");
+
+              if(fs.existsSync(packageJSONPath)) { // Only augment packages that exist. Sometimes we may have production installs in which development dependencies can be ignored
+                  console.log("Adding metadata fields to: "+packageJSONPath);
+                  var packageObj = JSON.parse(fs.readFileSync(packageJSONPath));
+
+                  if(dependency.integrity) {
+                      packageObj["_integrity"] = dependency.integrity;
+                  } else {
+                      packageObj["_integrity"] = "sha1-000000000000000000000000000="; // When no _integrity string has been provided (e.g. by Git dependencies), add a dummy one. It does not seem to harm and it bypasses downloads.
+                  }
+
+                  packageObj["_resolved"] = dependency.version; // Set the resolved version to the version identifier. This prevents NPM from cloning Git repositories.
+                  fs.writeFileSync(packageJSONPath, JSON.stringify(packageObj, null, 2));
+              }
+
+              // Augment transitive dependencies
+              if(dependency.dependencies !== undefined) {
+                  augmentDependencies(packageJSONDir, dependency.dependencies);
+              }
+          }
+      }
+
+      if(fs.existsSync("./package-lock.json")) {
+          var packageLock = JSON.parse(fs.readFileSync("./package-lock.json"));
+
+          if(packageLock.lockfileVersion !== 1) {
+             process.stderr.write("Sorry, I only understand lock file version 1!\n");
+             process.exit(1);
+          }
+
+          if(packageLock.dependencies !== undefined) {
+              augmentDependencies(".", packageLock.dependencies);
+          }
+      }
+    '';
+  };
+
+  # Reconstructs a package-lock file from the node_modules/ folder structure and package.json files with dummy sha1 hashes
+  reconstructPackageLock = writeTextFile {
+    name = "addintegrityfields.js";
+    text = ''
+      var fs = require('fs');
+      var path = require('path');
+
+      var packageObj = JSON.parse(fs.readFileSync("package.json"));
+
+      var lockObj = {
+          name: packageObj.name,
+          version: packageObj.version,
+          lockfileVersion: 1,
+          requires: true,
+          dependencies: {}
+      };
+
+      function augmentPackageJSON(filePath, dependencies) {
+          var packageJSON = path.join(filePath, "package.json");
+          if(fs.existsSync(packageJSON)) {
+              var packageObj = JSON.parse(fs.readFileSync(packageJSON));
+              dependencies[packageObj.name] = {
+                  version: packageObj.version,
+                  integrity: "sha1-000000000000000000000000000=",
+                  dependencies: {}
+              };
+              processDependencies(path.join(filePath, "node_modules"), dependencies[packageObj.name].dependencies);
+          }
+      }
+
+      function processDependencies(dir, dependencies) {
+          if(fs.existsSync(dir)) {
+              var files = fs.readdirSync(dir);
+
+              files.forEach(function(entry) {
+                  var filePath = path.join(dir, entry);
+                  var stats = fs.statSync(filePath);
+
+                  if(stats.isDirectory()) {
+                      if(entry.substr(0, 1) == "@") {
+                          // When we encounter a namespace folder, augment all packages belonging to the scope
+                          var pkgFiles = fs.readdirSync(filePath);
+
+                          pkgFiles.forEach(function(entry) {
+                              if(stats.isDirectory()) {
+                                  var pkgFilePath = path.join(filePath, entry);
+                                  augmentPackageJSON(pkgFilePath, dependencies);
+                              }
+                          });
+                      } else {
+                          augmentPackageJSON(filePath, dependencies);
+                      }
+                  }
+              });
+          }
+      }
+
+      processDependencies("node_modules", lockObj.dependencies);
+
+      fs.writeFileSync("package-lock.json", JSON.stringify(lockObj, null, 2));
+    '';
+  };
+
   # Builds and composes an NPM package including all its dependencies
-  buildNodePackage = { name, packageName, version, dependencies ? [], production ? true, npmFlags ? "", dontNpmInstall ? false, preRebuild ? "", ... }@args:
-    
-    stdenv.lib.makeOverridable stdenv.mkDerivation (builtins.removeAttrs args [ "dependencies" ] // {
+  buildNodePackage =
+    { name
+    , packageName
+    , version
+    , dependencies ? []
+    , buildInputs ? []
+    , production ? true
+    , npmFlags ? ""
+    , dontNpmInstall ? false
+    , bypassCache ? false
+    , preRebuild ? ""
+    , dontStrip ? true
+    , unpackPhase ? "true"
+    , buildPhase ? "true"
+    , ... }@args:
+
+    let
+      forceOfflineFlag = if bypassCache then "--offline" else "--registry http://www.example.com";
+      extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" "dontStrip" "dontNpmInstall" "preRebuild" "unpackPhase" "buildPhase" ];
+    in
+    stdenv.mkDerivation ({
       name = "node-${name}-${version}";
-      buildInputs = [ tarWrapper python nodejs ] ++ stdenv.lib.optional (stdenv.isLinux) utillinux ++ args.buildInputs or [];
-      dontStrip = args.dontStrip or true; # Striping may fail a build for some package deployments
-      
-      inherit dontNpmInstall preRebuild;
-      
-      unpackPhase = args.unpackPhase or "true";
-      
-      buildPhase = args.buildPhase or "true";
-      
+      buildInputs = [ tarWrapper python nodejs ]
+        ++ stdenv.lib.optional (stdenv.isLinux) utillinux
+        ++ stdenv.lib.optional (stdenv.isDarwin) libtool
+        ++ buildInputs;
+
+      inherit dontStrip; # Stripping may fail a build for some package deployments
+      inherit dontNpmInstall preRebuild unpackPhase buildPhase;
+
       compositionScript = composePackage args;
-      passAsFile = [ "compositionScript" ];
-      
-      installPhase = args.installPhase or ''
+      pinpointDependenciesScript = pinpointDependenciesOfPackage args;
+
+      passAsFile = [ "compositionScript" "pinpointDependenciesScript" ];
+
+      installPhase = ''
         # Create and enter a root node_modules/ folder
         mkdir -p $out/lib/node_modules
         cd $out/lib/node_modules
-          
+
         # Compose the package and all its dependencies
         source $compositionScriptPath
-        
+
+        # Pinpoint the versions of all dependencies to the ones that are actually being used
+        echo "pinpointing versions of dependencies..."
+        source $pinpointDependenciesScriptPath
+
         # Patch the shebangs of the bundled modules to prevent them from
         # calling executables outside the Nix store as much as possible
         patchShebangs .
-        
+
         # Deploy the Node.js package by running npm install. Since the
         # dependencies have been provided already by ourselves, it should not
         # attempt to install them again, which is good, because we want to make
@@ -196,23 +369,37 b' let'
         #
         # The other responsibilities of NPM are kept -- version checks, build
         # steps, postprocessing etc.
-        
+
         export HOME=$TMPDIR
         cd "${packageName}"
         runHook preRebuild
-        npm --registry http://www.example.com --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} rebuild
-        
+
+        ${stdenv.lib.optionalString bypassCache ''
+          if [ ! -f package-lock.json ]
+          then
+              echo "No package-lock.json file found, reconstructing..."
+              node ${reconstructPackageLock}
+          fi
+
+          node ${addIntegrityFieldsScript}
+        ''}
+
+        npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} rebuild
+
         if [ "$dontNpmInstall" != "1" ]
         then
-            npm --registry http://www.example.com --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} install
+            # NPM tries to download packages even when they already exist if npm-shrinkwrap is used.
+            rm -f npm-shrinkwrap.json
+
+            npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} install
         fi
-        
+
         # Create symlink to the deployed executable folder, if applicable
         if [ -d "$out/lib/node_modules/.bin" ]
         then
             ln -s $out/lib/node_modules/.bin $out/bin
         fi
-        
+
         # Create symlinks to the deployed manual page folders, if applicable
         if [ -d "$out/lib/node_modules/${packageName}/man" ]
         then
@@ -226,52 +413,111 b' let'
                 done
             done
         fi
+
+        # Run post install hook, if provided
+        runHook postInstall
       '';
-    });
+    } // extraArgs);
 
   # Builds a development shell
-  buildNodeShell = { name, packageName, version, src, dependencies ? [], production ? true, npmFlags ? "", dontNpmInstall ? false, ... }@args:
+  buildNodeShell =
+    { name
+    , packageName
+    , version
+    , src
+    , dependencies ? []
+    , buildInputs ? []
+    , production ? true
+    , npmFlags ? ""
+    , dontNpmInstall ? false
+    , bypassCache ? false
+    , dontStrip ? true
+    , unpackPhase ? "true"
+    , buildPhase ? "true"
+    , ... }@args:
+
     let
-      nodeDependencies = stdenv.mkDerivation {
+      forceOfflineFlag = if bypassCache then "--offline" else "--registry http://www.example.com";
+
+      extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" ];
+
+      nodeDependencies = stdenv.mkDerivation ({
         name = "node-dependencies-${name}-${version}";
-        
-        buildInputs = [ tarWrapper python nodejs ] ++ stdenv.lib.optional (stdenv.isLinux) utillinux ++ args.buildInputs or [];
-        
+
+        buildInputs = [ tarWrapper python nodejs ]
+          ++ stdenv.lib.optional (stdenv.isLinux) utillinux
+          ++ stdenv.lib.optional (stdenv.isDarwin) libtool
+          ++ buildInputs;
+
+        inherit dontStrip; # Stripping may fail a build for some package deployments
+        inherit dontNpmInstall unpackPhase buildPhase;
+
         includeScript = includeDependencies { inherit dependencies; };
-        passAsFile = [ "includeScript" ];
-        
-        buildCommand = ''
-          mkdir -p $out/lib
-          cd $out/lib
+        pinpointDependenciesScript = pinpointDependenciesOfPackage args;
+
+        passAsFile = [ "includeScript" "pinpointDependenciesScript" ];
+
+        installPhase = ''
+          mkdir -p $out/${packageName}
+          cd $out/${packageName}
+
           source $includeScriptPath
-          
+
           # Create fake package.json to make the npm commands work properly
-          cat > package.json <<EOF
-          {
-              "name": "${packageName}",
-              "version": "${version}"
-          }
-          EOF
-          
+          cp ${src}/package.json .
+          chmod 644 package.json
+          ${stdenv.lib.optionalString bypassCache ''
+            if [ -f ${src}/package-lock.json ]
+            then
+                cp ${src}/package-lock.json .
+            fi
+          ''}
+
+          # Pinpoint the versions of all dependencies to the ones that are actually being used
+          echo "pinpointing versions of dependencies..."
+          cd ..
+          ${stdenv.lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."}
+
+          source $pinpointDependenciesScriptPath
+          cd ${packageName}
+
           # Patch the shebangs of the bundled modules to prevent them from
           # calling executables outside the Nix store as much as possible
           patchShebangs .
-          
-          export HOME=$TMPDIR
-          npm --registry http://www.example.com --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} rebuild
-          
-          ${stdenv.lib.optionalString (!dontNpmInstall) ''
-            npm --registry http://www.example.com --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} install
+
+          export HOME=$PWD
+
+          ${stdenv.lib.optionalString bypassCache ''
+            if [ ! -f package-lock.json ]
+            then
+                echo "No package-lock.json file found, reconstructing..."
+                node ${reconstructPackageLock}
+            fi
+
+            node ${addIntegrityFieldsScript}
           ''}
 
+          npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} rebuild
+
+          ${stdenv.lib.optionalString (!dontNpmInstall) ''
+            # NPM tries to download packages even when they already exist if npm-shrinkwrap is used.
+            rm -f npm-shrinkwrap.json
+
+            npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} install
+          ''}
+
+          cd ..
+          ${stdenv.lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."}
+
+          mv ${packageName} lib
           ln -s $out/lib/node_modules/.bin $out/bin
         '';
-      };
+      } // extraArgs);
     in
-    stdenv.lib.makeOverridable stdenv.mkDerivation {
+    stdenv.mkDerivation {
       name = "node-shell-${name}-${version}";
-      
-      buildInputs = [ python nodejs ] ++ stdenv.lib.optional (stdenv.isLinux) utillinux ++ args.buildInputs or [];
+
+      buildInputs = [ python nodejs ] ++ stdenv.lib.optional (stdenv.isLinux) utillinux ++ buildInputs;
       buildCommand = ''
         mkdir -p $out/bin
         cat > $out/bin/shell <<EOF
@@ -281,7 +527,7 b' let'
         EOF
         chmod +x $out/bin/shell
       '';
-      
+
       # Provide the dependencies in a development shell through the NODE_PATH environment variable
       inherit nodeDependencies;
       shellHook = stdenv.lib.optionalString (dependencies != []) ''
@@ -289,4 +535,8 b' let'
       '';
     };
 in
-{ inherit buildNodeSourceDist buildNodePackage buildNodeShell; }
+{
+  buildNodeSourceDist = stdenv.lib.makeOverridable buildNodeSourceDist;
+  buildNodePackage = stdenv.lib.makeOverridable buildNodePackage;
+  buildNodeShell = stdenv.lib.makeOverridable buildNodeShell;
+}

pkgs/python-packages-overrides.nix

@@ -4,10 +4,13 b''
 # python-packages.nix. The main objective is to add needed dependencies of C
 # libraries and tweak the build instructions where needed.
 
-{ pkgs, basePythonPackages }:
+{ pkgs
+, basePythonPackages
+}:
 
 let
   sed = "sed -i";
+
   localLicenses = {
     repoze = {
       fullName = "Repoze License";
@@ -19,32 +22,41 b' in'
 
 self: super: {
 
-  appenlight-client = super.appenlight-client.override (attrs: {
+  "appenlight-client" = super."appenlight-client".override (attrs: {
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   });
 
-  Beaker = super.Beaker.override (attrs: {
+  "beaker" = super."beaker".override (attrs: {
     patches = [
-      ./patch-beaker-lock-func-debug.diff
+      ./patch_beaker/patch-beaker-lock-func-debug.diff
+      ./patch_beaker/patch-beaker-metadata-reuse.diff
     ];
   });
 
-  future = super.future.override (attrs: {
+  "gevent" = super."gevent".override (attrs: {
+    propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
+      # NOTE: (marcink) odd requirements from gevent aren't set properly,
+      # thus we need to inject psutil manually
+      self."psutil"
+    ];
+  });
+
+  "future" = super."future".override (attrs: {
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   });
 
-  testpath = super.testpath.override (attrs: {
+  "testpath" = super."testpath".override (attrs: {
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   });
 
-  gnureadline = super.gnureadline.override (attrs: {
-    buildInputs = attrs.buildInputs ++ [
+  "gnureadline" = super."gnureadline".override (attrs: {
+    buildInputs = [
       pkgs.ncurses
     ];
     patchPhase = ''
@@ -52,56 +64,50 b' self: super: {'
     '';
   });
 
-  gunicorn = super.gunicorn.override (attrs: {
-    propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
+  "gunicorn" = super."gunicorn".override (attrs: {
+    propagatedBuildInputs = [
       # johbo: futures is needed as long as we are on Python 2, otherwise
       # gunicorn explodes if used with multiple threads per worker.
-      self.futures
+      self."futures"
     ];
   });
 
-  nbconvert = super.nbconvert.override (attrs: {
+ "nbconvert" = super."nbconvert".override (attrs: {
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
       # marcink: plug in jupyter-client for notebook rendering
-      self.jupyter-client
+      self."jupyter-client"
     ];
   });
 
-  ipython = super.ipython.override (attrs: {
+  "ipython" = super."ipython".override (attrs: {
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
-      self.gnureadline
+      self."gnureadline"
     ];
   });
 
-  lxml = super.lxml.override (attrs: {
-    # johbo: On 16.09 we need this to compile on darwin, otherwise compilation
-    # fails on Darwin.
-    hardeningDisable = if pkgs.stdenv.isDarwin then [ "format" ] else null;
-    buildInputs = with self; [
+  "lxml" = super."lxml".override (attrs: {
+    buildInputs = [
       pkgs.libxml2
       pkgs.libxslt
     ];
+    propagatedBuildInputs = [
+      # Needed, so that "setup.py bdist_wheel" does work
+      self."wheel"
+    ];
   });
 
-  MySQL-python = super.MySQL-python.override (attrs: {
-    buildInputs = attrs.buildInputs ++ [
+  "mysql-python" = super."mysql-python".override (attrs: {
+    buildInputs = [
       pkgs.openssl
     ];
-    propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
+    propagatedBuildInputs = [
       pkgs.libmysql
       pkgs.zlib
     ];
   });
 
-  psutil = super.psutil.override (attrs: {
-    buildInputs = attrs.buildInputs ++
-      pkgs.lib.optional pkgs.stdenv.isDarwin pkgs.darwin.IOKit;
-  });
-
-  psycopg2 = super.psycopg2.override (attrs: {
-    buildInputs = attrs.buildInputs ++
-      pkgs.lib.optional pkgs.stdenv.isDarwin pkgs.openssl;
-    propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
+  "psycopg2" = super."psycopg2".override (attrs: {
+    propagatedBuildInputs = [
       pkgs.postgresql
     ];
     meta = {
@@ -109,8 +115,8 b' self: super: {'
     };
   });
 
-  pycurl = super.pycurl.override (attrs: {
-    propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
+  "pycurl" = super."pycurl".override (attrs: {
+    propagatedBuildInputs = [
       pkgs.curl
       pkgs.openssl
     ];
@@ -119,30 +125,23 b' self: super: {'
       export PYCURL_SSL_LIBRARY=openssl
     '';
     meta = {
-      # TODO: It is LGPL and MIT
       license = pkgs.lib.licenses.mit;
     };
   });
 
-  pyramid = super.pyramid.override (attrs: {
-    postFixup = ''
-      wrapPythonPrograms
-      # TODO: johbo: "wrapPython" adds this magic line which
-      # confuses pserve.
-      ${sed} '/import sys; sys.argv/d' $out/bin/.pserve-wrapped
-    '';
+  "pyramid" = super."pyramid".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
-  pyramid-debugtoolbar = super.pyramid-debugtoolbar.override (attrs: {
+  "pyramid-debugtoolbar" = super."pyramid-debugtoolbar".override (attrs: {
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal localLicenses.repoze ];
     };
   });
 
-  pysqlite = super.pysqlite.override (attrs: {
+  "pysqlite" = super."pysqlite".override (attrs: {
     propagatedBuildInputs = [
       pkgs.sqlite
     ];
@@ -151,41 +150,39 b' self: super: {'
     };
   });
 
-  pytest-runner = super.pytest-runner.override (attrs: {
+  "pytest-runner" = super."pytest-runner".override (attrs: {
     propagatedBuildInputs = [
-      self.setuptools-scm
+      self."setuptools-scm"
+    ];
+  });
+
+  "python-ldap" = super."python-ldap".override (attrs: {
+    propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
+      pkgs.openldap
+      pkgs.cyrus_sasl
+      pkgs.openssl
     ];
   });
 
-  python-ldap = super.python-ldap.override (attrs: {
-    propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
-      pkgs.cyrus_sasl
-      pkgs.openldap
-      pkgs.openssl
+  "python-pam" = super."python-pam".override (attrs: {
+    propagatedBuildInputs = [
+        pkgs.pam
     ];
-    # TODO: johbo: Remove the "or" once we drop 16.03 support.
-    NIX_CFLAGS_COMPILE = "-I${pkgs.cyrus_sasl.dev or pkgs.cyrus_sasl}/include/sasl";
+    # TODO: johbo: Check if this can be avoided, or transform into
+    # a real patch
+    patchPhase = ''
+      substituteInPlace pam.py \
+        --replace 'find_library("pam")' '"${pkgs.pam}/lib/libpam.so.0"'
+    '';
+    });
+
+  "pyzmq" = super."pyzmq".override (attrs: {
+    buildInputs = [
+      pkgs.czmq
+    ];
   });
 
-  python-pam = super.python-pam.override (attrs:
-    let
-      includeLibPam = pkgs.stdenv.isLinux;
-    in {
-      # TODO: johbo: Move the option up into the default.nix, we should
-      # include python-pam only on supported platforms.
-      propagatedBuildInputs = attrs.propagatedBuildInputs ++
-        pkgs.lib.optional includeLibPam [
-          pkgs.pam
-        ];
-      # TODO: johbo: Check if this can be avoided, or transform into
-      # a real patch
-      patchPhase = pkgs.lib.optionals includeLibPam ''
-        substituteInPlace pam.py \
-          --replace 'find_library("pam")' '"${pkgs.pam}/lib/libpam.so.0"'
-      '';
-    });
-
-  URLObject = super.URLObject.override (attrs: {
+  "urlobject" = super."urlobject".override (attrs: {
     meta = {
       license = {
         spdxId = "Unlicense";
@@ -195,56 +192,56 b' self: super: {'
     };
   });
 
-  docutils = super.docutils.override (attrs: {
+  "docutils" = super."docutils".override (attrs: {
     meta = {
       license = pkgs.lib.licenses.bsd2;
     };
   });
 
-  colander = super.colander.override (attrs: {
+  "colander" = super."colander".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
-  pyramid-beaker  = super.pyramid-beaker.override (attrs: {
+  "pyramid-beaker"  = super."pyramid-beaker".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
-  pyramid-mako = super.pyramid-mako.override (attrs: {
+  "pyramid-mako" = super."pyramid-mako".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
-  repoze.lru = super.repoze.lru.override (attrs: {
+  "repoze.lru" = super."repoze.lru".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
-  python-editor = super.python-editor.override (attrs: {
+  "python-editor" = super."python-editor".override (attrs: {
     meta = {
       license = pkgs.lib.licenses.asl20;
     };
   });
 
-  translationstring = super.translationstring.override (attrs: {
+  "translationstring" = super."translationstring".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
-  venusian = super.venusian.override (attrs: {
+  "venusian" = super."venusian".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
-  # Avoid that setuptools is replaced, this leads to trouble
-  # with buildPythonPackage.
-  setuptools = basePythonPackages.setuptools;
+  # Avoid that base packages screw up the build process
+  inherit (basePythonPackages)
+    setuptools;
 
 }

pytest.ini

@@ -1,6 +1,7 b''
 [pytest]
 testpaths = rhodecode
 norecursedirs = rhodecode/public rhodecode/templates tests/scripts
+cache_dir = /tmp/.pytest_cache
 
 pyramid_config = rhodecode/tests/rhodecode.ini
 vcsserver_protocol = http

release.nix

@@ -1,232 +1,21 b''
-#
-# About
-# =====
-#
-# This file defines jobs for our CI system and the attribute "build" is used
-# as the input for packaging.
-#
-#
-# CI details
-# ==========
-#
-# This file defines an attribute set of derivations. Each of these attributes is
-# then used in our CI system as one job to run. This way we keep the
-# configuration for the CI jobs as well under version control.
-#
-# Run CI jobs locally
-# -------------------
-#
-# Since it is all based on normal Nix derivations, the jobs can be tested
-# locally with a run of "nix-build" like the following example:
-#
-#    nix-build release.nix -A test-api -I vcsserver=~/rhodecode-vcsserver
-#
-# Note: Replace "~/rhodecode-vcsserver" with a path where a clone of the
-# vcsserver resides.
+# This file defines how to "build" for packaging.
 
 { pkgs ? import <nixpkgs> {}
-, doCheck ? true
+, doCheck ? false
 }:
 
 let
-
-  inherit (pkgs)
-    stdenv
-    system;
-
-  testing = import <nixpkgs/nixos/lib/testing.nix> {
-    inherit system;
-  };
-
-  runInMachine = testing.runInMachine;
-
-  sphinx = import ./docs/default.nix {};
-
-  mkDocs = kind: stdenv.mkDerivation {
-    name = kind;
-    srcs = [
-      (./. + (builtins.toPath "/${kind}"))
-      (builtins.filterSource
-        (path: type: baseNameOf path == "VERSION")
-        ./rhodecode)
-    ];
-    sourceRoot = kind;
-    buildInputs = [ sphinx ];
-    configurePhase = null;
-    buildPhase = ''
-      make SPHINXBUILD=sphinx-build html
-    '';
-    installPhase = ''
-      mkdir -p $out
-      mv _build/html $out/
-
-      mkdir -p $out/nix-support
-      echo "doc manual $out/html index.html" >> \
-        "$out/nix-support/hydra-build-products"
-    '';
-  };
-
-  enterprise = import ./default.nix {
+  enterprise_ce = import ./default.nix {
     inherit
+      doCheck
       pkgs;
 
-    # TODO: for quick local testing
-    doCheck = false;
-  };
+    # disable checkPhase for build
+    checkPhase = ''
+    '';
 
-  test-cfg = stdenv.mkDerivation {
-    name = "test-cfg";
-    unpackPhase = "true";
-    buildInputs = [
-      enterprise.src
-    ];
-    installPhase = ''
-      mkdir -p $out/etc
-      cp ${enterprise.src}/test.ini $out/etc/enterprise.ini
-      # TODO: johbo: Needed, so that the login works, this causes
-      # probably some side effects
-      substituteInPlace $out/etc/enterprise.ini --replace "is_test = True" ""
-
-      # Gevent configuration
-      cp $out/etc/enterprise.ini $out/etc/enterprise-gevent.ini;
-      cat >> $out/etc/enterprise-gevent.ini <<EOF
-
-      [server:main]
-      use = egg:gunicorn#main
-      worker_class = gevent
-      EOF
-
-      cp ${enterprise.src}/vcsserver/test.ini $out/etc/vcsserver.ini
-    '';
-  };
-
-  ac-test-drv = import ./acceptance_tests {
-    withExternals = false;
   };
 
-  # TODO: johbo: Currently abusing buildPythonPackage to make the
-  # needed environment for the ac-test tools.
-  mkAcTests = {
-    # Path to an INI file which will be used to run Enterprise.
-    #
-    # Intended usage is to provide different configuration files to
-    # run the tests against a different configuration.
-    enterpriseCfg ? "${test-cfg}/etc/enterprise.ini"
-
-    # Path to an INI file which will be used to run the VCSServer.
-  , vcsserverCfg ? "${test-cfg}/etc/vcsserver.ini"
-  }: pkgs.pythonPackages.buildPythonPackage {
-    name = "enterprise-ac-tests";
-    src = ./acceptance_tests;
-
-    buildInputs = with pkgs; [
-      curl
-      enterprise
-      ac-test-drv
-    ];
-
-    buildPhase = ''
-      cp ${enterpriseCfg} enterprise.ini
-
-      echo "Creating a fake home directory"
-      mkdir fake-home
-      export HOME=$PWD/fake-home
-
-      echo "Creating a repository directory"
-      mkdir repos
-
-      echo "Preparing the database"
-      rc-setup-app \
-        --user=admin \
-        --email=[e-mail] \
-        --password=secret \
-        --api-key=9999999999999999999999999999999999999999 \
-        --force-yes \
-        --repos=$PWD/repos \
-        enterprise.ini > /dev/null
-
-      echo "Starting rc-server"
-      vcsserver --config ${vcsserverCfg} >vcsserver.log 2>&1 &
-      rc-server enterprise.ini >rc-server.log 2>&1 &
-
-      while ! curl -f -s http://localhost:5000 > /dev/null
-      do
-          echo "Waiting for server to be ready..."
-          sleep 3
-      done
-      echo "Webserver is ready."
-
-      echo "Starting the test run"
-      py.test -c example.ini -vs --maxfail=5 tests
-
-      echo "Kill rc-server"
-      kill %2
-      kill %1
-    '';
-
-    # TODO: johbo: Use the install phase again once the normal mkDerivation
-    # can be used again.
-    postInstall = ''
-      mkdir -p $out
-      cp enterprise.ini $out
-      cp ${vcsserverCfg} $out/vcsserver.ini
-      cp rc-server.log $out
-      cp vcsserver.log $out
-
-      mkdir -p $out/nix-support
-      echo "report config $out enterprise.ini" >> $out/nix-support/hydra-build-products
-      echo "report config $out vcsserver.ini" >> $out/nix-support/hydra-build-products
-      echo "report rc-server $out rc-server.log" >> $out/nix-support/hydra-build-products
-      echo "report vcsserver $out vcsserver.log" >> $out/nix-support/hydra-build-products
-    '';
-  };
-
-  vcsserver = import <vcsserver> {
-    inherit pkgs;
-
-    # TODO: johbo: Think of a more elegant solution to this problem
-    pythonExternalOverrides = self: super: (enterprise.myPythonPackagesUnfix self);
-  };
-
-  runTests = optionString: (enterprise.override (attrs: {
-    doCheck = true;
-    name = "test-run";
-    buildInputs = attrs.buildInputs ++ [
-      vcsserver
-    ];
-    checkPhase = ''
-      py.test ${optionString} -vv -ra
-    '';
-    buildPhase = attrs.shellHook;
-    installPhase = ''
-      echo "Intentionally not installing anything"
-    '';
-    meta.description = "Enterprise test run ${optionString}";
-  }));
-
-  jobs = {
-
-    build = enterprise;
-
-    # johbo: Currently this is simply running the tests against the sources. Nicer
-    # would be to run xdist and against the installed application, so that we also
-    # cover the impact of installing the application.
-    test-api = runTests "rhodecode/api";
-    test-functional = runTests "rhodecode/tests/functional";
-    test-rest = runTests "rhodecode/tests --ignore=rhodecode/tests/functional";
-    test-full = runTests "rhodecode";
-
-    docs = mkDocs "docs";
-
-    aggregate = pkgs.releaseTools.aggregate {
-      name = "aggregated-jobs";
-      constituents = [
-        jobs.build
-        jobs.test-api
-        jobs.test-rest
-        jobs.docs
-      ];
-    };
-  };
-
-in jobs
+in {
+  build = enterprise_ce;
+}

requirements.txt

@@ -1,129 +1,129 b''
-## core
-setuptools==30.1.0
-setuptools-scm==1.15.6
+## dependencies
 
-amqp==2.2.2
+setuptools-scm==2.1.0
+amqp==2.3.1
 # not released authomatic that has updated some oauth providers
 https://code.rhodecode.com/upstream/authomatic/archive/90a9ce60cc405ae8a2bf5c3713acd5d78579a04e.tar.gz?md5=3c68720a1322b25254009518d1ff6801#egg=authomatic==0.1.0.post1
-Babel==1.3
-Beaker==1.9.1
-celery==4.1.0
-Chameleon==2.24
+babel==1.3
+beaker==1.9.1
+celery==4.1.1
+chameleon==2.24
 channelstream==0.5.2
 click==6.6
 colander==1.4.0
-configobj==5.0.6
-cssselect==1.0.1
+# our custom configobj
+https://code.rhodecode.com/upstream/configobj/archive/a11ff0a0bd4fbda9e3a91267e720f88329efb4a6.tar.gz?md5=9916c524ea11a6c418217af6b28d4b3c#egg=configobj==5.0.6
+cssselect==1.0.3
 decorator==4.1.2
-deform==2.0.4
+deform==2.0.5
 docutils==0.14.0
-dogpile.cache==0.6.4
+dogpile.cache==0.6.6
 dogpile.core==0.4.1
 ecdsa==0.13
-FormEncode==1.2.4
+formencode==1.2.4
 future==0.14.3
 futures==3.0.2
 gnureadline==6.3.8
 infrae.cache==1.0.1
-iso8601==0.1.12
+iso8601==0.1.11
 itsdangerous==0.24
-Jinja2==2.9.6
+jinja2==2.9.6
 billiard==3.5.0.3
-kombu==4.1.0
+kombu==4.2.0
 lxml==3.7.3
-Mako==1.0.7
-Markdown==2.6.11
-MarkupSafe==1.0.0
-msgpack-python==0.4.8
-MySQL-python==1.2.5
+mako==1.0.7
+markdown==2.6.11
+markupsafe==1.0.0
+msgpack-python==0.5.6
+mysql-python==1.2.5
+pymysql==0.8.1
+pyotp==2.2.6
 objgraph==3.1.1
 packaging==15.2
-Paste==2.0.3
-PasteDeploy==1.5.2
-PasteScript==2.0.2
+paste==2.0.3
+pastedeploy==1.5.2
+pastescript==2.0.2
 pathlib2==2.3.0
 peppercorn==0.5
-psutil==5.4.3
-psycopg2==2.7.3.2
+psutil==5.4.6
+psycopg2==2.7.4
 py-bcrypt==0.4
 pycrypto==2.6.1
-pycurl==7.19.5
+pycurl==7.43.0.2
 pyflakes==0.8.1
 pygments-markdown-lexer==0.1.0.dev39
-Pygments==2.2.0
+pygments==2.2.0
 pyparsing==1.5.7
 pyramid-beaker==0.8
-pyramid-debugtoolbar==4.3.0
+pyramid-debugtoolbar==4.4.0
 pyramid-jinja2==2.7
 pyramid-mako==1.0.2
-pyramid==1.9.1
+pyramid==1.9.2
 pysqlite==2.8.3
 python-dateutil
-python-ldap==2.4.45
-python-memcached==1.58
-python-pam==1.8.2
-pytz==2018.3
+python-ldap==3.1.0
+python-memcached==1.59
+python-pam==1.8.4
+pytz==2018.4
 tzlocal==1.5.1
 pyzmq==14.6.0
 py-gfm==0.1.3
 redis==2.10.6
 repoze.lru==0.7
 requests==2.9.1
-Routes==2.4.1
+routes==2.4.1
 setproctitle==1.1.10
 simplejson==3.11.1
 six==1.11.0
-SQLAlchemy==1.1.15
+sqlalchemy==1.1.18
 sshpubkeys==2.2.0
-subprocess32==3.2.7
+subprocess32==3.5.1
 supervisor==3.3.4
-Tempita==0.5.2
+tempita==0.5.2
 translationstring==1.3
 trollius==1.0.4
-urllib3==1.16
-URLObject==2.4.0
+urllib3==1.21
+urlobject==2.4.3
 venusian==1.1.0
-WebError==0.10.3
-WebHelpers2==2.0
-WebHelpers==1.3
-WebOb==1.7.4
-Whoosh==2.7.4
+weberror==0.10.3
+webhelpers2==2.0
+webhelpers==1.3
+webob==1.7.4
+whoosh==2.7.4
 wsgiref==0.1.2
 zope.cachedescriptors==4.3.1
 zope.deprecation==4.3.0
 zope.event==4.3.0
-zope.interface==4.4.3
+zope.interface==4.5.0
 
 
 # IPYTHON RENDERING
 # entrypoints backport, pypi version doesn't support egg installs
 https://code.rhodecode.com/upstream/entrypoints/archive/96e6d645684e1af3d7df5b5272f3fe85a546b233.tar.gz?md5=7db37771aea9ac9fefe093e5d6987313#egg=entrypoints==0.2.2.rhodecode-upstream1
 nbconvert==5.3.1
-bleach==2.1.2
+bleach==2.1.4
 nbformat==4.4.0
 jupyter_client==5.0.0
 
 ## cli tools
-alembic==0.9.8
+alembic==0.9.9
 invoke==0.13.0
 bumpversion==0.5.3
-transifex-client==0.12.5
 
 ## http servers
-gevent==1.2.2
+gevent==1.3.5
 greenlet==0.4.13
-gunicorn==19.7.1
+gunicorn==19.9.0
 waitress==1.1.0
-uWSGI==2.0.15
 
 ## debug
-ipdb==0.10.3
+ipdb==0.11.0
 ipython==5.1.0
-CProfileV==1.0.7
+cprofilev==1.0.7
 bottle==0.12.13
 
 ## rhodecode-tools, special case
-https://code.rhodecode.com/rhodecode-tools-ce/archive/v0.14.1.tar.gz?md5=0b9c2caad160b68889f8172ea54af7b2#egg=rhodecode-tools==0.14.1
+https://code.rhodecode.com/rhodecode-tools-ce/archive/v0.16.0.tar.gz?md5=87f81286b49156c2ac36f1a1acfa2164#egg=rhodecode-tools==0.16.0
 
 ## appenlight
 appenlight-client==0.6.25

requirements_test.txt

@@ -1,15 +1,14 b''
 # test related requirements
-pytest==3.2.5
-py==1.5.2
+pytest==3.6.0
+py==1.5.3
 pytest-cov==2.5.1
-pytest-sugar==0.9.0
-pytest-runner==3.0.0
-pytest-catchlog==1.2.2
-pytest-profiling==1.2.11
+pytest-sugar==0.9.1
+pytest-runner==4.2.0
+pytest-profiling==1.3.0
 gprof2dot==2017.9.19
-pytest-timeout==1.2.0
+pytest-timeout==1.2.1
 
 mock==1.0.1
-WebTest==2.0.29
+webtest==2.0.29
 cov-core==1.15.0
 coverage==3.7.1

rhodecode/VERSION

@@ -1,1 +1,1 b''
-4.12.4
  No newline at end of file
+4.13.0
  No newline at end of file

rhodecode/__init__.py

@@ -51,7 +51,7 b' PYRAMID_SETTINGS = {}'
 EXTENSIONS = {}
 
 __version__ = ('.'.join((str(each) for each in VERSION[:3])))
-__dbversion__ = 86  # defines current db version for migrations
+__dbversion__ = 90  # defines current db version for migrations
 __platform__ = platform.system()
 __license__ = 'AGPLv3, and Commercial License'
 __author__ = 'RhodeCode GmbH'

rhodecode/api/__init__.py

@@ -208,6 +208,7 b' def request_view(request):'
 
         # register our auth-user
         request.rpc_user = auth_u
+        request.environ['rc_auth_user_id'] = auth_u.user_id
 
         # now check if token is valid for API
         auth_token = request.rpc_api_key

rhodecode/api/tests/test_create_pull_request.py

@@ -119,6 +119,7 b' class TestCreatePullRequestApi(object):'
              'mandatory': True},
         ]
         data['reviewers'] = reviewers
+
         id_, params = build_data(
             self.apikey_regular, 'create_pull_request', **data)
         response = api_call(self.app, params)
@@ -251,8 +252,9 b' class TestCreatePullRequestApi(object):'
         id_, params = build_data(
             self.apikey_regular, 'create_pull_request', **data)
         response = api_call(self.app, params)
-        expected_message = 'The specified branch `{}` does not exist'.format(
-            branch_name)
+        expected_message = 'The specified value:{type}:`{name}` ' \
+                           'does not exist, or is not allowed.'.format(type='branch',
+                                                                       name=branch_name)
         assert_error(id_, expected_message, given=response.body)
 
     @pytest.mark.backends("git", "hg")
@@ -316,6 +318,7 b' class TestCreatePullRequestApi(object):'
         self.commit_ids = backend.create_master_repo(commits)
         self.source = backend.create_repo(heads=[source_head])
         self.target = backend.create_repo(heads=[target_head])
+
         data = {
             'source_repo': self.source.repo_name,
             'target_repo': self.target.repo_name,

rhodecode/api/tests/test_update_user.py

@@ -76,6 +76,8 b' class TestUpdateUser(object):'
             'user': ret
         }
         expected = ret
+        expected['user']['last_activity'] = response.json['result']['user'][
+            'last_activity']
         assert_ok(id_, expected, given=response.body)
 
     def test_api_update_user_by_user_id(self):
@@ -91,6 +93,8 b' class TestUpdateUser(object):'
             'user': ret
         }
         expected = ret
+        expected['user']['last_activity'] = response.json['result']['user'][
+            'last_activity']
         assert_ok(id_, expected, given=response.body)
 
     def test_api_update_user_default_user(self):

rhodecode/api/tests/test_utils.py

@@ -87,7 +87,8 b' class TestResolveRefOrError(object):'
         ref = 'ancestor:ref'
         with pytest.raises(JSONRPCError) as excinfo:
             utils.resolve_ref_or_error(ref, repo)
-        expected_message = 'The specified ancestor `ref` does not exist'
+        expected_message = (
+            'The specified value:ancestor:`ref` does not exist, or is not allowed.')
         assert excinfo.value.message == expected_message
 
     def test_branch_is_not_found(self):
@@ -99,7 +100,7 b' class TestResolveRefOrError(object):'
             with pytest.raises(JSONRPCError) as excinfo:
                 utils.resolve_ref_or_error(ref, repo)
         expected_message = (
-            'The specified branch `non-existing-one` does not exist')
+            'The specified value:branch:`non-existing-one` does not exist, or is not allowed.')
         assert excinfo.value.message == expected_message
 
     def test_bookmark_is_not_found(self):
@@ -111,7 +112,7 b' class TestResolveRefOrError(object):'
             with pytest.raises(JSONRPCError) as excinfo:
                 utils.resolve_ref_or_error(ref, repo)
         expected_message = (
-            'The specified bookmark `non-existing-one` does not exist')
+            'The specified value:bookmark:`non-existing-one` does not exist, or is not allowed.')
         assert excinfo.value.message == expected_message
 
     @pytest.mark.parametrize("ref", ['ref', '12345', 'a:b:c:d'])

rhodecode/api/utils.py

@@ -403,7 +403,7 b' def resolve_ref_or_error(ref, repo):'
         ref_hash = ref_hash or _get_ref_hash(repo, ref_type, ref_name)
     except (KeyError, ValueError):
         raise JSONRPCError(
-            'The specified {type} `{name}` does not exist'.format(
+            'The specified value:{type}:`{name}` does not exist, or is not allowed.'.format(
                 type=ref_type, name=ref_name))
 
     return ':'.join([ref_type, ref_name, ref_hash])
@@ -431,7 +431,6 b' def _get_commit_dict('
     }
 
 
-# TODO: mikhail: Think about moving this function to some library
 def _get_ref_hash(repo, type_, name):
     vcs_repo = repo.scm_instance()
     if type_ == 'branch' and vcs_repo.alias in ('hg', 'git'):

rhodecode/api/views/pull_request_api.py

@@ -284,7 +284,7 b' def merge_pull_request('
             raise JSONRPCError('userid is not the same as your user')
 
     check = MergeCheck.validate(
-        pull_request, user=apiuser, translator=request.translate)
+        pull_request, auth_user=apiuser, translator=request.translate)
     merge_possible = not check.failed
 
     if not merge_possible:
@@ -302,7 +302,7 b' def merge_pull_request('
         request.environ, repo_name=target_repo.repo_name,
         username=apiuser.username, action='push',
         scm=target_repo.repo_type)
-    merge_response = PullRequestModel().merge(
+    merge_response = PullRequestModel().merge_repo(
         pull_request, apiuser, extras=extras)
     if merge_response.executed:
         PullRequestModel().close_pull_request(
@@ -548,7 +548,8 b' def comment_pull_request('
         closing_pr=False,
         renderer=renderer,
         comment_type=comment_type,
-        resolves_comment_id=resolves_comment_id
+        resolves_comment_id=resolves_comment_id,
+        auth_user=apiuser
     )
 
     if allowed_to_change_status and status:
@@ -573,7 +574,8 b' def comment_pull_request('
 @jsonrpc_method()
 def create_pull_request(
         request, apiuser, source_repo, target_repo, source_ref, target_ref,
-        title=Optional(''), description=Optional(''), reviewers=Optional(None)):
+        title=Optional(''), description=Optional(''), description_renderer=Optional(''),
+        reviewers=Optional(None)):
     """
     Creates a new pull request.
 
@@ -598,6 +600,10 b' def create_pull_request('
     :type title: str
     :param description: Set the pull request description.
     :type description: Optional(str)
+    :type description_renderer: Optional(str)
+    :param description_renderer: Set pull request renderer for the description.
+        It should be 'rst', 'markdown' or 'plain'. If not give default
+        system renderer will be used
     :param reviewers: Set the new pull request reviewers list.
         Reviewer defined by review rules will be added automatically to the
         defined list.
@@ -607,7 +613,7 b' def create_pull_request('
             [{'username': 'nick', 'reasons': ['original author'], 'mandatory': <bool>}]
     """
 
-    source_db_repo =  get_repo_or_error(source_repo)
+    source_db_repo = get_repo_or_error(source_repo)
     target_db_repo = get_repo_or_error(target_repo)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin', 'repository.write', 'repository.read',)
@@ -678,7 +684,11 b' def create_pull_request('
             source_ref=title_source_ref,
             target=target_repo
         )
+    # fetch renderer, if set fallback to plain in case of PR
+    rc_config = SettingsModel().get_all_settings()
+    default_system_renderer = rc_config.get('rhodecode_markup_renderer', 'plain')
     description = Optional.extract(description)
+    description_renderer = Optional.extract(description_renderer) or default_system_renderer
 
     pull_request = PullRequestModel().create(
         created_by=apiuser.user_id,
@@ -690,7 +700,9 b' def create_pull_request('
         reviewers=reviewers,
         title=title,
         description=description,
+        description_renderer=description_renderer,
         reviewer_data=reviewer_rules,
+        auth_user=apiuser
     )
 
     Session().commit()
@@ -704,8 +716,8 b' def create_pull_request('
 @jsonrpc_method()
 def update_pull_request(
         request, apiuser, pullrequestid, repoid=Optional(None),
-        title=Optional(''), description=Optional(''), reviewers=Optional(None),
-        update_commits=Optional(None)):
+        title=Optional(''), description=Optional(''), description_renderer=Optional(''),
+        reviewers=Optional(None), update_commits=Optional(None)):
     """
     Updates a pull request.
 
@@ -719,6 +731,9 b' def update_pull_request('
     :type title: str
     :param description: Update pull request description.
     :type description: Optional(str)
+    :type description_renderer: Optional(str)
+    :param description_renderer: Update pull request renderer for the description.
+        It should be 'rst', 'markdown' or 'plain'
     :param reviewers: Update pull request reviewers list with new value.
     :type reviewers: Optional(list)
         Accepts username strings or objects of the format:
@@ -801,10 +816,15 b' def update_pull_request('
 
     title = Optional.extract(title)
     description = Optional.extract(description)
+    description_renderer = Optional.extract(description_renderer)
+
     if title or description:
         PullRequestModel().edit(
-            pull_request, title or pull_request.title,
-            description or pull_request.description, apiuser)
+            pull_request,
+            title or pull_request.title,
+            description or pull_request.description,
+            description_renderer or pull_request.description_renderer,
+            apiuser)
         Session().commit()
 
     commit_changes = {"added": [], "common": [], "removed": []}

rhodecode/api/views/repo_api.py

@@ -1469,7 +1469,8 b' def comment_commit('
             status_change_type=status,
             renderer=renderer,
             comment_type=comment_type,
-            resolves_comment_id=resolves_comment_id
+            resolves_comment_id=resolves_comment_id,
+            auth_user=apiuser
         )
         if status:
             # also do a status change

rhodecode/apps/_base/__init__.py

@@ -461,31 +461,38 b' class RepoRoutePredicate(object):'
     phash = text
 
     def __call__(self, info, request):
-
         if hasattr(request, 'vcs_call'):
             # skip vcs calls
             return
 
         repo_name = info['match']['repo_name']
         repo_model = repo.RepoModel()
-        by_name_match = repo_model.get_by_repo_name(repo_name, cache=True)
+
+        by_name_match = repo_model.get_by_repo_name(repo_name, cache=False)
 
-        def redirect_if_creating(db_repo):
+        def redirect_if_creating(route_info, db_repo):
+            skip_views = ['edit_repo_advanced_delete']
+            route = route_info['route']
+            # we should skip delete view so we can actually "remove" repositories
+            # if they get stuck in creating state.
+            if route.name in skip_views:
+                return
+
             if db_repo.repo_state in [repo.Repository.STATE_PENDING]:
-                raise HTTPFound(
-                    request.route_path('repo_creating',
-                                       repo_name=db_repo.repo_name))
+                repo_creating_url = request.route_path(
+                    'repo_creating', repo_name=db_repo.repo_name)
+                raise HTTPFound(repo_creating_url)
 
         if by_name_match:
             # register this as request object we can re-use later
             request.db_repo = by_name_match
-            redirect_if_creating(by_name_match)
+            redirect_if_creating(info, by_name_match)
             return True
 
         by_id_match = repo_model.get_repo_by_id(repo_name)
         if by_id_match:
             request.db_repo = by_id_match
-            redirect_if_creating(by_id_match)
+            redirect_if_creating(info, by_id_match)
             return True
 
         return False
@@ -516,7 +523,7 b' class RepoTypeRoutePredicate(object):'
         else:
             log.warning('Current view is not supported for repo type:%s',
                         rhodecode_db_repo.repo_type)
-            #
+
             # h.flash(h.literal(
             #     _('Action not supported for %s.' % rhodecode_repo.alias)),
             #     category='warning')
@@ -542,8 +549,7 b' class RepoGroupRoutePredicate(object):'
 
         repo_group_name = info['match']['repo_group_name']
         repo_group_model = repo_group.RepoGroupModel()
-        by_name_match = repo_group_model.get_by_group_name(
-            repo_group_name, cache=True)
+        by_name_match = repo_group_model.get_by_group_name(repo_group_name, cache=False)
 
         if by_name_match:
             # register this as request object we can re-use later
@@ -569,8 +575,7 b' class UserGroupRoutePredicate(object):'
 
         user_group_id = info['match']['user_group_id']
         user_group_model = user_group.UserGroup()
-        by_id_match = user_group_model.get(
-            user_group_id, cache=True)
+        by_id_match = user_group_model.get(user_group_id, cache=False)
 
         if by_id_match:
             # register this as request object we can re-use later
@@ -596,8 +601,7 b' class UserRoutePredicateBase(object):'
 
         user_id = info['match']['user_id']
         user_model = user.User()
-        by_id_match = user_model.get(
-            user_id, cache=True)
+        by_id_match = user_model.get(user_id, cache=False)
 
         if by_id_match:
             # register this as request object we can re-use later