release: Merge default into stable for release preparation

docs/release-notes/release-notes-4.23.0.rst

@@ -0,0 +1,89 b''
+|RCE| 4.23.0 |RNS|
+------------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2020-11-20
+
+
+New Features
+^^^^^^^^^^^^
+
+- Comments: introduced new draft comments.
+
+  * drafts are private to author
+  * not triggering any notifications
+  * sidebar doesn't display draft comments
+  * They are just placeholders for longer review.
+
+- Comments: when channelstream is enabled, comments are pushed live, so there's no
+  need to refresh page to see other participant comments.
+  New comments are marker in the sidebar.
+
+- Comments: multiple changes on comments navigation/display logic.
+
+  * toggle icon is smarter, open/hide windows according to actions. E.g commenting opens threads
+  * toggle are mor explicit
+  * possible to hide/show only single threads using the toggle icon.
+  * new UI for showing thread comments
+
+- Reviewers: new logic for author/commit-author rules.
+  It's not possible to define if author or commit author should be excluded, or always included in a review.
+- Reviewers: no reviewers would now allow a PR to be merged, unless review rules require some.
+  Use case is that pr can be created without review needed, maybe just for sharing, or CI checks
+- Pull requests: save permanently the state if sorting columns for pull-request grids.
+- Commit ranges: enable combined diff compare directly from range selector.
+
+
+General
+^^^^^^^
+
+- Authentication: enable custom names for auth plugins. It's possible to name the authentication
+  buttons now for SAML plugins.
+- Login: optimized UI for login/register/password reset windows.
+- Repo mapper: make it more resilient to errors, it's better it executes and skip certain
+  repositories, rather then crash whole mapper.
+- Markdown: improved styling, and fixed nl2br extensions to only do br on new elements not inline.
+- Pull requests: show pr version in the my-account and repo pr listing grids.
+- Archives: allowing to obtain archives without the commit short id in the name for
+  better automation of obtained artifacts.
+  New url flag called `?=with_hash=1` controls this
+- Error document: update info about stored exception retrieval.
+- Range diff: enable hovercards for commits in range-diff.
+
+
+Security
+^^^^^^^^
+
+
+
+Performance
+^^^^^^^^^^^
+
+- Improved logic of repo archive, now it's much faster to run archiver as VCSServer
+  communication was removed, and job is delegated to VCSServer itself.
+- Improved VCSServer startup times.
+- Notifications: skip double rendering just to generate email title/desc.
+  We'll re-use those now for better performance of creating notifications.
+- App: improve logging, and remove DB calls on app startup.
+
+
+Fixes
+^^^^^
+
+- Login/register: fixed header width problem on mobile devices
+- Exception tracker: don't fail on empty request in context of celery app for example.
+- Exceptions: improved reporting of unhandled vcsserver exceptions.
+- Sidebar: fixed refresh of TODOs url.
+- Remap-rescan: fixes #5636 initial rescan problem.
+- API: fixed SVN raw diff export. The API method was inconsistent, and used different logic.
+  Now it shares the same code as raw-diff from web-ui.
+
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- Scheduled feature release.
+  Please note that now the reviewers logic changed a bit, it's possible to create a pull request
+  Without any reviewers initially, and such pull request doesn't need to have an approval for merging.

.bumpversion.cfg

@@ -1,5 +1,5 b''
 [bumpversion]
-current_version = 4.22.0
+current_version = 4.23.0
 message = release: Bump version {current_version} to {new_version}
 
 [bumpversion:file:rhodecode/VERSION]

.release.cfg

@@ -1,33 +1,28 b''
 [DEFAULT]
 done = false
 
 [task:bump_version]
 done = true
 
 [task:rc_tools_pinned]
-done = true
 
 [task:fixes_on_stable]
-done = true
 
 [task:pip2nix_generated]
-done = true
 
 [task:changelog_updated]
-done = true
 
 [task:generate_api_docs]
-done = true
+
+[task:updated_translation]
 
 [release]
-state = prepared
-version = 4.22.0
-
-[task:updated_translation]
+state = in_progress
+version = 4.23.0
 
 [task:generate_js_routes]
 
 [task:updated_trial_license]
 
 [task:generate_oss_licenses]
 

docs/release-notes/release-notes.rst

@@ -1,148 +1,149 b''
 .. _rhodecode-release-notes-ref:
 
 Release Notes
 =============
 
 |RCE| 4.x Versions
 ------------------
 
 .. toctree::
    :maxdepth: 1
 
+   release-notes-4.23.0.rst
    release-notes-4.22.0.rst
    release-notes-4.21.0.rst
    release-notes-4.20.1.rst
    release-notes-4.20.0.rst
    release-notes-4.19.3.rst
    release-notes-4.19.2.rst
    release-notes-4.19.1.rst
    release-notes-4.19.0.rst
    release-notes-4.18.3.rst
    release-notes-4.18.2.rst
    release-notes-4.18.1.rst
    release-notes-4.18.0.rst
    release-notes-4.17.4.rst
    release-notes-4.17.3.rst
    release-notes-4.17.2.rst
    release-notes-4.17.1.rst
    release-notes-4.17.0.rst
    release-notes-4.16.2.rst
    release-notes-4.16.1.rst
    release-notes-4.16.0.rst
    release-notes-4.15.2.rst
    release-notes-4.15.1.rst
    release-notes-4.15.0.rst
    release-notes-4.14.1.rst
    release-notes-4.14.0.rst
    release-notes-4.13.3.rst
    release-notes-4.13.2.rst
    release-notes-4.13.1.rst
    release-notes-4.13.0.rst
    release-notes-4.12.4.rst
    release-notes-4.12.3.rst
    release-notes-4.12.2.rst
    release-notes-4.12.1.rst
    release-notes-4.12.0.rst
    release-notes-4.11.6.rst
    release-notes-4.11.5.rst
    release-notes-4.11.4.rst
    release-notes-4.11.3.rst
    release-notes-4.11.2.rst
    release-notes-4.11.1.rst
    release-notes-4.11.0.rst
    release-notes-4.10.6.rst
    release-notes-4.10.5.rst
    release-notes-4.10.4.rst
    release-notes-4.10.3.rst
    release-notes-4.10.2.rst
    release-notes-4.10.1.rst
    release-notes-4.10.0.rst
    release-notes-4.9.1.rst
    release-notes-4.9.0.rst
    release-notes-4.8.0.rst
    release-notes-4.7.2.rst
    release-notes-4.7.1.rst
    release-notes-4.7.0.rst
    release-notes-4.6.1.rst
    release-notes-4.6.0.rst
    release-notes-4.5.2.rst
    release-notes-4.5.1.rst
    release-notes-4.5.0.rst
    release-notes-4.4.2.rst
    release-notes-4.4.1.rst
    release-notes-4.4.0.rst
    release-notes-4.3.1.rst
    release-notes-4.3.0.rst
    release-notes-4.2.1.rst
    release-notes-4.2.0.rst
    release-notes-4.1.2.rst
    release-notes-4.1.1.rst
    release-notes-4.1.0.rst
    release-notes-4.0.1.rst
    release-notes-4.0.0.rst
 
 |RCE| 3.x Versions
 ------------------
 
 .. toctree::
    :maxdepth: 1
 
    release-notes-3.8.4.rst
    release-notes-3.8.3.rst
    release-notes-3.8.2.rst
    release-notes-3.8.1.rst
    release-notes-3.8.0.rst
    release-notes-3.7.1.rst
    release-notes-3.7.0.rst
    release-notes-3.6.1.rst
    release-notes-3.6.0.rst
    release-notes-3.5.2.rst
    release-notes-3.5.1.rst
    release-notes-3.5.0.rst
    release-notes-3.4.1.rst
    release-notes-3.4.0.rst
    release-notes-3.3.4.rst
    release-notes-3.3.3.rst
    release-notes-3.3.2.rst
    release-notes-3.3.1.rst
    release-notes-3.3.0.rst
    release-notes-3.2.3.rst
    release-notes-3.2.2.rst
    release-notes-3.2.1.rst
    release-notes-3.2.0.rst
    release-notes-3.1.1.rst
    release-notes-3.1.0.rst
    release-notes-3.0.2.rst
    release-notes-3.0.1.rst
    release-notes-3.0.0.rst
 
 |RCE| 2.x Versions
 ------------------
 
 .. toctree::
    :maxdepth: 1
 
    release-notes-2.2.8.rst
    release-notes-2.2.7.rst
    release-notes-2.2.6.rst
    release-notes-2.2.5.rst
    release-notes-2.2.4.rst
    release-notes-2.2.3.rst
    release-notes-2.2.2.rst
    release-notes-2.2.1.rst
    release-notes-2.2.0.rst
    release-notes-2.1.0.rst
    release-notes-2.0.2.rst
    release-notes-2.0.1.rst
    release-notes-2.0.0.rst
 
 |RCE| 1.x Versions
 ------------------
 
 .. toctree::
    :maxdepth: 1
 
    release-notes-1.7.2.rst
    release-notes-1.7.1.rst
    release-notes-1.7.0.rst
    release-notes-1.6.0.rst

pkgs/python-packages.nix

@@ -1,2509 +1,2509 b''
 # Generated by pip2nix 0.8.0.dev1
 # See https://github.com/johbo/pip2nix
 
 { pkgs, fetchurl, fetchgit, fetchhg }:
 
 self: super: {
   "alembic" = super.buildPythonPackage {
     name = "alembic-1.4.2";
     doCheck = false;
     propagatedBuildInputs = [
       self."sqlalchemy"
       self."mako"
       self."python-editor"
       self."python-dateutil"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/60/1e/cabc75a189de0fbb2841d0975243e59bde8b7822bacbb95008ac6fe9ad47/alembic-1.4.2.tar.gz";
       sha256 = "1gsdrzx9h7wfva200qvvsc9sn4w79mk2vs0bbnzjhxi1jw2b0nh3";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "amqp" = super.buildPythonPackage {
     name = "amqp-2.5.2";
     doCheck = false;
     propagatedBuildInputs = [
       self."vine"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/92/1d/433541994a5a69f4ad2fff39746ddbb0bdedb0ea0d85673eb0db68a7edd9/amqp-2.5.2.tar.gz";
       sha256 = "13dhhfxjrqcjybnq4zahg92mydhpg2l76nxcmq7d560687wsxwbp";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "apispec" = super.buildPythonPackage {
     name = "apispec-1.0.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."PyYAML"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/67/15/346c04988dd67d36007e28145504c520491930c878b1f484a97b27a8f497/apispec-1.0.0.tar.gz";
       sha256 = "1712w1anvqrvadjjpvai84vbaygaxabd3zz5lxihdzwzs4gvi9sp";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "appenlight-client" = super.buildPythonPackage {
     name = "appenlight-client-0.6.26";
     doCheck = false;
     propagatedBuildInputs = [
       self."webob"
       self."requests"
       self."six"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/2e/56/418fc10379b96e795ee39a15e69a730c222818af04c3821fa354eaa859ec/appenlight_client-0.6.26.tar.gz";
       sha256 = "0s9xw3sb8s3pk73k78nnq4jil3q4mk6bczfa1fmgfx61kdxl2712";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "asn1crypto" = super.buildPythonPackage {
     name = "asn1crypto-0.24.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/fc/f1/8db7daa71f414ddabfa056c4ef792e1461ff655c2ae2928a2b675bfed6b4/asn1crypto-0.24.0.tar.gz";
       sha256 = "0jaf8rf9dx1lf23xfv2cdd5h52f1qr3w8k63985bc35g3d220p4x";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "atomicwrites" = super.buildPythonPackage {
     name = "atomicwrites-1.3.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ec/0f/cd484ac8820fed363b374af30049adc8fd13065720fd4f4c6be8a2309da7/atomicwrites-1.3.0.tar.gz";
       sha256 = "19ngcscdf3jsqmpcxn6zl5b6anmsajb6izp1smcd1n02midl9abm";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "attrs" = super.buildPythonPackage {
     name = "attrs-19.3.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/98/c3/2c227e66b5e896e15ccdae2e00bbc69aa46e9a8ce8869cc5fa96310bf612/attrs-19.3.0.tar.gz";
       sha256 = "0wky4h28n7xnr6xv69p9z6kv8bzn50d10c3drmd9ds8gawbcxdzp";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "babel" = super.buildPythonPackage {
     name = "babel-1.3";
     doCheck = false;
     propagatedBuildInputs = [
       self."pytz"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/33/27/e3978243a03a76398c384c83f7ca879bc6e8f1511233a621fcada135606e/Babel-1.3.tar.gz";
       sha256 = "0bnin777lc53nxd1hp3apq410jj5wx92n08h7h4izpl4f4sx00lz";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "backports.shutil-get-terminal-size" = super.buildPythonPackage {
     name = "backports.shutil-get-terminal-size-1.0.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ec/9c/368086faa9c016efce5da3e0e13ba392c9db79e3ab740b763fe28620b18b/backports.shutil_get_terminal_size-1.0.0.tar.gz";
       sha256 = "107cmn7g3jnbkp826zlj8rrj19fam301qvaqf0f3905f5217lgki";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "beaker" = super.buildPythonPackage {
     name = "beaker-1.9.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."funcsigs"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ca/14/a626188d0d0c7b55dd7cf1902046c2743bd392a7078bb53073e13280eb1e/Beaker-1.9.1.tar.gz";
       sha256 = "08arsn61r255lhz6hcpn2lsiqpg30clla805ysx06wmbhvb6w9rj";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "beautifulsoup4" = super.buildPythonPackage {
     name = "beautifulsoup4-4.6.3";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/88/df/86bffad6309f74f3ff85ea69344a078fc30003270c8df6894fca7a3c72ff/beautifulsoup4-4.6.3.tar.gz";
       sha256 = "041dhalzjciw6qyzzq7a2k4h1yvyk76xigp35hv5ibnn448ydy4h";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "billiard" = super.buildPythonPackage {
     name = "billiard-3.6.1.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/68/1d/2aea8fbb0b1e1260a8a2e77352de2983d36d7ac01207cf14c2b9c6cc860e/billiard-3.6.1.0.tar.gz";
       sha256 = "09hzy3aqi7visy4vmf4xiish61n0rq5nd3iwjydydps8yrs9r05q";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "bleach" = super.buildPythonPackage {
     name = "bleach-3.1.3";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."webencodings"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/de/09/5267f8577a92487ed43bc694476c4629c6eca2e3c93fcf690a26bfe39e1d/bleach-3.1.3.tar.gz";
       sha256 = "0al437aw4p2xp83az5hhlrp913nsf0cg6kg4qj3fjhv4wakxipzq";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "bumpversion" = super.buildPythonPackage {
     name = "bumpversion-0.5.3";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/14/41/8c9da3549f8e00c84f0432c3a8cf8ed6898374714676aab91501d48760db/bumpversion-0.5.3.tar.gz";
       sha256 = "0zn7694yfipxg35ikkfh7kvgl2fissha3dnqad2c5bvsvmrwhi37";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "cachetools" = super.buildPythonPackage {
     name = "cachetools-3.1.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ae/37/7fd45996b19200e0cb2027a0b6bef4636951c4ea111bfad36c71287247f6/cachetools-3.1.1.tar.gz";
       sha256 = "16m69l6n6y1r1y7cklm92rr7v69ldig2n3lbl3j323w5jz7d78lf";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "celery" = super.buildPythonPackage {
     name = "celery-4.3.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."pytz"
       self."billiard"
       self."kombu"
       self."vine"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/a2/4b/d020836f751617e907e84753a41c92231cd4b673ff991b8ee9da52361323/celery-4.3.0.tar.gz";
       sha256 = "1y8y0gbgkwimpxqnxq2rm5qz2vy01fvjiybnpm00y5rzd2m34iac";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "certifi" = super.buildPythonPackage {
     name = "certifi-2020.4.5.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/b8/e2/a3a86a67c3fc8249ed305fc7b7d290ebe5e4d46ad45573884761ef4dea7b/certifi-2020.4.5.1.tar.gz";
       sha256 = "06b5gfs7wmmipln8f3z928d2mmx2j4b3x7pnqmj6cvmyfh8v7z2i";
     };
     meta = {
       license = [ pkgs.lib.licenses.mpl20 { fullName = "Mozilla Public License 2.0 (MPL 2.0)"; } ];
     };
   };
   "cffi" = super.buildPythonPackage {
     name = "cffi-1.12.3";
     doCheck = false;
     propagatedBuildInputs = [
       self."pycparser"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/93/1a/ab8c62b5838722f29f3daffcc8d4bd61844aa9b5f437341cc890ceee483b/cffi-1.12.3.tar.gz";
       sha256 = "0x075521fxwv0mfp4cqzk7lvmw4n94bjw601qkcv314z5s182704";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "chameleon" = super.buildPythonPackage {
     name = "chameleon-2.24";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/5a/9e/637379ffa13c5172b5c0e704833ffea6bf51cec7567f93fd6e903d53ed74/Chameleon-2.24.tar.gz";
       sha256 = "0ykqr7syxfa6h9adjfnsv1gdsca2xzm22vmic8859n0f0j09abj5";
     };
     meta = {
       license = [ { fullName = "BSD-like (http://repoze.org/license.html)"; } ];
     };
   };
   "channelstream" = super.buildPythonPackage {
     name = "channelstream-0.6.14";
     doCheck = false;
     propagatedBuildInputs = [
       self."gevent"
       self."ws4py"
       self."marshmallow"
       self."python-dateutil"
       self."pyramid"
       self."pyramid-jinja2"
       self."pyramid-apispec"
       self."itsdangerous"
       self."requests"
       self."six"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/d4/2d/86d6757ccd06ce673ee224123471da3d45251d061da7c580bfc259bad853/channelstream-0.6.14.tar.gz";
       sha256 = "0qgy5j3rj6c8cslzidh32glhkrhbbdxjc008y69v8a0y3zyaz2d3";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "chardet" = super.buildPythonPackage {
     name = "chardet-3.0.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/fc/bb/a5768c230f9ddb03acc9ef3f0d4a3cf93462473795d18e9535498c8f929d/chardet-3.0.4.tar.gz";
       sha256 = "1bpalpia6r5x1kknbk11p1fzph56fmmnp405ds8icksd3knr5aw4";
     };
     meta = {
       license = [ { fullName = "LGPL"; } { fullName = "GNU Library or Lesser General Public License (LGPL)"; } ];
     };
   };
   "click" = super.buildPythonPackage {
     name = "click-7.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/f8/5c/f60e9d8a1e77005f664b76ff8aeaee5bc05d0a91798afd7f53fc998dbc47/Click-7.0.tar.gz";
       sha256 = "1mzjixd4vjbjvzb6vylki9w1556a9qmdh35kzmq6cign46av952v";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "colander" = super.buildPythonPackage {
     name = "colander-1.7.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."translationstring"
       self."iso8601"
       self."enum34"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/db/e4/74ab06f54211917b41865cafc987ce511e35503de48da9bfe9358a1bdc3e/colander-1.7.0.tar.gz";
       sha256 = "1wl1bqab307lbbcjx81i28s3yl6dlm4rf15fxawkjb6j48x1cn6p";
     };
     meta = {
       license = [ { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "configobj" = super.buildPythonPackage {
     name = "configobj-5.0.6";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
     ];
     src = fetchurl {
       url = "https://code.rhodecode.com/upstream/configobj/artifacts/download/0-012de99a-b1e1-4f64-a5c0-07a98a41b324.tar.gz?md5=6a513f51fe04b2c18cf84c1395a7c626";
       sha256 = "0kqfrdfr14mw8yd8qwq14dv2xghpkjmd3yjsy8dfcbvpcc17xnxp";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "configparser" = super.buildPythonPackage {
     name = "configparser-4.0.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/16/4f/48975536bd488d3a272549eb795ac4a13a5f7fcdc8995def77fbef3532ee/configparser-4.0.2.tar.gz";
       sha256 = "1priacxym85yjcf68hh38w55nqswaxp71ryjyfdk222kg9l85ln7";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "contextlib2" = super.buildPythonPackage {
     name = "contextlib2-0.6.0.post1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/02/54/669207eb72e3d8ae8b38aa1f0703ee87a0e9f88f30d3c0a47bebdb6de242/contextlib2-0.6.0.post1.tar.gz";
       sha256 = "0bhnr2ac7wy5l85ji909gyljyk85n92w8pdvslmrvc8qih4r1x01";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   "cov-core" = super.buildPythonPackage {
     name = "cov-core-1.15.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."coverage"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/4b/87/13e75a47b4ba1be06f29f6d807ca99638bedc6b57fa491cd3de891ca2923/cov-core-1.15.0.tar.gz";
       sha256 = "0k3np9ymh06yv1ib96sb6wfsxjkqhmik8qfsn119vnhga9ywc52a";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "coverage" = super.buildPythonPackage {
     name = "coverage-4.5.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/85/d5/818d0e603685c4a613d56f065a721013e942088047ff1027a632948bdae6/coverage-4.5.4.tar.gz";
       sha256 = "0p0j4di6h8k6ica7jwwj09azdcg4ycxq60i9qsskmsg94cd9yzg0";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "cryptography" = super.buildPythonPackage {
     name = "cryptography-2.6.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."asn1crypto"
       self."six"
       self."cffi"
       self."enum34"
       self."ipaddress"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/07/ca/bc827c5e55918ad223d59d299fff92f3563476c3b00d0a9157d9c0217449/cryptography-2.6.1.tar.gz";
       sha256 = "19iwz5avym5zl6jrrrkym1rdaa9h61j20ph4cswsqgv8xg5j3j16";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "BSD or Apache License, Version 2.0"; } pkgs.lib.licenses.asl20 ];
     };
   };
   "cssselect" = super.buildPythonPackage {
     name = "cssselect-1.0.3";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/52/ea/f31e1d2e9eb130fda2a631e22eac369dc644e8807345fbed5113f2d6f92b/cssselect-1.0.3.tar.gz";
       sha256 = "011jqa2jhmydhi0iz4v1w3cr540z5zas8g2bw8brdw4s4b2qnv86";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "cssutils" = super.buildPythonPackage {
     name = "cssutils-1.0.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/5c/0b/c5f29d29c037e97043770b5e7c740b6252993e4b57f029b3cd03c78ddfec/cssutils-1.0.2.tar.gz";
       sha256 = "1bxchrbqzapwijap0yhlxdil1w9bmwvgx77aizlkhc2mcxjg1z52";
     };
     meta = {
       license = [ { fullName = "GNU Library or Lesser General Public License (LGPL)"; } { fullName = "LGPL 2.1 or later, see also http://cthedot.de/cssutils/"; } ];
     };
   };
   "decorator" = super.buildPythonPackage {
     name = "decorator-4.1.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/bb/e0/f6e41e9091e130bf16d4437dabbac3993908e4d6485ecbc985ef1352db94/decorator-4.1.2.tar.gz";
       sha256 = "1d8npb11kxyi36mrvjdpcjij76l5zfyrz2f820brf0l0rcw4vdkw";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "new BSD License"; } ];
     };
   };
   "deform" = super.buildPythonPackage {
     name = "deform-2.0.8";
     doCheck = false;
     propagatedBuildInputs = [
       self."chameleon"
       self."colander"
       self."iso8601"
       self."peppercorn"
       self."translationstring"
       self."zope.deprecation"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/21/d0/45fdf891a82722c02fc2da319cf2d1ae6b5abf9e470ad3762135a895a868/deform-2.0.8.tar.gz";
       sha256 = "0wbjv98sib96649aqaygzxnrkclyy50qij2rha6fn1i4c86bfdl9";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "defusedxml" = super.buildPythonPackage {
     name = "defusedxml-0.6.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/a4/5f/f8aa58ca0cf01cbcee728abc9d88bfeb74e95e6cb4334cfd5bed5673ea77/defusedxml-0.6.0.tar.gz";
       sha256 = "1xbp8fivl3wlbyg2jrvs4lalaqv1xp9a9f29p75wdx2s2d6h717n";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   "dm.xmlsec.binding" = super.buildPythonPackage {
     name = "dm.xmlsec.binding-1.3.7";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
       self."lxml"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/2c/9e/7651982d50252692991acdae614af821fd6c79bc8dcd598ad71d55be8fc7/dm.xmlsec.binding-1.3.7.tar.gz";
       sha256 = "03jjjscx1pz2nc0dwiw9nia02qbz1c6f0f9zkyr8fmvys2n5jkb3";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "docutils" = super.buildPythonPackage {
     name = "docutils-0.16";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/2f/e0/3d435b34abd2d62e8206171892f174b180cd37b09d57b924ca5c2ef2219d/docutils-0.16.tar.gz";
       sha256 = "1z3qliszqca9m719q3qhdkh0ghh90g500avzdgi7pl77x5h3mpn2";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal pkgs.lib.licenses.publicDomain pkgs.lib.licenses.gpl1 { fullName = "public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt)"; } pkgs.lib.licenses.psfl ];
     };
   };
   "dogpile.cache" = super.buildPythonPackage {
     name = "dogpile.cache-0.9.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."decorator"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ac/6a/9ac405686a94b7f009a20a50070a5786b0e1aedc707b88d40d0c4b51a82e/dogpile.cache-0.9.0.tar.gz";
       sha256 = "0sr1fn6b4k5bh0cscd9yi8csqxvj4ngzildav58x5p694mc86j5k";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "dogpile.core" = super.buildPythonPackage {
     name = "dogpile.core-0.4.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/0e/77/e72abc04c22aedf874301861e5c1e761231c288b5de369c18be8f4b5c9bb/dogpile.core-0.4.1.tar.gz";
       sha256 = "0xpdvg4kr1isfkrh1rfsh7za4q5a5s6l2kf9wpvndbwf3aqjyrdy";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "ecdsa" = super.buildPythonPackage {
     name = "ecdsa-0.13.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/51/76/139bf6e9b7b6684d5891212cdbd9e0739f2bfc03f380a1a6ffa700f392ac/ecdsa-0.13.2.tar.gz";
       sha256 = "116qaq7bh4lcynzi613960jhsnn19v0kmsqwahiwjfj14gx4y0sw";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "elasticsearch" = super.buildPythonPackage {
     name = "elasticsearch-6.3.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."urllib3"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/9d/ce/c4664e8380e379a9402ecfbaf158e56396da90d520daba21cfa840e0eb71/elasticsearch-6.3.1.tar.gz";
       sha256 = "12y93v0yn7a4xmf969239g8gb3l4cdkclfpbk1qc8hx5qkymrnma";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "elasticsearch-dsl" = super.buildPythonPackage {
     name = "elasticsearch-dsl-6.3.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."python-dateutil"
       self."elasticsearch"
       self."ipaddress"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/4c/0d/1549f50c591db6bb4e66cbcc8d34a6e537c3d89aa426b167c244fd46420a/elasticsearch-dsl-6.3.1.tar.gz";
       sha256 = "1gh8a0shqi105k325hgwb9avrpdjh0mc6mxwfg9ba7g6lssb702z";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "elasticsearch1" = super.buildPythonPackage {
     name = "elasticsearch1-1.10.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."urllib3"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/a6/eb/73e75f9681fa71e3157b8ee878534235d57f24ee64f0e77f8d995fb57076/elasticsearch1-1.10.0.tar.gz";
       sha256 = "0g89444kd5zwql4vbvyrmi2m6l6dcj6ga98j4hqxyyyz6z20aki2";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "elasticsearch1-dsl" = super.buildPythonPackage {
     name = "elasticsearch1-dsl-0.0.12";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."python-dateutil"
       self."elasticsearch1"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/eb/9d/785342775cb10eddc9b8d7457d618a423b4f0b89d8b2b2d1bc27190d71db/elasticsearch1-dsl-0.0.12.tar.gz";
       sha256 = "0ig1ly39v93hba0z975wnhbmzwj28w6w1sqlr2g7cn5spp732bhk";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "elasticsearch2" = super.buildPythonPackage {
     name = "elasticsearch2-2.5.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."urllib3"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/f6/09/f9b24aa6b1120bea371cd57ef6f57c7694cf16660469456a8be6c2bdbe22/elasticsearch2-2.5.1.tar.gz";
       sha256 = "19k2znpjfyp0hrq73cz7pjyj289040xpsxsm0xhh4jfh6y551g7k";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "entrypoints" = super.buildPythonPackage {
     name = "entrypoints-0.2.2";
     doCheck = false;
     propagatedBuildInputs = [
       self."configparser"
     ];
     src = fetchurl {
       url = "https://code.rhodecode.com/upstream/entrypoints/artifacts/download/0-8e9ee9e4-c4db-409c-b07e-81568fd1832d.tar.gz?md5=3a027b8ff1d257b91fe257de6c43357d";
       sha256 = "0qih72n2myclanplqipqxpgpj9d2yhff1pz5d02zq1cfqyd173w5";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "enum34" = super.buildPythonPackage {
     name = "enum34-1.1.10";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/11/c4/2da1f4952ba476677a42f25cd32ab8aaf0e1c0d0e00b89822b835c7e654c/enum34-1.1.10.tar.gz";
       sha256 = "0j7ji699fwswm4vg6w1v07fkbf8dkzdm6gfh88jvs5nqgr3sgrnc";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "formencode" = super.buildPythonPackage {
     name = "formencode-1.2.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/8e/59/0174271a6f004512e0201188593e6d319db139d14cb7490e488bbb078015/FormEncode-1.2.4.tar.gz";
       sha256 = "1fgy04sdy4yry5xcjls3x3xy30dqwj58ycnkndim819jx0788w42";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   "funcsigs" = super.buildPythonPackage {
     name = "funcsigs-1.0.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/94/4a/db842e7a0545de1cdb0439bb80e6e42dfe82aaeaadd4072f2263a4fbed23/funcsigs-1.0.2.tar.gz";
       sha256 = "0l4g5818ffyfmfs1a924811azhjj8ax9xd1cffr1mzd3ycn0zfx7";
     };
     meta = {
       license = [ { fullName = "ASL"; } pkgs.lib.licenses.asl20 ];
     };
   };
   "functools32" = super.buildPythonPackage {
     name = "functools32-3.2.3.post2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/c5/60/6ac26ad05857c601308d8fb9e87fa36d0ebf889423f47c3502ef034365db/functools32-3.2.3-2.tar.gz";
       sha256 = "0v8ya0b58x47wp216n1zamimv4iw57cxz3xxhzix52jkw3xks9gn";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   "future" = super.buildPythonPackage {
     name = "future-0.14.3";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/83/80/8ef3a11a15f8eaafafa0937b20c1b3f73527e69ab6b3fa1cf94a5a96aabb/future-0.14.3.tar.gz";
       sha256 = "1savk7jx7hal032f522c5ajhh8fra6gmnadrj9adv5qxi18pv1b2";
     };
     meta = {
       license = [ { fullName = "OSI Approved"; } pkgs.lib.licenses.mit ];
     };
   };
   "futures" = super.buildPythonPackage {
     name = "futures-3.0.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/f8/e7/fc0fcbeb9193ba2d4de00b065e7fd5aecd0679e93ce95a07322b2b1434f4/futures-3.0.2.tar.gz";
       sha256 = "0mz2pbgxbc2nbib1szifi07whjbfs4r02pv2z390z7p410awjgyw";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "gevent" = super.buildPythonPackage {
     name = "gevent-1.5.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."greenlet"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/5a/79/2c63d385d017b5dd7d70983a463dfd25befae70c824fedb857df6e72eff2/gevent-1.5.0.tar.gz";
       sha256 = "0aac3d4vhv5n4rsb6cqzq0d1xx9immqz4fmpddw35yxkwdc450dj";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "gnureadline" = super.buildPythonPackage {
     name = "gnureadline-6.3.8";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/50/64/86085c823cd78f9df9d8e33dce0baa71618016f8860460b82cf6610e1eb3/gnureadline-6.3.8.tar.gz";
       sha256 = "0ddhj98x2nv45iz4aadk4b9m0b1kpsn1xhcbypn5cd556knhiqjq";
     };
     meta = {
       license = [ { fullName = "GNU General Public License v3 (GPLv3)"; } pkgs.lib.licenses.gpl1 ];
     };
   };
   "gprof2dot" = super.buildPythonPackage {
     name = "gprof2dot-2017.9.19";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/9d/36/f977122502979f3dfb50704979c9ed70e6b620787942b089bf1af15f5aba/gprof2dot-2017.9.19.tar.gz";
       sha256 = "17ih23ld2nzgc3xwgbay911l6lh96jp1zshmskm17n1gg2i7mg6f";
     };
     meta = {
       license = [ { fullName = "GNU Lesser General Public License v3 or later (LGPLv3+)"; } { fullName = "LGPL"; } ];
     };
   };
   "greenlet" = super.buildPythonPackage {
     name = "greenlet-0.4.15";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/f8/e8/b30ae23b45f69aa3f024b46064c0ac8e5fcb4f22ace0dca8d6f9c8bbe5e7/greenlet-0.4.15.tar.gz";
       sha256 = "1g4g1wwc472ds89zmqlpyan3fbnzpa8qm48z3z1y6mlk44z485ll";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "gunicorn" = super.buildPythonPackage {
     name = "gunicorn-19.9.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/47/52/68ba8e5e8ba251e54006a49441f7ccabca83b6bef5aedacb4890596c7911/gunicorn-19.9.0.tar.gz";
       sha256 = "1wzlf4xmn6qjirh5w81l6i6kqjnab1n1qqkh7zsj1yb6gh4n49ps";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "hupper" = super.buildPythonPackage {
     name = "hupper-1.10.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/41/24/ea90fef04706e54bd1635c05c50dc9cf87cda543c59303a03e7aa7dda0ce/hupper-1.10.2.tar.gz";
       sha256 = "0am0p6g5cz6xmcaf04xq8q6dzdd9qz0phj6gcmpsckf2mcyza61q";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "idna" = super.buildPythonPackage {
     name = "idna-2.8";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ad/13/eb56951b6f7950cadb579ca166e448ba77f9d24efc03edd7e55fa57d04b7/idna-2.8.tar.gz";
       sha256 = "01rlkigdxg17sf9yar1jl8n18ls59367wqh59hnawlyg53vb6my3";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "BSD-like"; } ];
     };
   };
   "importlib-metadata" = super.buildPythonPackage {
     name = "importlib-metadata-1.6.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."zipp"
       self."pathlib2"
       self."contextlib2"
       self."configparser"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/b4/1b/baab42e3cd64c9d5caac25a9d6c054f8324cdc38975a44d600569f1f7158/importlib_metadata-1.6.0.tar.gz";
       sha256 = "07icyggasn38yv2swdrd8z6i0plazmc9adavsdkbqqj91j53ll9l";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "infrae.cache" = super.buildPythonPackage {
     name = "infrae.cache-1.0.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."beaker"
       self."repoze.lru"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/bb/f0/e7d5e984cf6592fd2807dc7bc44a93f9d18e04e6a61f87fdfb2622422d74/infrae.cache-1.0.1.tar.gz";
       sha256 = "1dvqsjn8vw253wz9d1pz17j79mf4bs53dvp2qxck2qdp1am1njw4";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "invoke" = super.buildPythonPackage {
     name = "invoke-0.13.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/47/bf/d07ef52fa1ac645468858bbac7cb95b246a972a045e821493d17d89c81be/invoke-0.13.0.tar.gz";
       sha256 = "0794vhgxfmkh0vzkkg5cfv1w82g3jc3xr18wim29far9qpx9468s";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "ipaddress" = super.buildPythonPackage {
     name = "ipaddress-1.0.23";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/b9/9a/3e9da40ea28b8210dd6504d3fe9fe7e013b62bf45902b458d1cdc3c34ed9/ipaddress-1.0.23.tar.gz";
       sha256 = "1qp743h30s04m3cg3yk3fycad930jv17q7dsslj4mfw0jlvf1y5p";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   "ipdb" = super.buildPythonPackage {
     name = "ipdb-0.13.2";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
       self."ipython"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/2c/bb/a3e1a441719ebd75c6dac8170d3ddba884b7ee8a5c0f9aefa7297386627a/ipdb-0.13.2.tar.gz";
       sha256 = "0jcd849rx30y3wcgzsqbn06v0yjlzvb9x3076q0yxpycdwm1ryvp";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "ipython" = super.buildPythonPackage {
     name = "ipython-5.1.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
       self."decorator"
       self."pickleshare"
       self."simplegeneric"
       self."traitlets"
       self."prompt-toolkit"
       self."pygments"
       self."pexpect"
       self."backports.shutil-get-terminal-size"
       self."pathlib2"
       self."pexpect"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/89/63/a9292f7cd9d0090a0f995e1167f3f17d5889dcbc9a175261719c513b9848/ipython-5.1.0.tar.gz";
       sha256 = "0qdrf6aj9kvjczd5chj1my8y2iq09am9l8bb2a1334a52d76kx3y";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "ipython-genutils" = super.buildPythonPackage {
     name = "ipython-genutils-0.2.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/e8/69/fbeffffc05236398ebfcfb512b6d2511c622871dca1746361006da310399/ipython_genutils-0.2.0.tar.gz";
       sha256 = "1a4bc9y8hnvq6cp08qs4mckgm6i6ajpndp4g496rvvzcfmp12bpb";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "iso8601" = super.buildPythonPackage {
     name = "iso8601-0.1.12";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/45/13/3db24895497345fb44c4248c08b16da34a9eb02643cea2754b21b5ed08b0/iso8601-0.1.12.tar.gz";
       sha256 = "10nyvvnrhw2w3p09v1ica4lgj6f4g9j3kkfx17qmraiq3w7b5i29";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "isodate" = super.buildPythonPackage {
     name = "isodate-0.6.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/b1/80/fb8c13a4cd38eb5021dc3741a9e588e4d1de88d895c1910c6fc8a08b7a70/isodate-0.6.0.tar.gz";
       sha256 = "1n7jkz68kk5pwni540pr5zdh99bf6ywydk1p5pdrqisrawylldif";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "itsdangerous" = super.buildPythonPackage {
     name = "itsdangerous-1.1.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/68/1a/f27de07a8a304ad5fa817bbe383d1238ac4396da447fa11ed937039fa04b/itsdangerous-1.1.0.tar.gz";
       sha256 = "068zpbksq5q2z4dckh2k1zbcq43ay74ylqn77rni797j0wyh66rj";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "jinja2" = super.buildPythonPackage {
     name = "jinja2-2.9.6";
     doCheck = false;
     propagatedBuildInputs = [
       self."markupsafe"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/90/61/f820ff0076a2599dd39406dcb858ecb239438c02ce706c8e91131ab9c7f1/Jinja2-2.9.6.tar.gz";
       sha256 = "1zzrkywhziqffrzks14kzixz7nd4yh2vc0fb04a68vfd2ai03anx";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "jsonschema" = super.buildPythonPackage {
     name = "jsonschema-2.6.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."functools32"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/58/b9/171dbb07e18c6346090a37f03c7e74410a1a56123f847efed59af260a298/jsonschema-2.6.0.tar.gz";
       sha256 = "00kf3zmpp9ya4sydffpifn0j0mzm342a2vzh82p6r0vh10cg7xbg";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "jupyter-client" = super.buildPythonPackage {
     name = "jupyter-client-5.0.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."traitlets"
       self."jupyter-core"
       self."pyzmq"
       self."python-dateutil"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/e5/6f/65412ed462202b90134b7e761b0b7e7f949e07a549c1755475333727b3d0/jupyter_client-5.0.0.tar.gz";
       sha256 = "0nxw4rqk4wsjhc87gjqd7pv89cb9dnimcfnmcmp85bmrvv1gjri7";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "jupyter-core" = super.buildPythonPackage {
     name = "jupyter-core-4.5.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."traitlets"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/4a/de/ff4ca734656d17ebe0450807b59d728f45277e2e7f4b82bc9aae6cb82961/jupyter_core-4.5.0.tar.gz";
       sha256 = "1xr4pbghwk5hayn5wwnhb7z95380r45p79gf5if5pi1akwg7qvic";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "kombu" = super.buildPythonPackage {
     name = "kombu-4.6.6";
     doCheck = false;
     propagatedBuildInputs = [
       self."amqp"
       self."importlib-metadata"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/20/e6/bc2d9affba6138a1dc143f77fef253e9e08e238fa7c0688d917c09005e96/kombu-4.6.6.tar.gz";
       sha256 = "11mxpcy8mg1l35bgbhba70v29bydr2hrhdbdlb4lg98m3m5vaq0p";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "lxml" = super.buildPythonPackage {
     name = "lxml-4.2.5";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/4b/20/ddf5eb3bd5c57582d2b4652b4bbcf8da301bdfe5d805cb94e805f4d7464d/lxml-4.2.5.tar.gz";
       sha256 = "0zw0y9hs0nflxhl9cs6ipwwh53szi3w2x06wl0k9cylyqac0cwin";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "mako" = super.buildPythonPackage {
     name = "mako-1.1.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."markupsafe"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/b0/3c/8dcd6883d009f7cae0f3157fb53e9afb05a0d3d33b3db1268ec2e6f4a56b/Mako-1.1.0.tar.gz";
       sha256 = "0jqa3qfpykyn4fmkn0kh6043sfls7br8i2bsdbccazcvk9cijsd3";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "markdown" = super.buildPythonPackage {
     name = "markdown-2.6.11";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/b3/73/fc5c850f44af5889192dff783b7b0d8f3fe8d30b65c8e3f78f8f0265fecf/Markdown-2.6.11.tar.gz";
       sha256 = "108g80ryzykh8bj0i7jfp71510wrcixdi771lf2asyghgyf8cmm8";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "markupsafe" = super.buildPythonPackage {
     name = "markupsafe-1.1.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/b9/2e/64db92e53b86efccfaea71321f597fa2e1b2bd3853d8ce658568f7a13094/MarkupSafe-1.1.1.tar.gz";
       sha256 = "0sqipg4fk7xbixqd8kq6rlkxj664d157bdwbh93farcphf92x1r9";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal pkgs.lib.licenses.bsd3 ];
     };
   };
   "marshmallow" = super.buildPythonPackage {
     name = "marshmallow-2.18.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ad/0b/5799965d1c6d5f608d684e2c0dce8a828e0309a3bfe8327d9418a89f591c/marshmallow-2.18.0.tar.gz";
       sha256 = "1g0aafpjn7yaxq06yndy8c7rs9n42adxkqq1ayhlr869pr06d3lm";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "mistune" = super.buildPythonPackage {
     name = "mistune-0.8.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/2d/a4/509f6e7783ddd35482feda27bc7f72e65b5e7dc910eca4ab2164daf9c577/mistune-0.8.4.tar.gz";
       sha256 = "0vkmsh0x480rni51lhyvigfdf06b9247z868pk3bal1wnnfl58sr";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "mock" = super.buildPythonPackage {
     name = "mock-3.0.5";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."funcsigs"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/2e/ab/4fe657d78b270aa6a32f027849513b829b41b0f28d9d8d7f8c3d29ea559a/mock-3.0.5.tar.gz";
       sha256 = "1hrp6j0yrx2xzylfv02qa8kph661m6yq4p0mc8fnimch9j4psrc3";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "OSI Approved :: BSD License"; } ];
     };
   };
   "more-itertools" = super.buildPythonPackage {
     name = "more-itertools-5.0.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/dd/26/30fc0d541d9fdf55faf5ba4b0fd68f81d5bd2447579224820ad525934178/more-itertools-5.0.0.tar.gz";
       sha256 = "1r12cm6mcdwdzz7d47a6g4l437xsvapdlgyhqay3i2nrlv03da9q";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "msgpack-python" = super.buildPythonPackage {
     name = "msgpack-python-0.5.6";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/8a/20/6eca772d1a5830336f84aca1d8198e5a3f4715cd1c7fc36d3cc7f7185091/msgpack-python-0.5.6.tar.gz";
       sha256 = "16wh8qgybmfh4pjp8vfv78mdlkxfmcasg78lzlnm6nslsfkci31p";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "mysql-python" = super.buildPythonPackage {
     name = "mysql-python-1.2.5";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/a5/e9/51b544da85a36a68debe7a7091f068d802fc515a3a202652828c73453cad/MySQL-python-1.2.5.zip";
       sha256 = "0x0c2jg0bb3pp84njaqiic050qkyd7ymwhfvhipnimg58yv40441";
     };
     meta = {
       license = [ pkgs.lib.licenses.gpl1 ];
     };
   };
   "nbconvert" = super.buildPythonPackage {
     name = "nbconvert-5.3.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."mistune"
       self."jinja2"
       self."pygments"
       self."traitlets"
       self."jupyter-core"
       self."nbformat"
       self."entrypoints"
       self."bleach"
       self."pandocfilters"
       self."testpath"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/b9/a4/d0a0938ad6f5eeb4dea4e73d255c617ef94b0b2849d51194c9bbdb838412/nbconvert-5.3.1.tar.gz";
       sha256 = "1f9dkvpx186xjm4xab0qbph588mncp4vqk3fmxrsnqs43mks9c8j";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "nbformat" = super.buildPythonPackage {
     name = "nbformat-4.4.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."ipython-genutils"
       self."traitlets"
       self."jsonschema"
       self."jupyter-core"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/6e/0e/160754f7ae3e984863f585a3743b0ed1702043a81245907c8fae2d537155/nbformat-4.4.0.tar.gz";
       sha256 = "00nlf08h8yc4q73nphfvfhxrcnilaqanb8z0mdy6nxk0vzq4wjgp";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "packaging" = super.buildPythonPackage {
     name = "packaging-20.3";
     doCheck = false;
     propagatedBuildInputs = [
       self."pyparsing"
       self."six"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/65/37/83e3f492eb52d771e2820e88105f605335553fe10422cba9d256faeb1702/packaging-20.3.tar.gz";
       sha256 = "18xpablq278janh03bai9xd4kz9b0yfp6vflazn725ns9x3jna9w";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "BSD or Apache License, Version 2.0"; } pkgs.lib.licenses.asl20 ];
     };
   };
   "pandocfilters" = super.buildPythonPackage {
     name = "pandocfilters-1.4.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/4c/ea/236e2584af67bb6df960832731a6e5325fd4441de001767da328c33368ce/pandocfilters-1.4.2.tar.gz";
       sha256 = "1a8d9b7s48gmq9zj0pmbyv2sivn5i7m6mybgpkk4jm5vd7hp1pdk";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "paste" = super.buildPythonPackage {
     name = "paste-3.4.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/79/4a/45821b71dd40000507549afd1491546afad8279c0a87527c88776a794158/Paste-3.4.0.tar.gz";
       sha256 = "16sichvhyci1gaarkjs35mai8vphh7b244qm14hj1isw38nx4c03";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pastedeploy" = super.buildPythonPackage {
     name = "pastedeploy-2.1.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/c4/e9/972a1c20318b3ae9edcab11a6cef64308fbae5d0d45ab52c6f8b2b8f35b8/PasteDeploy-2.1.0.tar.gz";
       sha256 = "16qsq5y6mryslmbp5pn35x4z8z3ndp5rpgl42h226879nrw9hmg7";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pastescript" = super.buildPythonPackage {
     name = "pastescript-3.2.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."paste"
       self."pastedeploy"
       self."six"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ff/47/45c6f5a3cb8f5abf786fea98dbb8d02400a55768a9b623afb7df12346c61/PasteScript-3.2.0.tar.gz";
       sha256 = "1b3jq7xh383nvrrlblk05m37345bv97xrhx77wshllba3h7mq3wv";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pathlib2" = super.buildPythonPackage {
     name = "pathlib2-2.3.5";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."scandir"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/94/d8/65c86584e7e97ef824a1845c72bbe95d79f5b306364fa778a3c3e401b309/pathlib2-2.3.5.tar.gz";
       sha256 = "0s4qa8c082fdkb17izh4mfgwrjd1n5pya18wvrbwqdvvb5xs9nbc";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "peppercorn" = super.buildPythonPackage {
     name = "peppercorn-0.6";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/e4/77/93085de7108cdf1a0b092ff443872a8f9442c736d7ddebdf2f27627935f4/peppercorn-0.6.tar.gz";
       sha256 = "1ip4bfwcpwkq9hz2dai14k2cyabvwrnvcvrcmzxmqm04g8fnimwn";
     };
     meta = {
       license = [ { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "pexpect" = super.buildPythonPackage {
     name = "pexpect-4.8.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."ptyprocess"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/e5/9b/ff402e0e930e70467a7178abb7c128709a30dfb22d8777c043e501bc1b10/pexpect-4.8.0.tar.gz";
       sha256 = "032cg337h8awydgypz6f4wx848lw8dyrj4zy988x0lyib4ws8rgw";
     };
     meta = {
       license = [ pkgs.lib.licenses.isc { fullName = "ISC License (ISCL)"; } ];
     };
   };
   "pickleshare" = super.buildPythonPackage {
     name = "pickleshare-0.7.5";
     doCheck = false;
     propagatedBuildInputs = [
       self."pathlib2"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/d8/b6/df3c1c9b616e9c0edbc4fbab6ddd09df9535849c64ba51fcb6531c32d4d8/pickleshare-0.7.5.tar.gz";
       sha256 = "1jmghg3c53yp1i8cm6pcrm280ayi8621rwyav9fac7awjr3kss47";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "plaster" = super.buildPythonPackage {
     name = "plaster-1.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/37/e1/56d04382d718d32751017d32f351214384e529b794084eee20bb52405563/plaster-1.0.tar.gz";
       sha256 = "1hy8k0nv2mxq94y5aysk6hjk9ryb4bsd13g83m60hcyzxz3wflc3";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "plaster-pastedeploy" = super.buildPythonPackage {
     name = "plaster-pastedeploy-0.7";
     doCheck = false;
     propagatedBuildInputs = [
       self."pastedeploy"
       self."plaster"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/99/69/2d3bc33091249266a1bd3cf24499e40ab31d54dffb4a7d76fe647950b98c/plaster_pastedeploy-0.7.tar.gz";
       sha256 = "1zg7gcsvc1kzay1ry5p699rg2qavfsxqwl17mqxzr0gzw6j9679r";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pluggy" = super.buildPythonPackage {
     name = "pluggy-0.13.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."importlib-metadata"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/f8/04/7a8542bed4b16a65c2714bf76cf5a0b026157da7f75e87cc88774aa10b14/pluggy-0.13.1.tar.gz";
       sha256 = "1c35qyhvy27q9ih9n899f3h4sdnpgq027dbiilly2qb5cvgarchm";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "premailer" = super.buildPythonPackage {
     name = "premailer-3.6.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."lxml"
       self."cssselect"
       self."cssutils"
       self."requests"
       self."cachetools"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/62/da/2f43cdf9d3d79c80c4856a12389a1f257d65fe9ccc44bc6b4383c8a18e33/premailer-3.6.1.tar.gz";
       sha256 = "08pshx7a110k4ll20x0xhpvyn3kkipkrbgxjjn7ncdxs54ihdhgw";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl { fullName = "Python"; } ];
     };
   };
   "prompt-toolkit" = super.buildPythonPackage {
     name = "prompt-toolkit-1.0.18";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."wcwidth"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/c5/64/c170e5b1913b540bf0c8ab7676b21fdd1d25b65ddeb10025c6ca43cccd4c/prompt_toolkit-1.0.18.tar.gz";
       sha256 = "09h1153wgr5x2ny7ds0w2m81n3bb9j8hjb8sjfnrg506r01clkyx";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "psutil" = super.buildPythonPackage {
     name = "psutil-5.7.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/c4/b8/3512f0e93e0db23a71d82485ba256071ebef99b227351f0f5540f744af41/psutil-5.7.0.tar.gz";
       sha256 = "03jykdi3dgf1cdal9bv4fq9zjvzj9l9bs99gi5ar81sdl5nc2pk8";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "psycopg2" = super.buildPythonPackage {
     name = "psycopg2-2.8.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/84/d7/6a93c99b5ba4d4d22daa3928b983cec66df4536ca50b22ce5dcac65e4e71/psycopg2-2.8.4.tar.gz";
       sha256 = "1djvh98pi4hjd8rxbq8qzc63bg8v78k33yg6pl99wak61b6fb67q";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 { fullName = "GNU Library or Lesser General Public License (LGPL)"; } { fullName = "LGPL with exceptions or ZPL"; } ];
     };
   };
   "ptyprocess" = super.buildPythonPackage {
     name = "ptyprocess-0.6.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/7d/2d/e4b8733cf79b7309d84c9081a4ab558c89d8c89da5961bf4ddb050ca1ce0/ptyprocess-0.6.0.tar.gz";
       sha256 = "1h4lcd3w5nrxnsk436ar7fwkiy5rfn5wj2xwy9l0r4mdqnf2jgwj";
     };
     meta = {
       license = [  ];
     };
   };
   "py" = super.buildPythonPackage {
     name = "py-1.8.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/f1/5a/87ca5909f400a2de1561f1648883af74345fe96349f34f737cdfc94eba8c/py-1.8.0.tar.gz";
       sha256 = "0lsy1gajva083pzc7csj1cvbmminb7b4l6a0prdzyb3fd829nqyw";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "py-bcrypt" = super.buildPythonPackage {
     name = "py-bcrypt-0.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/68/b1/1c3068c5c4d2e35c48b38dcc865301ebfdf45f54507086ac65ced1fd3b3d/py-bcrypt-0.4.tar.gz";
       sha256 = "0y6smdggwi5s72v6p1nn53dg6w05hna3d264cq6kas0lap73p8az";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "py-gfm" = super.buildPythonPackage {
     name = "py-gfm-0.1.4";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
       self."markdown"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/06/ee/004a03a1d92bb386dae44f6dd087db541bc5093374f1637d4d4ae5596cc2/py-gfm-0.1.4.tar.gz";
       sha256 = "0zip06g2isivx8fzgqd4n9qzsa22c25jas1rsb7m2rnjg72m0rzg";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "pyasn1" = super.buildPythonPackage {
     name = "pyasn1-0.4.8";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/a4/db/fffec68299e6d7bad3d504147f9094830b704527a7fc098b721d38cc7fa7/pyasn1-0.4.8.tar.gz";
       sha256 = "1fnhbi3rmk47l9851gbik0flfr64vs5j0hbqx24cafjap6gprxxf";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "pyasn1-modules" = super.buildPythonPackage {
     name = "pyasn1-modules-0.2.6";
     doCheck = false;
     propagatedBuildInputs = [
       self."pyasn1"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/f1/a9/a1ef72a0e43feff643cf0130a08123dea76205e7a0dda37e3efb5f054a31/pyasn1-modules-0.2.6.tar.gz";
       sha256 = "08hph9j1r018drnrny29l7dl2q0cin78csswrhwrh8jmq61pmha3";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal pkgs.lib.licenses.bsd2 ];
     };
   };
   "pycparser" = super.buildPythonPackage {
     name = "pycparser-2.20";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/0f/86/e19659527668d70be91d0369aeaa055b4eb396b0f387a4f92293a20035bd/pycparser-2.20.tar.gz";
       sha256 = "1w0m3xvlrzq4lkbvd1ngfm8mdw64r1yxy6n7djlw6qj5d0km6ird";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "pycrypto" = super.buildPythonPackage {
     name = "pycrypto-2.6.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/60/db/645aa9af249f059cc3a368b118de33889219e0362141e75d4eaf6f80f163/pycrypto-2.6.1.tar.gz";
       sha256 = "0g0ayql5b9mkjam8hym6zyg6bv77lbh66rv1fyvgqb17kfc1xkpj";
     };
     meta = {
       license = [ pkgs.lib.licenses.publicDomain ];
     };
   };
   "pycurl" = super.buildPythonPackage {
     name = "pycurl-7.43.0.3";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ac/b3/0f3979633b7890bab6098d84c84467030b807a1e2b31f5d30103af5a71ca/pycurl-7.43.0.3.tar.gz";
       sha256 = "13nsvqhvnmnvfk75s8iynqsgszyv06cjp4drd3psi7zpbh63623g";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit { fullName = "LGPL/MIT"; } { fullName = "GNU Library or Lesser General Public License (LGPL)"; } ];
     };
   };
   "pygments" = super.buildPythonPackage {
     name = "pygments-2.4.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/7e/ae/26808275fc76bf2832deb10d3a3ed3107bc4de01b85dcccbe525f2cd6d1e/Pygments-2.4.2.tar.gz";
       sha256 = "15v2sqm5g12bqa0c7wikfh9ck2nl97ayizy1hpqhmws5gqalq748";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "pymysql" = super.buildPythonPackage {
     name = "pymysql-0.8.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/44/39/6bcb83cae0095a31b6be4511707fdf2009d3e29903a55a0494d3a9a2fac0/PyMySQL-0.8.1.tar.gz";
       sha256 = "0a96crz55bw4h6myh833skrli7b0ck89m3x673y2z2ryy7zrpq9l";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pyotp" = super.buildPythonPackage {
     name = "pyotp-2.3.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/f7/15/395c4945ea6bc37e8811280bb675615cb4c2b2c1cd70bdc43329da91a386/pyotp-2.3.0.tar.gz";
       sha256 = "18d13ikra1iq0xyfqfm72zhgwxi2qi9ps6z1a6zmqp4qrn57wlzw";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pyparsing" = super.buildPythonPackage {
     name = "pyparsing-2.4.7";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/c1/47/dfc9c342c9842bbe0036c7f763d2d6686bcf5eb1808ba3e170afdb282210/pyparsing-2.4.7.tar.gz";
       sha256 = "1hgc8qrbq1ymxbwfbjghv01fm3fbpjwpjwi0bcailxxzhf3yq0y2";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pyramid" = super.buildPythonPackage {
     name = "pyramid-1.10.4";
     doCheck = false;
     propagatedBuildInputs = [
       self."hupper"
       self."plaster"
       self."plaster-pastedeploy"
       self."setuptools"
       self."translationstring"
       self."venusian"
       self."webob"
       self."zope.deprecation"
       self."zope.interface"
       self."repoze.lru"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/c2/43/1ae701c9c6bb3a434358e678a5e72c96e8aa55cf4cb1d2fa2041b5dd38b7/pyramid-1.10.4.tar.gz";
       sha256 = "0rkxs1ajycg2zh1c94xlmls56mx5m161sn8112skj0amza6cn36q";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "pyramid-debugtoolbar" = super.buildPythonPackage {
     name = "pyramid-debugtoolbar-4.6.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."pyramid"
       self."pyramid-mako"
       self."repoze.lru"
       self."pygments"
       self."ipaddress"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/99/f6/b8603f82c18275be293921bc3a2184205056ca505747bf64ab8a0c08e124/pyramid_debugtoolbar-4.6.1.tar.gz";
       sha256 = "185z7q8n959ga5331iczwra2iljwkidfx4qn6bbd7vm3rm4w6llv";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "pyramid-jinja2" = super.buildPythonPackage {
     name = "pyramid-jinja2-2.7";
     doCheck = false;
     propagatedBuildInputs = [
       self."pyramid"
       self."zope.deprecation"
       self."jinja2"
       self."markupsafe"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/d8/80/d60a7233823de22ce77bd864a8a83736a1fe8b49884b08303a2e68b2c853/pyramid_jinja2-2.7.tar.gz";
       sha256 = "1sz5s0pp5jqhf4w22w9527yz8hgdi4mhr6apd6vw1gm5clghh8aw";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "pyramid-apispec" = super.buildPythonPackage {
     name = "pyramid-apispec-0.3.2";
     doCheck = false;
     propagatedBuildInputs = [
       self."apispec"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/2a/30/1dea5d81ea635449572ba60ec3148310d75ae4530c3c695f54b0991bb8c7/pyramid_apispec-0.3.2.tar.gz";
       sha256 = "0ffrcqp9dkykivhfcq0v9lgy6w0qhwl6x78925vfjmayly9r8da0";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "pyramid-mailer" = super.buildPythonPackage {
     name = "pyramid-mailer-0.15.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."pyramid"
       self."repoze.sendmail"
       self."transaction"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/a0/f2/6febf5459dff4d7e653314d575469ad2e11b9d2af2c3606360e1c67202f2/pyramid_mailer-0.15.1.tar.gz";
       sha256 = "16vg8jb203jgb7b0hd6wllfqvp542qh2ry1gjai2m6qpv5agy2pc";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "pyramid-mako" = super.buildPythonPackage {
     name = "pyramid-mako-1.1.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."pyramid"
       self."mako"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/63/7b/5e2af68f675071a6bad148c1c393928f0ef5fcd94e95cbf53b89d6471a83/pyramid_mako-1.1.0.tar.gz";
       sha256 = "1qj0m091mnii86j2q1d82yir22nha361rvhclvg3s70z8iiwhrh0";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "pysqlite" = super.buildPythonPackage {
     name = "pysqlite-2.8.3";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/42/02/981b6703e3c83c5b25a829c6e77aad059f9481b0bbacb47e6e8ca12bd731/pysqlite-2.8.3.tar.gz";
       sha256 = "1424gwq9sil2ffmnizk60q36vydkv8rxs6m7xs987kz8cdc37lqp";
     };
     meta = {
       license = [ { fullName = "zlib/libpng License"; } { fullName = "zlib/libpng license"; } ];
     };
   };
   "pytest" = super.buildPythonPackage {
     name = "pytest-4.6.5";
     doCheck = false;
     propagatedBuildInputs = [
       self."py"
       self."six"
       self."packaging"
       self."attrs"
       self."atomicwrites"
       self."pluggy"
       self."importlib-metadata"
       self."wcwidth"
       self."funcsigs"
       self."pathlib2"
       self."more-itertools"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/2a/c6/1d1f32f6a5009900521b12e6560fb6b7245b0d4bc3fb771acd63d10e30e1/pytest-4.6.5.tar.gz";
       sha256 = "0iykwwfp4h181nd7rsihh2120b0rkawlw7rvbl19sgfspncr3hwg";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pytest-cov" = super.buildPythonPackage {
     name = "pytest-cov-2.7.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."pytest"
       self."coverage"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/bb/0f/3db7ff86801883b21d5353b258c994b1b8e2abbc804e2273b8d0fd19004b/pytest-cov-2.7.1.tar.gz";
       sha256 = "0filvmmyqm715azsl09ql8hy2x7h286n6d8z5x42a1wpvvys83p0";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal pkgs.lib.licenses.mit ];
     };
   };
   "pytest-profiling" = super.buildPythonPackage {
     name = "pytest-profiling-1.7.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."pytest"
       self."gprof2dot"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/39/70/22a4b33739f07f1732a63e33bbfbf68e0fa58cfba9d200e76d01921eddbf/pytest-profiling-1.7.0.tar.gz";
       sha256 = "0abz9gi26jpcfdzgsvwad91555lpgdc8kbymicmms8k2fqa8z4wk";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pytest-runner" = super.buildPythonPackage {
     name = "pytest-runner-5.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/d9/6d/4b41a74b31720e25abd4799be72d54811da4b4d0233e38b75864dcc1f7ad/pytest-runner-5.1.tar.gz";
       sha256 = "0ykfcnpp8c22winj63qzc07l5axwlc9ikl8vn05sc32gv3417815";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pytest-sugar" = super.buildPythonPackage {
     name = "pytest-sugar-0.9.2";
     doCheck = false;
     propagatedBuildInputs = [
       self."pytest"
       self."termcolor"
       self."packaging"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/55/59/f02f78d1c80f7e03e23177f60624c8106d4f23d124c921df103f65692464/pytest-sugar-0.9.2.tar.gz";
       sha256 = "1asq7yc4g8bx2sn7yy974mhc9ywvaihasjab4inkirdwn9s7mn7w";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "pytest-timeout" = super.buildPythonPackage {
     name = "pytest-timeout-1.3.3";
     doCheck = false;
     propagatedBuildInputs = [
       self."pytest"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/13/48/7a166eaa29c1dca6cc253e3ba5773ff2e4aa4f567c1ea3905808e95ac5c1/pytest-timeout-1.3.3.tar.gz";
       sha256 = "1cczcjhw4xx5sjkhxlhc5c1bkr7x6fcyx12wrnvwfckshdvblc2a";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit { fullName = "DFSG approved"; } ];
     };
   };
   "python-dateutil" = super.buildPythonPackage {
     name = "python-dateutil-2.8.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/be/ed/5bbc91f03fa4c839c4c7360375da77f9659af5f7086b7a7bdda65771c8e0/python-dateutil-2.8.1.tar.gz";
       sha256 = "0g42w7k5007iv9dam6gnja2ry8ydwirh99mgdll35s12pyfzxsvk";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal pkgs.lib.licenses.asl20 { fullName = "Dual License"; } ];
     };
   };
   "python-editor" = super.buildPythonPackage {
     name = "python-editor-1.0.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/0a/85/78f4a216d28343a67b7397c99825cff336330893f00601443f7c7b2f2234/python-editor-1.0.4.tar.gz";
       sha256 = "0yrjh8w72ivqxi4i7xsg5b1vz15x8fg51xra7c3bgfyxqnyadzai";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 { fullName = "Apache"; } ];
     };
   };
   "python-ldap" = super.buildPythonPackage {
     name = "python-ldap-3.2.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."pyasn1"
       self."pyasn1-modules"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ea/93/596f875e003c770447f4b99267820a0c769dd2dc3ae3ed19afe460fcbad0/python-ldap-3.2.0.tar.gz";
       sha256 = "13nvrhp85yr0jyxixcjj012iw8l9wynxxlykm9j3alss6waln73x";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   "python-memcached" = super.buildPythonPackage {
     name = "python-memcached-1.59";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/90/59/5faf6e3cd8a568dd4f737ddae4f2e54204fd8c51f90bf8df99aca6c22318/python-memcached-1.59.tar.gz";
       sha256 = "0kvyapavbirk2x3n1jx4yb9nyigrj1s3x15nm3qhpvhkpqvqdqm2";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   "python-pam" = super.buildPythonPackage {
     name = "python-pam-1.8.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/01/16/544d01cae9f28e0292dbd092b6b8b0bf222b528f362ee768a5bed2140111/python-pam-1.8.4.tar.gz";
       sha256 = "16whhc0vr7gxsbzvsnq65nq8fs3wwmx755cavm8kkczdkz4djmn8";
     };
     meta = {
       license = [ { fullName = "License :: OSI Approved :: MIT License"; } pkgs.lib.licenses.mit ];
     };
   };
   "python-saml" = super.buildPythonPackage {
     name = "python-saml-2.4.2";
     doCheck = false;
     propagatedBuildInputs = [
       self."dm.xmlsec.binding"
       self."isodate"
       self."defusedxml"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/79/a8/a6611017e0883102fd5e2b73c9d90691b8134e38247c04ee1531d3dc647c/python-saml-2.4.2.tar.gz";
       sha256 = "0dls4hwvf13yg7x5yfjrghbywg8g38vn5vr0rsf70hli3ydbfm43";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pytz" = super.buildPythonPackage {
     name = "pytz-2019.3";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/82/c3/534ddba230bd4fbbd3b7a3d35f3341d014cca213f369a9940925e7e5f691/pytz-2019.3.tar.gz";
       sha256 = "1ghrk1wg45d3nymj7bf4zj03n3bh64xmczhk4pfi577hdkdhcb5h";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pyzmq" = super.buildPythonPackage {
     name = "pyzmq-14.6.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/8a/3b/5463d5a9d712cd8bbdac335daece0d69f6a6792da4e3dd89956c0db4e4e6/pyzmq-14.6.0.tar.gz";
       sha256 = "1frmbjykvhmdg64g7sn20c9fpamrsfxwci1nhhg8q7jgz5pq0ikp";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "LGPL+BSD"; } { fullName = "GNU Library or Lesser General Public License (LGPL)"; } ];
     };
   };
   "PyYAML" = super.buildPythonPackage {
     name = "PyYAML-5.3.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/64/c2/b80047c7ac2478f9501676c988a5411ed5572f35d1beff9cae07d321512c/PyYAML-5.3.1.tar.gz";
       sha256 = "0pb4zvkfxfijkpgd1b86xjsqql97ssf1knbd1v53wkg1qm9cgsmq";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "regex" = super.buildPythonPackage {
     name = "regex-2020.9.27";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/93/8c/17f45cdfb39b13d4b5f909e4b4c2917abcbdef9c0036919a0399769148cf/regex-2020.9.27.tar.gz";
       sha256 = "179ngfzwbsjvn5vhyzdahvmg0f7acahkwwy9bpjy1pv08bm2mwx6";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   "redis" = super.buildPythonPackage {
     name = "redis-3.4.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ef/2e/2c0f59891db7db087a7eeaa79bc7c7f2c039e71a2b5b0a41391e9d462926/redis-3.4.1.tar.gz";
       sha256 = "07yaj0j9fs7xdkg5bg926fa990khyigjbp31si8ai20vj8sv7kqd";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "repoze.lru" = super.buildPythonPackage {
     name = "repoze.lru-0.7";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/12/bc/595a77c4b5e204847fdf19268314ef59c85193a9dc9f83630fc459c0fee5/repoze.lru-0.7.tar.gz";
       sha256 = "0xzz1aw2smy8hdszrq8yhnklx6w1r1mf55061kalw3iq35gafa84";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "repoze.sendmail" = super.buildPythonPackage {
     name = "repoze.sendmail-4.4.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
       self."zope.interface"
       self."transaction"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/12/4e/8ef1fd5c42765d712427b9c391419a77bd48877886d2cbc5e9f23c8cad9b/repoze.sendmail-4.4.1.tar.gz";
       sha256 = "096ln02jr2afk7ab9j2czxqv2ryqq7m86ah572nqplx52iws73ks";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "requests" = super.buildPythonPackage {
     name = "requests-2.22.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."chardet"
       self."idna"
       self."urllib3"
       self."certifi"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/01/62/ddcf76d1d19885e8579acb1b1df26a852b03472c0e46d2b959a714c90608/requests-2.22.0.tar.gz";
       sha256 = "1d5ybh11jr5sm7xp6mz8fyc7vrp4syifds91m7sj60xalal0gq0i";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "rhodecode-enterprise-ce" = super.buildPythonPackage {
-    name = "rhodecode-enterprise-ce-4.22.0";
+    name = "rhodecode-enterprise-ce-4.23.0";
     buildInputs = [
       self."pytest"
       self."py"
       self."pytest-cov"
       self."pytest-sugar"
       self."pytest-runner"
       self."pytest-profiling"
       self."pytest-timeout"
       self."gprof2dot"
       self."mock"
       self."cov-core"
       self."coverage"
       self."webtest"
       self."beautifulsoup4"
       self."configobj"
     ];
     doCheck = true;
     propagatedBuildInputs = [
       self."amqp"
       self."babel"
       self."beaker"
       self."bleach"
       self."celery"
       self."channelstream"
       self."click"
       self."colander"
       self."configobj"
       self."cssselect"
       self."cryptography"
       self."decorator"
       self."deform"
       self."docutils"
       self."dogpile.cache"
       self."dogpile.core"
       self."formencode"
       self."future"
       self."futures"
       self."infrae.cache"
       self."iso8601"
       self."itsdangerous"
       self."kombu"
       self."lxml"
       self."mako"
       self."markdown"
       self."markupsafe"
       self."msgpack-python"
       self."pyotp"
       self."packaging"
       self."pathlib2"
       self."paste"
       self."pastedeploy"
       self."pastescript"
       self."peppercorn"
       self."premailer"
       self."psutil"
       self."py-bcrypt"
       self."pycurl"
       self."pycrypto"
       self."pygments"
       self."pyparsing"
       self."pyramid-debugtoolbar"
       self."pyramid-mako"
       self."pyramid"
       self."pyramid-mailer"
       self."python-dateutil"
       self."python-ldap"
       self."python-memcached"
       self."python-pam"
       self."python-saml"
       self."pytz"
       self."tzlocal"
       self."pyzmq"
       self."py-gfm"
       self."regex"
       self."redis"
       self."repoze.lru"
       self."requests"
       self."routes"
       self."simplejson"
       self."six"
       self."sqlalchemy"
       self."sshpubkeys"
       self."subprocess32"
       self."supervisor"
       self."translationstring"
       self."urllib3"
       self."urlobject"
       self."venusian"
       self."weberror"
       self."webhelpers2"
       self."webob"
       self."whoosh"
       self."wsgiref"
       self."zope.cachedescriptors"
       self."zope.deprecation"
       self."zope.event"
       self."zope.interface"
       self."mysql-python"
       self."pymysql"
       self."pysqlite"
       self."psycopg2"
       self."nbconvert"
       self."nbformat"
       self."jupyter-client"
       self."jupyter-core"
       self."alembic"
       self."invoke"
       self."bumpversion"
       self."gevent"
       self."greenlet"
       self."gunicorn"
       self."waitress"
       self."ipdb"
       self."ipython"
       self."rhodecode-tools"
       self."appenlight-client"
       self."pytest"
       self."py"
       self."pytest-cov"
       self."pytest-sugar"
       self."pytest-runner"
       self."pytest-profiling"
       self."pytest-timeout"
       self."gprof2dot"
       self."mock"
       self."cov-core"
       self."coverage"
       self."webtest"
       self."beautifulsoup4"
     ];
     src = ./.;
     meta = {
       license = [ { fullName = "Affero GNU General Public License v3 or later (AGPLv3+)"; } { fullName = "AGPLv3, and Commercial License"; } ];
     };
   };
   "rhodecode-tools" = super.buildPythonPackage {
     name = "rhodecode-tools-1.4.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."click"
       self."future"
       self."six"
       self."mako"
       self."markupsafe"
       self."requests"
       self."urllib3"
       self."whoosh"
       self."elasticsearch"
       self."elasticsearch-dsl"
       self."elasticsearch2"
       self."elasticsearch1-dsl"
     ];
     src = fetchurl {
       url = "https://code.rhodecode.com/rhodecode-tools-ce/artifacts/download/0-ed54e749-2ef5-4bc7-ae7f-7900e3c2aa15.tar.gz?sha256=76f024bad3a1e55fdb3d64f13f5b77ff21a12fee699918de2110fe21effd5a3a";
       sha256 = "0fjszppj3zhh47g1i6b9xqps28gzfxdkzwb47pdmzrd1sfx29w3n";
     };
     meta = {
       license = [ { fullName = "Apache 2.0 and Proprietary"; } ];
     };
   };
   "routes" = super.buildPythonPackage {
     name = "routes-2.4.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."repoze.lru"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/33/38/ea827837e68d9c7dde4cff7ec122a93c319f0effc08ce92a17095576603f/Routes-2.4.1.tar.gz";
       sha256 = "1zamff3m0kc4vyfniyhxpkkcqv1rrgnmh37ykxv34nna1ws47vi6";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "scandir" = super.buildPythonPackage {
     name = "scandir-1.10.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/df/f5/9c052db7bd54d0cbf1bc0bb6554362bba1012d03e5888950a4f5c5dadc4e/scandir-1.10.0.tar.gz";
       sha256 = "1bkqwmf056pkchf05ywbnf659wqlp6lljcdb0y88wr9f0vv32ijd";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "New BSD License"; } ];
     };
   };
   "setproctitle" = super.buildPythonPackage {
     name = "setproctitle-1.1.10";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/5a/0d/dc0d2234aacba6cf1a729964383e3452c52096dc695581248b548786f2b3/setproctitle-1.1.10.tar.gz";
       sha256 = "163kplw9dcrw0lffq1bvli5yws3rngpnvrxrzdw89pbphjjvg0v2";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "setuptools" = super.buildPythonPackage {
     name = "setuptools-44.1.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ed/7b/bbf89ca71e722b7f9464ebffe4b5ee20a9e5c9a555a56e2d3914bb9119a6/setuptools-44.1.0.zip";
       sha256 = "1jja896zvd1ppccnjbhkgagxbwchgq6vfamp6qn1hvywq6q9cjkr";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "simplegeneric" = super.buildPythonPackage {
     name = "simplegeneric-0.8.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/3d/57/4d9c9e3ae9a255cd4e1106bb57e24056d3d0709fc01b2e3e345898e49d5b/simplegeneric-0.8.1.zip";
       sha256 = "0wwi1c6md4vkbcsfsf8dklf3vr4mcdj4mpxkanwgb6jb1432x5yw";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "simplejson" = super.buildPythonPackage {
     name = "simplejson-3.16.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/e3/24/c35fb1c1c315fc0fffe61ea00d3f88e85469004713dab488dee4f35b0aff/simplejson-3.16.0.tar.gz";
       sha256 = "19cws1syk8jzq2pw43878dv6fjkb0ifvjpx0i9aajix6kc9jkwxi";
     };
     meta = {
       license = [ { fullName = "Academic Free License (AFL)"; } pkgs.lib.licenses.mit ];
     };
   };
   "six" = super.buildPythonPackage {
     name = "six-1.11.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/16/d8/bc6316cf98419719bd59c91742194c111b6f2e85abac88e496adefaf7afe/six-1.11.0.tar.gz";
       sha256 = "1scqzwc51c875z23phj48gircqjgnn3af8zy2izjwmnlxrxsgs3h";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "sqlalchemy" = super.buildPythonPackage {
     name = "sqlalchemy-1.3.15";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/8c/30/4134e726dd5ed13728ff814fa91fc01c447ad8700504653fe99d91fdd34b/SQLAlchemy-1.3.15.tar.gz";
       sha256 = "0iglkvymfp35zm5pxy5kzqvcv96kkas0chqdx7xpla86sspa9k64";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "sshpubkeys" = super.buildPythonPackage {
     name = "sshpubkeys-3.1.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."cryptography"
       self."ecdsa"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/00/23/f7508a12007c96861c3da811992f14283d79c819d71a217b3e12d5196649/sshpubkeys-3.1.0.tar.gz";
       sha256 = "105g2li04nm1hb15a2y6hm9m9k7fbrkd5l3gy12w3kgcmsf3k25k";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "subprocess32" = super.buildPythonPackage {
     name = "subprocess32-3.5.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/32/c8/564be4d12629b912ea431f1a50eb8b3b9d00f1a0b1ceff17f266be190007/subprocess32-3.5.4.tar.gz";
       sha256 = "17f7mvwx2271s1wrl0qac3wjqqnrqag866zs3qc8v5wp0k43fagb";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   "supervisor" = super.buildPythonPackage {
     name = "supervisor-4.1.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/de/87/ee1ad8fa533a4b5f2c7623f4a2b585d3c1947af7bed8e65bc7772274320e/supervisor-4.1.0.tar.gz";
       sha256 = "10q36sa1jqljyyyl7cif52akpygl5kmlqq9x91hmx53f8zh6zj1d";
     };
     meta = {
       license = [ { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "tempita" = super.buildPythonPackage {
     name = "tempita-0.5.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/56/c8/8ed6eee83dbddf7b0fc64dd5d4454bc05e6ccaafff47991f73f2894d9ff4/Tempita-0.5.2.tar.gz";
       sha256 = "177wwq45slfyajd8csy477bmdmzipyw0dm7i85k3akb7m85wzkna";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "termcolor" = super.buildPythonPackage {
     name = "termcolor-1.1.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/8a/48/a76be51647d0eb9f10e2a4511bf3ffb8cc1e6b14e9e4fab46173aa79f981/termcolor-1.1.0.tar.gz";
       sha256 = "0fv1vq14rpqwgazxg4981904lfyp84mnammw7y046491cv76jv8x";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "testpath" = super.buildPythonPackage {
     name = "testpath-0.4.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/2c/b3/5d57205e896d8998d77ad12aa42ebce75cd97d8b9a97d00ba078c4c9ffeb/testpath-0.4.4.tar.gz";
       sha256 = "0zpcmq22dz79ipvvsfnw1ykpjcaj6xyzy7ws77s5b5ql3hka7q30";
     };
     meta = {
       license = [  ];
     };
   };
   "traitlets" = super.buildPythonPackage {
     name = "traitlets-4.3.3";
     doCheck = false;
     propagatedBuildInputs = [
       self."ipython-genutils"
       self."six"
       self."decorator"
       self."enum34"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/75/b0/43deb021bc943f18f07cbe3dac1d681626a48997b7ffa1e7fb14ef922b21/traitlets-4.3.3.tar.gz";
       sha256 = "1xsrwgivpkxlbr4dfndfsi098s29yqgswgjc1qqn69yxklvfw8yh";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "transaction" = super.buildPythonPackage {
     name = "transaction-2.4.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."zope.interface"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/9d/7d/0e8af0d059e052b9dcf2bb5a08aad20ae3e238746bdd3f8701a60969b363/transaction-2.4.0.tar.gz";
       sha256 = "17wz1y524ca07vr03yddy8dv0gbscs06dbdywmllxv5rc725jq3j";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "translationstring" = super.buildPythonPackage {
     name = "translationstring-1.3";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/5e/eb/bee578cc150b44c653b63f5ebe258b5d0d812ddac12497e5f80fcad5d0b4/translationstring-1.3.tar.gz";
       sha256 = "0bdpcnd9pv0131dl08h4zbcwmgc45lyvq3pa224xwan5b3x4rr2f";
     };
     meta = {
       license = [ { fullName = "BSD-like (http://repoze.org/license.html)"; } ];
     };
   };
   "tzlocal" = super.buildPythonPackage {
     name = "tzlocal-1.5.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."pytz"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/cb/89/e3687d3ed99bc882793f82634e9824e62499fdfdc4b1ae39e211c5b05017/tzlocal-1.5.1.tar.gz";
       sha256 = "0kiciwiqx0bv0fbc913idxibc4ygg4cb7f8rcpd9ij2shi4bigjf";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "urllib3" = super.buildPythonPackage {
     name = "urllib3-1.25.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/9a/8b/ea6d2beb2da6e331e9857d0a60b79ed4f72dcbc4e2c7f2d2521b0480fda2/urllib3-1.25.2.tar.gz";
       sha256 = "1nq2k4pss1ihsjh02r41sqpjpm5rfqkjfysyq7g7n2i1p7c66c55";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "urlobject" = super.buildPythonPackage {
     name = "urlobject-2.4.3";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/e2/b8/1d0a916f4b34c4618846e6da0e4eeaa8fcb4a2f39e006434fe38acb74b34/URLObject-2.4.3.tar.gz";
       sha256 = "1ahc8ficzfvr2avln71immfh4ls0zyv6cdaa5xmkdj5rd87f5cj7";
     };
     meta = {
       license = [ pkgs.lib.licenses.publicDomain ];
     };
   };
   "venusian" = super.buildPythonPackage {
     name = "venusian-1.2.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/7e/6f/40a9d43ac77cb51cb62be5b5662d170f43f8037bdc4eab56336c4ca92bb7/venusian-1.2.0.tar.gz";
       sha256 = "0ghyx66g8ikx9nx1mnwqvdcqm11i1vlq0hnvwl50s48bp22q5v34";
     };
     meta = {
       license = [ { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "vine" = super.buildPythonPackage {
     name = "vine-1.3.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/1c/e1/79fb8046e607dd6c2ad05c9b8ebac9d0bd31d086a08f02699e96fc5b3046/vine-1.3.0.tar.gz";
       sha256 = "11ydsbhl1vabndc2r979dv61s6j2b0giq6dgvryifvq1m7bycghk";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "waitress" = super.buildPythonPackage {
     name = "waitress-1.3.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/a6/e6/708da7bba65898e5d759ade8391b1077e49d07be0b0223c39f5be04def56/waitress-1.3.1.tar.gz";
       sha256 = "1iysl8ka3l4cdrr0r19fh1cv28q41mwpvgsb81ji7k4shkb0k3i7";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "wcwidth" = super.buildPythonPackage {
     name = "wcwidth-0.1.9";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/25/9d/0acbed6e4a4be4fc99148f275488580968f44ddb5e69b8ceb53fc9df55a0/wcwidth-0.1.9.tar.gz";
       sha256 = "1wf5ycjx8s066rdvr0fgz4xds9a8zhs91c4jzxvvymm1c8l8cwzf";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "webencodings" = super.buildPythonPackage {
     name = "webencodings-0.5.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/0b/02/ae6ceac1baeda530866a85075641cec12989bd8d31af6d5ab4a3e8c92f47/webencodings-0.5.1.tar.gz";
       sha256 = "08qrgrc4hrximb2gqnl69g01s93rhf2842jfxdjljc1dbwj1qsmk";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "weberror" = super.buildPythonPackage {
     name = "weberror-0.13.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."webob"
       self."tempita"
       self."pygments"
       self."paste"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/07/0a/09ca5eb0fab5c0d17b380026babe81c96ecebb13f2b06c3203432dd7be72/WebError-0.13.1.tar.gz";
       sha256 = "0r4qvnf2r92gfnpa1kwygh4j2x6j3axg2i4an6hyxwg2gpaqp7y1";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "webhelpers2" = super.buildPythonPackage {
     name = "webhelpers2-2.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."markupsafe"
       self."six"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ff/30/56342c6ea522439e3662427c8d7b5e5b390dff4ff2dc92d8afcb8ab68b75/WebHelpers2-2.0.tar.gz";
       sha256 = "0aphva1qmxh83n01p53f5fd43m4srzbnfbz5ajvbx9aj2aipwmcs";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "webob" = super.buildPythonPackage {
     name = "webob-1.8.5";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/9d/1a/0c89c070ee2829c934cb6c7082287c822e28236a4fcf90063e6be7c35532/WebOb-1.8.5.tar.gz";
       sha256 = "11khpzaxc88q31v25ic330gsf56fwmbdc9b30br8mvp0fmwspah5";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "webtest" = super.buildPythonPackage {
     name = "webtest-2.0.34";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."webob"
       self."waitress"
       self."beautifulsoup4"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/2c/74/a0e63feee438735d628631e2b70d82280276a930637ac535479e5fad9427/WebTest-2.0.34.tar.gz";
       sha256 = "0x1y2c8z4fmpsny4hbp6ka37si2g10r5r2jwxhvv5mx7g3blq4bi";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "whoosh" = super.buildPythonPackage {
     name = "whoosh-2.7.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/25/2b/6beed2107b148edc1321da0d489afc4617b9ed317ef7b72d4993cad9b684/Whoosh-2.7.4.tar.gz";
       sha256 = "10qsqdjpbc85fykc1vgcs8xwbgn4l2l52c8d83xf1q59pwyn79bw";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal pkgs.lib.licenses.bsd2 ];
     };
   };
   "ws4py" = super.buildPythonPackage {
     name = "ws4py-0.5.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/53/20/4019a739b2eefe9282d3822ef6a225250af964b117356971bd55e274193c/ws4py-0.5.1.tar.gz";
       sha256 = "10slbbf2jm4hpr92jx7kh7mhf48sjl01v2w4d8z3f1p0ybbp7l19";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "wsgiref" = super.buildPythonPackage {
     name = "wsgiref-0.1.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/41/9e/309259ce8dff8c596e8c26df86dbc4e848b9249fd36797fd60be456f03fc/wsgiref-0.1.2.zip";
       sha256 = "0y8fyjmpq7vwwm4x732w97qbkw78rjwal5409k04cw4m03411rn7";
     };
     meta = {
       license = [ { fullName = "PSF or ZPL"; } ];
     };
   };
   "zipp" = super.buildPythonPackage {
     name = "zipp-1.2.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."contextlib2"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/78/08/d52f0ea643bc1068d6dc98b412f4966a9b63255d20911a23ac3220c033c4/zipp-1.2.0.tar.gz";
       sha256 = "1c91lnv1bxjimh8as27hz7bghsjkkbxn1d37xq7in9c82iai0167";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "zope.cachedescriptors" = super.buildPythonPackage {
     name = "zope.cachedescriptors-4.3.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/2f/89/ebe1890cc6d3291ebc935558fa764d5fffe571018dbbee200e9db78762cb/zope.cachedescriptors-4.3.1.tar.gz";
       sha256 = "0jhr3m5p74c6r7k8iv0005b8bfsialih9d7zl5vx38rf5xq1lk8z";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "zope.deprecation" = super.buildPythonPackage {
     name = "zope.deprecation-4.4.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/34/da/46e92d32d545dd067b9436279d84c339e8b16de2ca393d7b892bc1e1e9fd/zope.deprecation-4.4.0.tar.gz";
       sha256 = "1pz2cv7gv9y1r3m0bdv7ks1alagmrn5msm5spwdzkb2by0w36i8d";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "zope.event" = super.buildPythonPackage {
     name = "zope.event-4.4";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/4c/b2/51c0369adcf5be2334280eed230192ab3b03f81f8efda9ddea6f65cc7b32/zope.event-4.4.tar.gz";
       sha256 = "1ksbc726av9xacml6jhcfyn828hlhb9xlddpx6fcvnlvmpmpvhk9";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "zope.interface" = super.buildPythonPackage {
     name = "zope.interface-4.6.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/4e/d0/c9d16bd5b38de44a20c6dc5d5ed80a49626fafcb3db9f9efdc2a19026db6/zope.interface-4.6.0.tar.gz";
       sha256 = "1rgh2x3rcl9r0v0499kf78xy86rnmanajf4ywmqb943wpk50sg8v";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
 
 ### Test requirements
 
   
 }

rhodecode/VERSION

@@ -1,1 +1,1 b''
-4.22.0
  No newline at end of file
+4.23.0
  No newline at end of file

rhodecode/__init__.py

@@ -1,60 +1,60 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 from collections import OrderedDict
 
 import sys
 import platform
 
 VERSION = tuple(open(os.path.join(
     os.path.dirname(__file__), 'VERSION')).read().split('.'))
 
 BACKENDS = OrderedDict()
 
 BACKENDS['hg'] = 'Mercurial repository'
 BACKENDS['git'] = 'Git repository'
 BACKENDS['svn'] = 'Subversion repository'
 
 
 CELERY_ENABLED = False
 CELERY_EAGER = False
 
 # link to config for pyramid
 CONFIG = {}
 
 # Populated with the settings dictionary from application init in
 # rhodecode.conf.environment.load_pyramid_environment
 PYRAMID_SETTINGS = {}
 
 # Linked module for extensions
 EXTENSIONS = {}
 
 __version__ = ('.'.join((str(each) for each in VERSION[:3])))
-__dbversion__ = 110  # defines current db version for migrations
+__dbversion__ = 112  # defines current db version for migrations
 __platform__ = platform.system()
 __license__ = 'AGPLv3, and Commercial License'
 __author__ = 'RhodeCode GmbH'
 __url__ = 'https://code.rhodecode.com'
 
 is_windows = __platform__ in ['Windows']
 is_unix = not is_windows
 is_test = False
 disable_error_handler = False

rhodecode/api/utils.py

@@ -1,452 +1,458 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 JSON RPC utils
 """
 
 import collections
 import logging
 
 from rhodecode.api.exc import JSONRPCError
 from rhodecode.lib.auth import (
     HasPermissionAnyApi, HasRepoPermissionAnyApi, HasRepoGroupPermissionAnyApi)
 from rhodecode.lib.utils import safe_unicode
 from rhodecode.lib.vcs.exceptions import RepositoryError
 from rhodecode.lib.view_utils import get_commit_from_ref_name
 from rhodecode.lib.utils2 import str2bool
 
 log = logging.getLogger(__name__)
 
 
 class OAttr(object):
     """
     Special Option that defines other attribute, and can default to them
 
     Example::
 
         def test(apiuser, userid=Optional(OAttr('apiuser')):
             user = Optional.extract(userid, evaluate_locals=local())
             #if we pass in userid, we get it, else it will default to apiuser
             #attribute
     """
 
     def __init__(self, attr_name):
         self.attr_name = attr_name
 
     def __repr__(self):
         return '<OptionalAttr:%s>' % self.attr_name
 
     def __call__(self):
         return self
 
 
 class Optional(object):
     """
     Defines an optional parameter::
 
         param = param.getval() if isinstance(param, Optional) else param
         param = param() if isinstance(param, Optional) else param
 
     is equivalent of::
 
         param = Optional.extract(param)
 
     """
 
     def __init__(self, type_):
         self.type_ = type_
 
     def __repr__(self):
         return '<Optional:%s>' % self.type_.__repr__()
 
     def __call__(self):
         return self.getval()
 
     def getval(self, evaluate_locals=None):
         """
         returns value from this Optional instance
         """
         if isinstance(self.type_, OAttr):
             param_name = self.type_.attr_name
             if evaluate_locals:
                 return evaluate_locals[param_name]
             # use params name
             return param_name
         return self.type_
 
     @classmethod
     def extract(cls, val, evaluate_locals=None, binary=None):
         """
         Extracts value from Optional() instance
 
         :param val:
         :return: original value if it's not Optional instance else
             value of instance
         """
         if isinstance(val, cls):
             val = val.getval(evaluate_locals)
 
         if binary:
             val = str2bool(val)
 
         return val
 
 
 def parse_args(cli_args, key_prefix=''):
     from rhodecode.lib.utils2 import (escape_split)
     kwargs = collections.defaultdict(dict)
     for el in escape_split(cli_args, ','):
         kv = escape_split(el, '=', 1)
         if len(kv) == 2:
             k, v = kv
             kwargs[key_prefix + k] = v
     return kwargs
 
 
 def get_origin(obj):
     """
     Get origin of permission from object.
 
     :param obj:
     """
     origin = 'permission'
 
     if getattr(obj, 'owner_row', '') and getattr(obj, 'admin_row', ''):
         # admin and owner case, maybe we should use dual string ?
         origin = 'owner'
     elif getattr(obj, 'owner_row', ''):
         origin = 'owner'
     elif getattr(obj, 'admin_row', ''):
         origin = 'super-admin'
     return origin
 
 
 def store_update(updates, attr, name):
     """
     Stores param in updates dict if it's not instance of Optional
     allows easy updates of passed in params
     """
     if not isinstance(attr, Optional):
         updates[name] = attr
 
 
 def has_superadmin_permission(apiuser):
     """
     Return True if apiuser is admin or return False
 
     :param apiuser:
     """
     if HasPermissionAnyApi('hg.admin')(user=apiuser):
         return True
     return False
 
 
 def validate_repo_permissions(apiuser, repoid, repo, perms):
     """
     Raise JsonRPCError if apiuser is not authorized or return True
 
     :param apiuser:
     :param repoid:
     :param repo:
     :param perms:
     """
     if not HasRepoPermissionAnyApi(*perms)(
             user=apiuser, repo_name=repo.repo_name):
         raise JSONRPCError('repository `%s` does not exist' % repoid)
 
     return True
 
 
 def validate_repo_group_permissions(apiuser, repogroupid, repo_group, perms):
     """
     Raise JsonRPCError if apiuser is not authorized or return True
 
     :param apiuser:
     :param repogroupid: just the id of repository group
     :param repo_group: instance of repo_group
     :param perms:
     """
     if not HasRepoGroupPermissionAnyApi(*perms)(
             user=apiuser, group_name=repo_group.group_name):
         raise JSONRPCError(
             'repository group `%s` does not exist' % repogroupid)
 
     return True
 
 
 def validate_set_owner_permissions(apiuser, owner):
     if isinstance(owner, Optional):
         owner = get_user_or_error(apiuser.user_id)
     else:
         if has_superadmin_permission(apiuser):
             owner = get_user_or_error(owner)
         else:
             # forbid setting owner for non-admins
             raise JSONRPCError(
                 'Only RhodeCode super-admin can specify `owner` param')
     return owner
 
 
 def get_user_or_error(userid):
     """
     Get user by id or name or return JsonRPCError if not found
 
     :param userid:
     """
     from rhodecode.model.user import UserModel
     user_model = UserModel()
 
     if isinstance(userid, (int, long)):
         try:
             user = user_model.get_user(userid)
         except ValueError:
             user = None
     else:
         user = user_model.get_by_username(userid)
 
     if user is None:
         raise JSONRPCError(
             'user `%s` does not exist' % (userid,))
     return user
 
 
 def get_repo_or_error(repoid):
     """
     Get repo by id or name or return JsonRPCError if not found
 
     :param repoid:
     """
     from rhodecode.model.repo import RepoModel
     repo_model = RepoModel()
 
     if isinstance(repoid, (int, long)):
         try:
             repo = repo_model.get_repo(repoid)
         except ValueError:
             repo = None
     else:
         repo = repo_model.get_by_repo_name(repoid)
 
     if repo is None:
         raise JSONRPCError(
             'repository `%s` does not exist' % (repoid,))
     return repo
 
 
 def get_repo_group_or_error(repogroupid):
     """
     Get repo group by id or name or return JsonRPCError if not found
 
     :param repogroupid:
     """
     from rhodecode.model.repo_group import RepoGroupModel
     repo_group_model = RepoGroupModel()
 
     if isinstance(repogroupid, (int, long)):
         try:
             repo_group = repo_group_model._get_repo_group(repogroupid)
         except ValueError:
             repo_group = None
     else:
         repo_group = repo_group_model.get_by_group_name(repogroupid)
 
     if repo_group is None:
         raise JSONRPCError(
             'repository group `%s` does not exist' % (repogroupid,))
     return repo_group
 
 
 def get_user_group_or_error(usergroupid):
     """
     Get user group by id or name or return JsonRPCError if not found
 
     :param usergroupid:
     """
     from rhodecode.model.user_group import UserGroupModel
     user_group_model = UserGroupModel()
 
     if isinstance(usergroupid, (int, long)):
         try:
             user_group = user_group_model.get_group(usergroupid)
         except ValueError:
             user_group = None
     else:
         user_group = user_group_model.get_by_name(usergroupid)
 
     if user_group is None:
         raise JSONRPCError(
             'user group `%s` does not exist' % (usergroupid,))
     return user_group
 
 
 def get_perm_or_error(permid, prefix=None):
     """
     Get permission by id or name or return JsonRPCError if not found
 
     :param permid:
     """
     from rhodecode.model.permission import PermissionModel
 
     perm = PermissionModel.cls.get_by_key(permid)
     if perm is None:
         msg = 'permission `{}` does not exist.'.format(permid)
         if prefix:
             msg += ' Permission should start with prefix: `{}`'.format(prefix)
         raise JSONRPCError(msg)
 
     if prefix:
         if not perm.permission_name.startswith(prefix):
             raise JSONRPCError('permission `%s` is invalid, '
                                'should start with %s' % (permid, prefix))
     return perm
 
 
 def get_gist_or_error(gistid):
     """
     Get gist by id or gist_access_id or return JsonRPCError if not found
 
     :param gistid:
     """
     from rhodecode.model.gist import GistModel
 
     gist = GistModel.cls.get_by_access_id(gistid)
     if gist is None:
         raise JSONRPCError('gist `%s` does not exist' % (gistid,))
     return gist
 
 
 def get_pull_request_or_error(pullrequestid):
     """
     Get pull request by id or return JsonRPCError if not found
 
     :param pullrequestid:
     """
     from rhodecode.model.pull_request import PullRequestModel
 
     try:
         pull_request = PullRequestModel().get(int(pullrequestid))
     except ValueError:
         raise JSONRPCError('pullrequestid must be an integer')
     if not pull_request:
         raise JSONRPCError('pull request `%s` does not exist' % (
             pullrequestid,))
     return pull_request
 
 
-def build_commit_data(commit, detail_level):
+def build_commit_data(rhodecode_vcs_repo, commit, detail_level):
+    commit2 = commit
+    commit1 = commit.first_parent
+
     parsed_diff = []
     if detail_level == 'extended':
         for f_path in commit.added_paths:
             parsed_diff.append(_get_commit_dict(filename=f_path, op='A'))
         for f_path in commit.changed_paths:
             parsed_diff.append(_get_commit_dict(filename=f_path, op='M'))
         for f_path in commit.removed_paths:
             parsed_diff.append(_get_commit_dict(filename=f_path, op='D'))
 
     elif detail_level == 'full':
-        from rhodecode.lib.diffs import DiffProcessor
-        diff_processor = DiffProcessor(commit.diff())
+        from rhodecode.lib import diffs
+
+        _diff = rhodecode_vcs_repo.get_diff(commit1, commit2,)
+        diff_processor = diffs.DiffProcessor(_diff, format='newdiff', show_full_diff=True)
+
         for dp in diff_processor.prepare():
             del dp['stats']['ops']
             _stats = dp['stats']
             parsed_diff.append(_get_commit_dict(
                 filename=dp['filename'], op=dp['operation'],
                 new_revision=dp['new_revision'],
                 old_revision=dp['old_revision'],
                 raw_diff=dp['raw_diff'], stats=_stats))
 
     return parsed_diff
 
 
 def get_commit_or_error(ref, repo):
     try:
         ref_type, _, ref_hash = ref.split(':')
     except ValueError:
         raise JSONRPCError(
             'Ref `{ref}` given in a wrong format. Please check the API'
             ' documentation for more details'.format(ref=ref))
     try:
         # TODO: dan: refactor this to use repo.scm_instance().get_commit()
         # once get_commit supports ref_types
         return get_commit_from_ref_name(repo, ref_hash)
     except RepositoryError:
         raise JSONRPCError('Ref `{ref}` does not exist'.format(ref=ref))
 
 
 def _get_ref_hash(repo, type_, name):
     vcs_repo = repo.scm_instance()
     if type_ in ['branch'] and vcs_repo.alias in ('hg', 'git'):
         return vcs_repo.branches[name]
     elif type_ in ['bookmark', 'book'] and vcs_repo.alias == 'hg':
         return vcs_repo.bookmarks[name]
     else:
         raise ValueError()
 
 
 def resolve_ref_or_error(ref, repo, allowed_ref_types=None):
     allowed_ref_types = allowed_ref_types or ['bookmark', 'book', 'tag', 'branch']
 
     def _parse_ref(type_, name, hash_=None):
         return type_, name, hash_
 
     try:
         ref_type, ref_name, ref_hash = _parse_ref(*ref.split(':'))
     except TypeError:
         raise JSONRPCError(
             'Ref `{ref}` given in a wrong format. Please check the API'
             ' documentation for more details'.format(ref=ref))
 
     if ref_type not in allowed_ref_types:
         raise JSONRPCError(
             'Ref `{ref}` type is not allowed. '
             'Only:{allowed_refs} are possible.'.format(
                 ref=ref, allowed_refs=allowed_ref_types))
 
     try:
         ref_hash = ref_hash or _get_ref_hash(repo, ref_type, ref_name)
     except (KeyError, ValueError):
         raise JSONRPCError(
             'The specified value:{type}:`{name}` does not exist, or is not allowed.'.format(
                 type=ref_type, name=ref_name))
 
     return ':'.join([ref_type, ref_name, ref_hash])
 
 
 def _get_commit_dict(
         filename, op, new_revision=None, old_revision=None,
         raw_diff=None, stats=None):
     if stats is None:
         stats = {
             "added": None,
             "binary": None,
             "deleted": None
         }
     return {
         "filename": safe_unicode(filename),
         "op": op,
 
         # extra details
         "new_revision": new_revision,
         "old_revision": old_revision,
 
         "raw_diff": raw_diff,
         "stats": stats
     }

rhodecode/api/views/repo_api.py

@@ -1,2523 +1,2524 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import time
 
 import rhodecode
 from rhodecode.api import (
     jsonrpc_method, JSONRPCError, JSONRPCForbidden, JSONRPCValidationError)
 from rhodecode.api.utils import (
     has_superadmin_permission, Optional, OAttr, get_repo_or_error,
     get_user_group_or_error, get_user_or_error, validate_repo_permissions,
     get_perm_or_error, parse_args, get_origin, build_commit_data,
     validate_set_owner_permissions)
 from rhodecode.lib import audit_logger, rc_cache, channelstream
 from rhodecode.lib import repo_maintenance
 from rhodecode.lib.auth import (
     HasPermissionAnyApi, HasUserGroupPermissionAnyApi,
     HasRepoPermissionAnyApi)
 from rhodecode.lib.celerylib.utils import get_task_id
 from rhodecode.lib.utils2 import (
     str2bool, time_to_datetime, safe_str, safe_int, safe_unicode)
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.exceptions import (
     StatusChangeOnClosedPullRequestError, CommentVersionMismatch)
 from rhodecode.lib.vcs import RepositoryError
 from rhodecode.lib.vcs.exceptions import NodeDoesNotExistError
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.db import (
     Session, ChangesetStatus, RepositoryField, Repository, RepoGroup,
     ChangesetComment)
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.pull_request import PullRequestModel
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.scm import ScmModel, RepoList
 from rhodecode.model.settings import SettingsModel, VcsSettingsModel
 from rhodecode.model import validation_schema
 from rhodecode.model.validation_schema.schemas import repo_schema
 
 log = logging.getLogger(__name__)
 
 
 @jsonrpc_method()
 def get_repo(request, apiuser, repoid, cache=Optional(True)):
     """
     Gets an existing repository by its name or repository_id.
 
     The members section so the output returns users groups or users
     associated with that repository.
 
     This command can only be run using an |authtoken| with admin rights,
     or users with at least read rights to the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository id.
     :type repoid: str or int
     :param cache: use the cached value for last changeset
     :type: cache: Optional(bool)
 
     Example output:
 
     .. code-block:: bash
 
         {
           "error": null,
           "id": <repo_id>,
           "result": {
             "clone_uri": null,
             "created_on": "timestamp",
             "description": "repo description",
             "enable_downloads": false,
             "enable_locking": false,
             "enable_statistics": false,
             "followers": [
               {
                 "active": true,
                 "admin": false,
                 "api_key": "****************************************",
                 "api_keys": [
                   "****************************************"
                 ],
                 "email": "[e-mail]",
                 "emails": [
                   "[e-mail]"
                 ],
                 "extern_name": "rhodecode",
                 "extern_type": "rhodecode",
                 "firstname": "username",
                 "ip_addresses": [],
                 "language": null,
                 "last_login": "2015-09-16T17:16:35.854",
                 "lastname": "surname",
                 "user_id": <user_id>,
                 "username": "name"
               }
             ],
             "fork_of": "parent-repo",
             "landing_rev": [
               "rev",
               "tip"
             ],
             "last_changeset": {
               "author": "User <[e-mail]>",
               "branch": "default",
               "date": "timestamp",
               "message": "last commit message",
               "parents": [
                 {
                   "raw_id": "commit-id"
                 }
               ],
               "raw_id": "commit-id",
               "revision": <revision number>,
               "short_id": "short id"
             },
             "lock_reason": null,
             "locked_by": null,
             "locked_date": null,
             "owner": "owner-name",
             "permissions": [
               {
                 "name": "super-admin-name",
                 "origin": "super-admin",
                 "permission": "repository.admin",
                 "type": "user"
               },
               {
                 "name": "owner-name",
                 "origin": "owner",
                 "permission": "repository.admin",
                 "type": "user"
               },
               {
                 "name": "user-group-name",
                 "origin": "permission",
                 "permission": "repository.write",
                 "type": "user_group"
               }
             ],
             "private": true,
             "repo_id": 676,
             "repo_name": "user-group/repo-name",
             "repo_type": "hg"
           }
         }
     """
 
     repo = get_repo_or_error(repoid)
     cache = Optional.extract(cache)
 
     include_secrets = False
     if has_superadmin_permission(apiuser):
         include_secrets = True
     else:
         # check if we have at least read permission for this repo !
         _perms = (
             'repository.admin', 'repository.write', 'repository.read',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     permissions = []
     for _user in repo.permissions():
         user_data = {
             'name': _user.username,
             'permission': _user.permission,
             'origin': get_origin(_user),
             'type': "user",
         }
         permissions.append(user_data)
 
     for _user_group in repo.permission_user_groups():
         user_group_data = {
             'name': _user_group.users_group_name,
             'permission': _user_group.permission,
             'origin': get_origin(_user_group),
             'type': "user_group",
         }
         permissions.append(user_group_data)
 
     following_users = [
         user.user.get_api_data(include_secrets=include_secrets)
         for user in repo.followers]
 
     if not cache:
         repo.update_commit_cache()
     data = repo.get_api_data(include_secrets=include_secrets)
     data['permissions'] = permissions
     data['followers'] = following_users
     return data
 
 
 @jsonrpc_method()
 def get_repos(request, apiuser, root=Optional(None), traverse=Optional(True)):
     """
     Lists all existing repositories.
 
     This command can only be run using an |authtoken| with admin rights,
     or users with at least read rights to |repos|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param root: specify root repository group to fetch repositories.
         filters the returned repositories to be members of given root group.
     :type root: Optional(None)
     :param traverse: traverse given root into subrepositories. With this flag
         set to False, it will only return top-level repositories from `root`.
         if root is empty it will return just top-level repositories.
     :type traverse: Optional(True)
 
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: [
                   {
                     "repo_id" :          "<repo_id>",
                     "repo_name" :        "<reponame>"
                     "repo_type" :        "<repo_type>",
                     "clone_uri" :        "<clone_uri>",
                     "private": :         "<bool>",
                     "created_on" :       "<datetimecreated>",
                     "description" :      "<description>",
                     "landing_rev":       "<landing_rev>",
                     "owner":             "<repo_owner>",
                     "fork_of":           "<name_of_fork_parent>",
                     "enable_downloads":  "<bool>",
                     "enable_locking":    "<bool>",
                     "enable_statistics": "<bool>",
                   },
                   ...
                 ]
         error:  null
     """
 
     include_secrets = has_superadmin_permission(apiuser)
     _perms = ('repository.read', 'repository.write', 'repository.admin',)
     extras = {'user': apiuser}
 
     root = Optional.extract(root)
     traverse = Optional.extract(traverse, binary=True)
 
     if root:
         # verify parent existance, if it's empty return an error
         parent = RepoGroup.get_by_group_name(root)
         if not parent:
             raise JSONRPCError(
                 'Root repository group `{}` does not exist'.format(root))
 
         if traverse:
             repos = RepoModel().get_repos_for_root(root=root, traverse=traverse)
         else:
             repos = RepoModel().get_repos_for_root(root=parent)
     else:
         if traverse:
             repos = RepoModel().get_all()
         else:
             # return just top-level
             repos = RepoModel().get_repos_for_root(root=None)
 
     repo_list = RepoList(repos, perm_set=_perms, extra_kwargs=extras)
     return [repo.get_api_data(include_secrets=include_secrets)
             for repo in repo_list]
 
 
 @jsonrpc_method()
 def get_repo_changeset(request, apiuser, repoid, revision,
                        details=Optional('basic')):
     """
     Returns information about a changeset.
 
     Additionally parameters define the amount of details returned by
     this function.
 
     This command can only be run using an |authtoken| with admin rights,
     or users with at least read rights to the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository id
     :type repoid: str or int
     :param revision: revision for which listing should be done
     :type revision: str
     :param details: details can be 'basic|extended|full' full gives diff
         info details like the diff itself, and number of changed files etc.
     :type details: Optional(str)
 
     """
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin', 'repository.write', 'repository.read',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     changes_details = Optional.extract(details)
     _changes_details_types = ['basic', 'extended', 'full']
     if changes_details not in _changes_details_types:
         raise JSONRPCError(
             'ret_type must be one of %s' % (
                 ','.join(_changes_details_types)))
 
+    vcs_repo = repo.scm_instance()
     pre_load = ['author', 'branch', 'date', 'message', 'parents',
                 'status', '_commit', '_file_paths']
 
     try:
-        cs = repo.get_commit(commit_id=revision, pre_load=pre_load)
+        commit = repo.get_commit(commit_id=revision, pre_load=pre_load)
     except TypeError as e:
         raise JSONRPCError(safe_str(e))
-    _cs_json = cs.__json__()
-    _cs_json['diff'] = build_commit_data(cs, changes_details)
+    _cs_json = commit.__json__()
+    _cs_json['diff'] = build_commit_data(vcs_repo, commit, changes_details)
     if changes_details == 'full':
-        _cs_json['refs'] = cs._get_refs()
+        _cs_json['refs'] = commit._get_refs()
     return _cs_json
 
 
 @jsonrpc_method()
 def get_repo_changesets(request, apiuser, repoid, start_rev, limit,
                         details=Optional('basic')):
     """
     Returns a set of commits limited by the number starting
     from the `start_rev` option.
 
     Additional parameters define the amount of details returned by this
     function.
 
     This command can only be run using an |authtoken| with admin rights,
     or users with at least read rights to |repos|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository ID.
     :type repoid: str or int
     :param start_rev: The starting revision from where to get changesets.
     :type start_rev: str
     :param limit: Limit the number of commits to this amount
     :type limit: str or int
     :param details: Set the level of detail returned. Valid option are:
         ``basic``, ``extended`` and ``full``.
     :type details: Optional(str)
 
     .. note::
 
        Setting the parameter `details` to the value ``full`` is extensive
        and returns details like the diff itself, and the number
        of changed files.
 
     """
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin', 'repository.write', 'repository.read',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     changes_details = Optional.extract(details)
     _changes_details_types = ['basic', 'extended', 'full']
     if changes_details not in _changes_details_types:
         raise JSONRPCError(
             'ret_type must be one of %s' % (
                 ','.join(_changes_details_types)))
 
     limit = int(limit)
     pre_load = ['author', 'branch', 'date', 'message', 'parents',
                 'status', '_commit', '_file_paths']
 
     vcs_repo = repo.scm_instance()
     # SVN needs a special case to distinguish its index and commit id
     if vcs_repo and vcs_repo.alias == 'svn' and (start_rev == '0'):
         start_rev = vcs_repo.commit_ids[0]
 
     try:
         commits = vcs_repo.get_commits(
             start_id=start_rev, pre_load=pre_load, translate_tags=False)
     except TypeError as e:
         raise JSONRPCError(safe_str(e))
     except Exception:
         log.exception('Fetching of commits failed')
         raise JSONRPCError('Error occurred during commit fetching')
 
     ret = []
     for cnt, commit in enumerate(commits):
         if cnt >= limit != -1:
             break
         _cs_json = commit.__json__()
-        _cs_json['diff'] = build_commit_data(commit, changes_details)
+        _cs_json['diff'] = build_commit_data(vcs_repo, commit, changes_details)
         if changes_details == 'full':
             _cs_json['refs'] = {
                 'branches': [commit.branch],
                 'bookmarks': getattr(commit, 'bookmarks', []),
                 'tags': commit.tags
             }
         ret.append(_cs_json)
     return ret
 
 
 @jsonrpc_method()
 def get_repo_nodes(request, apiuser, repoid, revision, root_path,
                    ret_type=Optional('all'), details=Optional('basic'),
                    max_file_bytes=Optional(None)):
     """
     Returns a list of nodes and children in a flat list for a given
     path at given revision.
 
     It's possible to specify ret_type to show only `files` or `dirs`.
 
     This command can only be run using an |authtoken| with admin rights,
     or users with at least read rights to |repos|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository ID.
     :type repoid: str or int
     :param revision: The revision for which listing should be done.
     :type revision: str
     :param root_path: The path from which to start displaying.
     :type root_path: str
     :param ret_type: Set the return type. Valid options are
         ``all`` (default), ``files`` and ``dirs``.
     :type ret_type: Optional(str)
     :param details: Returns extended information about nodes, such as
         md5, binary, and or content.
         The valid options are ``basic`` and ``full``.
     :type details: Optional(str)
     :param max_file_bytes: Only return file content under this file size bytes
     :type details: Optional(int)
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: [
                     {
                       "binary": false,
                       "content": "File line",
                       "extension": "md",
                       "lines": 2,
                       "md5": "059fa5d29b19c0657e384749480f6422",
                       "mimetype": "text/x-minidsrc",
                       "name": "file.md",
                       "size": 580,
                       "type": "file"
                     },
                   ...
                 ]
         error:  null
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin', 'repository.write', 'repository.read',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     ret_type = Optional.extract(ret_type)
     details = Optional.extract(details)
     _extended_types = ['basic', 'full']
     if details not in _extended_types:
         raise JSONRPCError('ret_type must be one of %s' % (','.join(_extended_types)))
     extended_info = False
     content = False
     if details == 'basic':
         extended_info = True
 
     if details == 'full':
         extended_info = content = True
 
     _map = {}
     try:
         # check if repo is not empty by any chance, skip quicker if it is.
         _scm = repo.scm_instance()
         if _scm.is_empty():
             return []
 
         _d, _f = ScmModel().get_nodes(
             repo, revision, root_path, flat=False,
             extended_info=extended_info, content=content,
             max_file_bytes=max_file_bytes)
         _map = {
             'all': _d + _f,
             'files': _f,
             'dirs': _d,
         }
         return _map[ret_type]
     except KeyError:
         raise JSONRPCError(
             'ret_type must be one of %s' % (','.join(sorted(_map.keys()))))
     except Exception:
         log.exception("Exception occurred while trying to get repo nodes")
         raise JSONRPCError(
             'failed to get repo: `%s` nodes' % repo.repo_name
         )
 
 
 @jsonrpc_method()
 def get_repo_file(request, apiuser, repoid, commit_id, file_path,
                   max_file_bytes=Optional(None), details=Optional('basic'),
                   cache=Optional(True)):
     """
     Returns a single file from repository at given revision.
 
     This command can only be run using an |authtoken| with admin rights,
     or users with at least read rights to |repos|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository ID.
     :type repoid: str or int
     :param commit_id: The revision for which listing should be done.
     :type commit_id: str
     :param file_path: The path from which to start displaying.
     :type file_path: str
     :param details: Returns different set of information about nodes.
         The valid options are ``minimal`` ``basic`` and ``full``.
     :type details: Optional(str)
     :param max_file_bytes: Only return file content under this file size bytes
     :type max_file_bytes: Optional(int)
     :param cache: Use internal caches for fetching files. If disabled fetching
         files is slower but more memory efficient
     :type cache: Optional(bool)
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: {
             "binary": false,
             "extension": "py",
             "lines": 35,
             "content": "....",
             "md5": "76318336366b0f17ee249e11b0c99c41",
             "mimetype": "text/x-python",
             "name": "python.py",
             "size": 817,
             "type": "file",
         }
         error:  null
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin', 'repository.write', 'repository.read',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     cache = Optional.extract(cache, binary=True)
     details = Optional.extract(details)
     _extended_types = ['minimal', 'minimal+search', 'basic', 'full']
     if details not in _extended_types:
         raise JSONRPCError(
             'ret_type must be one of %s, got %s' % (','.join(_extended_types)), details)
     extended_info = False
     content = False
 
     if details == 'minimal':
         extended_info = False
 
     elif details == 'basic':
         extended_info = True
 
     elif details == 'full':
         extended_info = content = True
 
     file_path = safe_unicode(file_path)
     try:
         # check if repo is not empty by any chance, skip quicker if it is.
         _scm = repo.scm_instance()
         if _scm.is_empty():
             return None
 
         node = ScmModel().get_node(
             repo, commit_id, file_path, extended_info=extended_info,
             content=content, max_file_bytes=max_file_bytes, cache=cache)
     except NodeDoesNotExistError:
         raise JSONRPCError(u'There is no file in repo: `{}` at path `{}` for commit: `{}`'.format(
             repo.repo_name, file_path, commit_id))
     except Exception:
         log.exception(u"Exception occurred while trying to get repo %s file",
                       repo.repo_name)
         raise JSONRPCError(u'failed to get repo: `{}` file at path {}'.format(
             repo.repo_name, file_path))
 
     return node
 
 
 @jsonrpc_method()
 def get_repo_fts_tree(request, apiuser, repoid, commit_id, root_path):
     """
     Returns a list of tree nodes for path at given revision. This api is built
     strictly for usage in full text search building, and shouldn't be consumed
 
     This command can only be run using an |authtoken| with admin rights,
     or users with at least read rights to |repos|.
 
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin', 'repository.write', 'repository.read',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     repo_id = repo.repo_id
     cache_seconds = safe_int(rhodecode.CONFIG.get('rc_cache.cache_repo.expiration_time'))
     cache_on = cache_seconds > 0
 
     cache_namespace_uid = 'cache_repo.{}'.format(repo_id)
     region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
 
     def compute_fts_tree(cache_ver, repo_id, commit_id, root_path):
         return ScmModel().get_fts_data(repo_id, commit_id, root_path)
 
     try:
         # check if repo is not empty by any chance, skip quicker if it is.
         _scm = repo.scm_instance()
         if _scm.is_empty():
             return []
     except RepositoryError:
         log.exception("Exception occurred while trying to get repo nodes")
         raise JSONRPCError('failed to get repo: `%s` nodes' % repo.repo_name)
 
     try:
         # we need to resolve commit_id to a FULL sha for cache to work correctly.
         # sending 'master' is a pointer that needs to be translated to current commit.
         commit_id = _scm.get_commit(commit_id=commit_id).raw_id
         log.debug(
             'Computing FTS REPO TREE for repo_id %s commit_id `%s` '
             'with caching: %s[TTL: %ss]' % (
                 repo_id, commit_id, cache_on, cache_seconds or 0))
 
         tree_files = compute_fts_tree(rc_cache.FILE_TREE_CACHE_VER, repo_id, commit_id, root_path)
         return tree_files
 
     except Exception:
         log.exception("Exception occurred while trying to get repo nodes")
         raise JSONRPCError('failed to get repo: `%s` nodes' % repo.repo_name)
 
 
 @jsonrpc_method()
 def get_repo_refs(request, apiuser, repoid):
     """
     Returns a dictionary of current references. It returns
     bookmarks, branches, closed_branches, and tags for given repository
 
     It's possible to specify ret_type to show only `files` or `dirs`.
 
     This command can only be run using an |authtoken| with admin rights,
     or users with at least read rights to |repos|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository ID.
     :type repoid: str or int
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         "result": {
             "bookmarks": {
               "dev": "5611d30200f4040ba2ab4f3d64e5b06408a02188",
               "master": "367f590445081d8ec8c2ea0456e73ae1f1c3d6cf"
             },
             "branches": {
               "default": "5611d30200f4040ba2ab4f3d64e5b06408a02188",
               "stable": "367f590445081d8ec8c2ea0456e73ae1f1c3d6cf"
             },
             "branches_closed": {},
             "tags": {
               "tip": "5611d30200f4040ba2ab4f3d64e5b06408a02188",
               "v4.4.0": "1232313f9e6adac5ce5399c2a891dc1e72b79022",
               "v4.4.1": "cbb9f1d329ae5768379cdec55a62ebdd546c4e27",
               "v4.4.2": "24ffe44a27fcd1c5b6936144e176b9f6dd2f3a17",
             }
         }
         error:  null
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin', 'repository.write', 'repository.read',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     try:
         # check if repo is not empty by any chance, skip quicker if it is.
         vcs_instance = repo.scm_instance()
         refs = vcs_instance.refs()
         return refs
     except Exception:
         log.exception("Exception occurred while trying to get repo refs")
         raise JSONRPCError(
             'failed to get repo: `%s` references' % repo.repo_name
         )
 
 
 @jsonrpc_method()
 def create_repo(
         request, apiuser, repo_name, repo_type,
         owner=Optional(OAttr('apiuser')),
         description=Optional(''),
         private=Optional(False),
         clone_uri=Optional(None),
         push_uri=Optional(None),
         landing_rev=Optional(None),
         enable_statistics=Optional(False),
         enable_locking=Optional(False),
         enable_downloads=Optional(False),
         copy_permissions=Optional(False)):
     """
     Creates a repository.
 
     * If the repository name contains "/", repository will be created inside
       a repository group or nested repository groups
 
       For example "foo/bar/repo1" will create |repo| called "repo1" inside
       group "foo/bar". You have to have permissions to access and write to
       the last repository group ("bar" in this example)
 
     This command can only be run using an |authtoken| with at least
     permissions to create repositories, or write permissions to
     parent repository groups.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repo_name: Set the repository name.
     :type repo_name: str
     :param repo_type: Set the repository type; 'hg','git', or 'svn'.
     :type repo_type: str
     :param owner: user_id or username
     :type owner: Optional(str)
     :param description: Set the repository description.
     :type description: Optional(str)
     :param private: set repository as private
     :type private: bool
     :param clone_uri: set clone_uri
     :type clone_uri: str
     :param push_uri: set push_uri
     :type push_uri: str
     :param landing_rev: <rev_type>:<rev>, e.g branch:default, book:dev, rev:abcd
     :type landing_rev: str
     :param enable_locking:
     :type enable_locking: bool
     :param enable_downloads:
     :type enable_downloads: bool
     :param enable_statistics:
     :type enable_statistics: bool
     :param copy_permissions: Copy permission from group in which the
         repository is being created.
     :type copy_permissions: bool
 
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: {
                   "msg": "Created new repository `<reponame>`",
                   "success": true,
                   "task": "<celery task id or None if done sync>"
                 }
         error:  null
 
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
          'failed to create repository `<repo_name>`'
       }
 
     """
 
     owner = validate_set_owner_permissions(apiuser, owner)
 
     description = Optional.extract(description)
     copy_permissions = Optional.extract(copy_permissions)
     clone_uri = Optional.extract(clone_uri)
     push_uri = Optional.extract(push_uri)
 
     defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
     if isinstance(private, Optional):
         private = defs.get('repo_private') or Optional.extract(private)
     if isinstance(repo_type, Optional):
         repo_type = defs.get('repo_type')
     if isinstance(enable_statistics, Optional):
         enable_statistics = defs.get('repo_enable_statistics')
     if isinstance(enable_locking, Optional):
         enable_locking = defs.get('repo_enable_locking')
     if isinstance(enable_downloads, Optional):
         enable_downloads = defs.get('repo_enable_downloads')
 
     landing_ref, _label = ScmModel.backend_landing_ref(repo_type)
     ref_choices, _labels = ScmModel().get_repo_landing_revs(request.translate)
     ref_choices = list(set(ref_choices + [landing_ref]))
 
     landing_commit_ref = Optional.extract(landing_rev) or landing_ref
 
     schema = repo_schema.RepoSchema().bind(
         repo_type_options=rhodecode.BACKENDS.keys(),
         repo_ref_options=ref_choices,
         repo_type=repo_type,
         # user caller
         user=apiuser)
 
     try:
         schema_data = schema.deserialize(dict(
             repo_name=repo_name,
             repo_type=repo_type,
             repo_owner=owner.username,
             repo_description=description,
             repo_landing_commit_ref=landing_commit_ref,
             repo_clone_uri=clone_uri,
             repo_push_uri=push_uri,
             repo_private=private,
             repo_copy_permissions=copy_permissions,
             repo_enable_statistics=enable_statistics,
             repo_enable_downloads=enable_downloads,
             repo_enable_locking=enable_locking))
     except validation_schema.Invalid as err:
         raise JSONRPCValidationError(colander_exc=err)
 
     try:
         data = {
             'owner': owner,
             'repo_name': schema_data['repo_group']['repo_name_without_group'],
             'repo_name_full': schema_data['repo_name'],
             'repo_group': schema_data['repo_group']['repo_group_id'],
             'repo_type': schema_data['repo_type'],
             'repo_description': schema_data['repo_description'],
             'repo_private': schema_data['repo_private'],
             'clone_uri': schema_data['repo_clone_uri'],
             'push_uri': schema_data['repo_push_uri'],
             'repo_landing_rev': schema_data['repo_landing_commit_ref'],
             'enable_statistics': schema_data['repo_enable_statistics'],
             'enable_locking': schema_data['repo_enable_locking'],
             'enable_downloads': schema_data['repo_enable_downloads'],
             'repo_copy_permissions': schema_data['repo_copy_permissions'],
         }
 
         task = RepoModel().create(form_data=data, cur_user=owner.user_id)
         task_id = get_task_id(task)
         # no commit, it's done in RepoModel, or async via celery
         return {
             'msg': "Created new repository `%s`" % (schema_data['repo_name'],),
             'success': True,  # cannot return the repo data here since fork
             # can be done async
             'task': task_id
         }
     except Exception:
         log.exception(
             u"Exception while trying to create the repository %s",
             schema_data['repo_name'])
         raise JSONRPCError(
             'failed to create repository `%s`' % (schema_data['repo_name'],))
 
 
 @jsonrpc_method()
 def add_field_to_repo(request, apiuser, repoid, key, label=Optional(''),
                       description=Optional('')):
     """
     Adds an extra field to a repository.
 
     This command can only be run using an |authtoken| with at least
     write permissions to the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository id.
     :type repoid: str or int
     :param key: Create a unique field key for this repository.
     :type key: str
     :param label:
     :type label: Optional(str)
     :param description:
     :type description: Optional(str)
     """
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     label = Optional.extract(label) or key
     description = Optional.extract(description)
 
     field = RepositoryField.get_by_key_name(key, repo)
     if field:
         raise JSONRPCError('Field with key '
                            '`%s` exists for repo `%s`' % (key, repoid))
 
     try:
         RepoModel().add_repo_field(repo, key, field_label=label,
                                    field_desc=description)
         Session().commit()
         return {
             'msg': "Added new repository field `%s`" % (key,),
             'success': True,
         }
     except Exception:
         log.exception("Exception occurred while trying to add field to repo")
         raise JSONRPCError(
             'failed to create new field for repository `%s`' % (repoid,))
 
 
 @jsonrpc_method()
 def remove_field_from_repo(request, apiuser, repoid, key):
     """
     Removes an extra field from a repository.
 
     This command can only be run using an |authtoken| with at least
     write permissions to the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository ID.
     :type repoid: str or int
     :param key: Set the unique field key for this repository.
     :type key: str
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     field = RepositoryField.get_by_key_name(key, repo)
     if not field:
         raise JSONRPCError('Field with key `%s` does not '
                            'exists for repo `%s`' % (key, repoid))
 
     try:
         RepoModel().delete_repo_field(repo, field_key=key)
         Session().commit()
         return {
             'msg': "Deleted repository field `%s`" % (key,),
             'success': True,
         }
     except Exception:
         log.exception(
             "Exception occurred while trying to delete field from repo")
         raise JSONRPCError(
             'failed to delete field for repository `%s`' % (repoid,))
 
 
 @jsonrpc_method()
 def update_repo(
         request, apiuser, repoid, repo_name=Optional(None),
         owner=Optional(OAttr('apiuser')), description=Optional(''),
         private=Optional(False),
         clone_uri=Optional(None), push_uri=Optional(None),
         landing_rev=Optional(None), fork_of=Optional(None),
         enable_statistics=Optional(False),
         enable_locking=Optional(False),
         enable_downloads=Optional(False), fields=Optional('')):
     """
     Updates a repository with the given information.
 
     This command can only be run using an |authtoken| with at least
     admin permissions to the |repo|.
 
     * If the repository name contains "/", repository will be updated
       accordingly with a repository group or nested repository groups
 
       For example repoid=repo-test name="foo/bar/repo-test" will update |repo|
       called "repo-test" and place it inside group "foo/bar".
       You have to have permissions to access and write to the last repository
       group ("bar" in this example)
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: repository name or repository ID.
     :type repoid: str or int
     :param repo_name: Update the |repo| name, including the
         repository group it's in.
     :type repo_name: str
     :param owner: Set the |repo| owner.
     :type owner: str
     :param fork_of: Set the |repo| as fork of another |repo|.
     :type fork_of: str
     :param description: Update the |repo| description.
     :type description: str
     :param private: Set the |repo| as private. (True | False)
     :type private: bool
     :param clone_uri: Update the |repo| clone URI.
     :type clone_uri: str
     :param landing_rev: Set the |repo| landing revision. e.g branch:default, book:dev, rev:abcd
     :type landing_rev: str
     :param enable_statistics: Enable statistics on the |repo|, (True | False).
     :type enable_statistics: bool
     :param enable_locking: Enable |repo| locking.
     :type enable_locking: bool
     :param enable_downloads: Enable downloads from the |repo|, (True | False).
     :type enable_downloads: bool
     :param fields: Add extra fields to the |repo|. Use the following
         example format: ``field_key=field_val,field_key2=fieldval2``.
         Escape ', ' with \,
     :type fields: str
     """
 
     repo = get_repo_or_error(repoid)
 
     include_secrets = False
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
     else:
         include_secrets = True
 
     updates = dict(
         repo_name=repo_name
         if not isinstance(repo_name, Optional) else repo.repo_name,
 
         fork_id=fork_of
         if not isinstance(fork_of, Optional) else repo.fork.repo_name if repo.fork else None,
 
         user=owner
         if not isinstance(owner, Optional) else repo.user.username,
 
         repo_description=description
         if not isinstance(description, Optional) else repo.description,
 
         repo_private=private
         if not isinstance(private, Optional) else repo.private,
 
         clone_uri=clone_uri
         if not isinstance(clone_uri, Optional) else repo.clone_uri,
 
         push_uri=push_uri
         if not isinstance(push_uri, Optional) else repo.push_uri,
 
         repo_landing_rev=landing_rev
         if not isinstance(landing_rev, Optional) else repo._landing_revision,
 
         repo_enable_statistics=enable_statistics
         if not isinstance(enable_statistics, Optional) else repo.enable_statistics,
 
         repo_enable_locking=enable_locking
         if not isinstance(enable_locking, Optional) else repo.enable_locking,
 
         repo_enable_downloads=enable_downloads
         if not isinstance(enable_downloads, Optional) else repo.enable_downloads)
 
     landing_ref, _label = ScmModel.backend_landing_ref(repo.repo_type)
     ref_choices, _labels = ScmModel().get_repo_landing_revs(
         request.translate, repo=repo)
     ref_choices = list(set(ref_choices + [landing_ref]))
 
     old_values = repo.get_api_data()
     repo_type = repo.repo_type
     schema = repo_schema.RepoSchema().bind(
         repo_type_options=rhodecode.BACKENDS.keys(),
         repo_ref_options=ref_choices,
         repo_type=repo_type,
         # user caller
         user=apiuser,
         old_values=old_values)
     try:
         schema_data = schema.deserialize(dict(
             # we save old value, users cannot change type
             repo_type=repo_type,
 
             repo_name=updates['repo_name'],
             repo_owner=updates['user'],
             repo_description=updates['repo_description'],
             repo_clone_uri=updates['clone_uri'],
             repo_push_uri=updates['push_uri'],
             repo_fork_of=updates['fork_id'],
             repo_private=updates['repo_private'],
             repo_landing_commit_ref=updates['repo_landing_rev'],
             repo_enable_statistics=updates['repo_enable_statistics'],
             repo_enable_downloads=updates['repo_enable_downloads'],
             repo_enable_locking=updates['repo_enable_locking']))
     except validation_schema.Invalid as err:
         raise JSONRPCValidationError(colander_exc=err)
 
     # save validated data back into the updates dict
     validated_updates = dict(
         repo_name=schema_data['repo_group']['repo_name_without_group'],
         repo_group=schema_data['repo_group']['repo_group_id'],
 
         user=schema_data['repo_owner'],
         repo_description=schema_data['repo_description'],
         repo_private=schema_data['repo_private'],
         clone_uri=schema_data['repo_clone_uri'],
         push_uri=schema_data['repo_push_uri'],
         repo_landing_rev=schema_data['repo_landing_commit_ref'],
         repo_enable_statistics=schema_data['repo_enable_statistics'],
         repo_enable_locking=schema_data['repo_enable_locking'],
         repo_enable_downloads=schema_data['repo_enable_downloads'],
     )
 
     if schema_data['repo_fork_of']:
         fork_repo = get_repo_or_error(schema_data['repo_fork_of'])
         validated_updates['fork_id'] = fork_repo.repo_id
 
     # extra fields
     fields = parse_args(Optional.extract(fields), key_prefix='ex_')
     if fields:
         validated_updates.update(fields)
 
     try:
         RepoModel().update(repo, **validated_updates)
         audit_logger.store_api(
             'repo.edit', action_data={'old_data': old_values},
             user=apiuser, repo=repo)
         Session().commit()
         return {
             'msg': 'updated repo ID:%s %s' % (repo.repo_id, repo.repo_name),
             'repository': repo.get_api_data(include_secrets=include_secrets)
         }
     except Exception:
         log.exception(
             u"Exception while trying to update the repository %s",
             repoid)
         raise JSONRPCError('failed to update repo `%s`' % repoid)
 
 
 @jsonrpc_method()
 def fork_repo(request, apiuser, repoid, fork_name,
               owner=Optional(OAttr('apiuser')),
               description=Optional(''),
               private=Optional(False),
               clone_uri=Optional(None),
               landing_rev=Optional(None),
               copy_permissions=Optional(False)):
     """
     Creates a fork of the specified |repo|.
 
     * If the fork_name contains "/", fork will be created inside
       a repository group or nested repository groups
 
       For example "foo/bar/fork-repo" will create fork called "fork-repo"
       inside group "foo/bar". You have to have permissions to access and
       write to the last repository group ("bar" in this example)
 
     This command can only be run using an |authtoken| with minimum
     read permissions of the forked repo, create fork permissions for an user.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set repository name or repository ID.
     :type repoid: str or int
     :param fork_name: Set the fork name, including it's repository group membership.
     :type fork_name: str
     :param owner: Set the fork owner.
     :type owner: str
     :param description: Set the fork description.
     :type description: str
     :param copy_permissions: Copy permissions from parent |repo|. The
         default is False.
     :type copy_permissions: bool
     :param private: Make the fork private. The default is False.
     :type private: bool
     :param landing_rev: Set the landing revision. E.g branch:default, book:dev, rev:abcd
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_for_response>
         api_key : "<api_key>"
         args:     {
                     "repoid" :          "<reponame or repo_id>",
                     "fork_name":        "<forkname>",
                     "owner":            "<username or user_id = Optional(=apiuser)>",
                     "description":      "<description>",
                     "copy_permissions": "<bool>",
                     "private":          "<bool>",
                     "landing_rev":      "<landing_rev>"
                   }
 
     Example error output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: {
                   "msg": "Created fork of `<reponame>` as `<forkname>`",
                   "success": true,
                   "task": "<celery task id or None if done sync>"
                 }
         error:  null
 
     """
 
     repo = get_repo_or_error(repoid)
     repo_name = repo.repo_name
 
     if not has_superadmin_permission(apiuser):
         # check if we have at least read permission for
         # this repo that we fork !
         _perms = ('repository.admin', 'repository.write', 'repository.read')
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
         # check if the regular user has at least fork permissions as well
         if not HasPermissionAnyApi('hg.fork.repository')(user=apiuser):
             raise JSONRPCForbidden()
 
     # check if user can set owner parameter
     owner = validate_set_owner_permissions(apiuser, owner)
 
     description = Optional.extract(description)
     copy_permissions = Optional.extract(copy_permissions)
     clone_uri = Optional.extract(clone_uri)
 
     landing_ref, _label = ScmModel.backend_landing_ref(repo.repo_type)
     ref_choices, _labels = ScmModel().get_repo_landing_revs(request.translate)
     ref_choices = list(set(ref_choices + [landing_ref]))
     landing_commit_ref = Optional.extract(landing_rev) or landing_ref
 
     private = Optional.extract(private)
 
     schema = repo_schema.RepoSchema().bind(
         repo_type_options=rhodecode.BACKENDS.keys(),
         repo_ref_options=ref_choices,
         repo_type=repo.repo_type,
         # user caller
         user=apiuser)
 
     try:
         schema_data = schema.deserialize(dict(
             repo_name=fork_name,
             repo_type=repo.repo_type,
             repo_owner=owner.username,
             repo_description=description,
             repo_landing_commit_ref=landing_commit_ref,
             repo_clone_uri=clone_uri,
             repo_private=private,
             repo_copy_permissions=copy_permissions))
     except validation_schema.Invalid as err:
         raise JSONRPCValidationError(colander_exc=err)
 
     try:
         data = {
             'fork_parent_id': repo.repo_id,
 
             'repo_name': schema_data['repo_group']['repo_name_without_group'],
             'repo_name_full': schema_data['repo_name'],
             'repo_group': schema_data['repo_group']['repo_group_id'],
             'repo_type': schema_data['repo_type'],
             'description': schema_data['repo_description'],
             'private': schema_data['repo_private'],
             'copy_permissions': schema_data['repo_copy_permissions'],
             'landing_rev': schema_data['repo_landing_commit_ref'],
         }
 
         task = RepoModel().create_fork(data, cur_user=owner.user_id)
         # no commit, it's done in RepoModel, or async via celery
         task_id = get_task_id(task)
 
         return {
             'msg': 'Created fork of `%s` as `%s`' % (
                 repo.repo_name, schema_data['repo_name']),
             'success': True,  # cannot return the repo data here since fork
             # can be done async
             'task': task_id
         }
     except Exception:
         log.exception(
             u"Exception while trying to create fork %s",
             schema_data['repo_name'])
         raise JSONRPCError(
             'failed to fork repository `%s` as `%s`' % (
                 repo_name, schema_data['repo_name']))
 
 
 @jsonrpc_method()
 def delete_repo(request, apiuser, repoid, forks=Optional('')):
     """
     Deletes a repository.
 
     * When the `forks` parameter is set it's possible to detach or delete
       forks of deleted repository.
 
     This command can only be run using an |authtoken| with admin
     permissions on the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository ID.
     :type repoid: str or int
     :param forks: Set to `detach` or `delete` forks from the |repo|.
     :type forks: Optional(str)
 
     Example error output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: {
                   "msg": "Deleted repository `<reponame>`",
                   "success": true
                 }
         error:  null
     """
 
     repo = get_repo_or_error(repoid)
     repo_name = repo.repo_name
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     try:
         handle_forks = Optional.extract(forks)
         _forks_msg = ''
         _forks = [f for f in repo.forks]
         if handle_forks == 'detach':
             _forks_msg = ' ' + 'Detached %s forks' % len(_forks)
         elif handle_forks == 'delete':
             _forks_msg = ' ' + 'Deleted %s forks' % len(_forks)
         elif _forks:
             raise JSONRPCError(
                 'Cannot delete `%s` it still contains attached forks' %
                 (repo.repo_name,)
             )
         old_data = repo.get_api_data()
         RepoModel().delete(repo, forks=forks)
 
         repo = audit_logger.RepoWrap(repo_id=None,
                                      repo_name=repo.repo_name)
 
         audit_logger.store_api(
             'repo.delete', action_data={'old_data': old_data},
             user=apiuser, repo=repo)
 
         ScmModel().mark_for_invalidation(repo_name, delete=True)
         Session().commit()
         return {
             'msg': 'Deleted repository `%s`%s' % (repo_name, _forks_msg),
             'success': True
         }
     except Exception:
         log.exception("Exception occurred while trying to delete repo")
         raise JSONRPCError(
             'failed to delete repository `%s`' % (repo_name,)
         )
 
 
 #TODO: marcink, change name ?
 @jsonrpc_method()
 def invalidate_cache(request, apiuser, repoid, delete_keys=Optional(False)):
     """
     Invalidates the cache for the specified repository.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Sets the repository name or repository ID.
     :type repoid: str or int
     :param delete_keys: This deletes the invalidated keys instead of
         just flagging them.
     :type delete_keys: Optional(``True`` | ``False``)
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         'msg': Cache for repository `<repository name>` was invalidated,
         'repository': <repository name>
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error : {
          'Error occurred during cache invalidation action'
       }
 
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin', 'repository.write',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     delete = Optional.extract(delete_keys)
     try:
         ScmModel().mark_for_invalidation(repo.repo_name, delete=delete)
         return {
             'msg': 'Cache for repository `%s` was invalidated' % (repoid,),
             'repository': repo.repo_name
         }
     except Exception:
         log.exception(
             "Exception occurred while trying to invalidate repo cache")
         raise JSONRPCError(
             'Error occurred during cache invalidation action'
         )
 
 
 #TODO: marcink, change name ?
 @jsonrpc_method()
 def lock(request, apiuser, repoid, locked=Optional(None),
          userid=Optional(OAttr('apiuser'))):
     """
     Sets the lock state of the specified |repo| by the given user.
     From more information, see :ref:`repo-locking`.
 
     * If the ``userid`` option is not set, the repository is locked to the
       user who called the method.
     * If the ``locked`` parameter is not set, the current lock state of the
       repository is displayed.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Sets the repository name or repository ID.
     :type repoid: str or int
     :param locked: Sets the lock state.
     :type locked: Optional(``True`` | ``False``)
     :param userid: Set the repository lock to this user.
     :type userid: Optional(str or int)
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         'repo': '<reponame>',
         'locked': <bool: lock state>,
         'locked_since': <int: lock timestamp>,
         'locked_by': <username of person who made the lock>,
         'lock_reason': <str: reason for locking>,
         'lock_state_changed': <bool: True if lock state has been changed in this request>,
         'msg': 'Repo `<reponame>` locked by `<username>` on <timestamp>.'
         or
         'msg': 'Repo `<repository name>` not locked.'
         or
         'msg': 'User `<user name>` set lock state for repo `<repository name>` to `<new lock state>`'
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         'Error occurred locking repository `<reponame>`'
       }
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         # check if we have at least write permission for this repo !
         _perms = ('repository.admin', 'repository.write',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
         # make sure normal user does not pass someone else userid,
         # he is not allowed to do that
         if not isinstance(userid, Optional) and userid != apiuser.user_id:
             raise JSONRPCError('userid is not the same as your user')
 
     if isinstance(userid, Optional):
         userid = apiuser.user_id
 
     user = get_user_or_error(userid)
 
     if isinstance(locked, Optional):
         lockobj = repo.locked
 
         if lockobj[0] is None:
             _d = {
                 'repo': repo.repo_name,
                 'locked': False,
                 'locked_since': None,
                 'locked_by': None,
                 'lock_reason': None,
                 'lock_state_changed': False,
                 'msg': 'Repo `%s` not locked.' % repo.repo_name
             }
             return _d
         else:
             _user_id, _time, _reason = lockobj
             lock_user = get_user_or_error(userid)
             _d = {
                 'repo': repo.repo_name,
                 'locked': True,
                 'locked_since': _time,
                 'locked_by': lock_user.username,
                 'lock_reason': _reason,
                 'lock_state_changed': False,
                 'msg': ('Repo `%s` locked by `%s` on `%s`.'
                         % (repo.repo_name, lock_user.username,
                            json.dumps(time_to_datetime(_time))))
             }
             return _d
 
     # force locked state through a flag
     else:
         locked = str2bool(locked)
         lock_reason = Repository.LOCK_API
         try:
             if locked:
                 lock_time = time.time()
                 Repository.lock(repo, user.user_id, lock_time, lock_reason)
             else:
                 lock_time = None
                 Repository.unlock(repo)
             _d = {
                 'repo': repo.repo_name,
                 'locked': locked,
                 'locked_since': lock_time,
                 'locked_by': user.username,
                 'lock_reason': lock_reason,
                 'lock_state_changed': True,
                 'msg': ('User `%s` set lock state for repo `%s` to `%s`'
                         % (user.username, repo.repo_name, locked))
             }
             return _d
         except Exception:
             log.exception(
                 "Exception occurred while trying to lock repository")
             raise JSONRPCError(
                 'Error occurred locking repository `%s`' % repo.repo_name
             )
 
 
 @jsonrpc_method()
 def comment_commit(
         request, apiuser, repoid, commit_id, message, status=Optional(None),
         comment_type=Optional(ChangesetComment.COMMENT_TYPE_NOTE),
         resolves_comment_id=Optional(None), extra_recipients=Optional([]),
         userid=Optional(OAttr('apiuser')), send_email=Optional(True)):
     """
     Set a commit comment, and optionally change the status of the commit.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository ID.
     :type repoid: str or int
     :param commit_id: Specify the commit_id for which to set a comment.
     :type commit_id: str
     :param message: The comment text.
     :type message: str
     :param status: (**Optional**) status of commit, one of: 'not_reviewed',
         'approved', 'rejected', 'under_review'
     :type status: str
     :param comment_type: Comment type, one of: 'note', 'todo'
     :type comment_type: Optional(str), default: 'note'
     :param resolves_comment_id: id of comment which this one will resolve
     :type resolves_comment_id: Optional(int)
     :param extra_recipients: list of user ids or usernames to add
         notifications for this comment. Acts like a CC for notification
     :type extra_recipients: Optional(list)
     :param userid: Set the user name of the comment creator.
     :type userid: Optional(str or int)
     :param send_email: Define if this comment should also send email notification
     :type send_email: Optional(bool)
 
     Example error output:
 
     .. code-block:: bash
 
         {
             "id" : <id_given_in_input>,
             "result" : {
                 "msg": "Commented on commit `<commit_id>` for repository `<repoid>`",
                 "status_change": null or <status>,
                 "success": true
             },
             "error" :  null
         }
 
     """
     _ = request.translate
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.read', 'repository.write', 'repository.admin')
         validate_repo_permissions(apiuser, repoid, repo, _perms)
     db_repo_name = repo.repo_name
 
     try:
         commit = repo.scm_instance().get_commit(commit_id=commit_id)
         commit_id = commit.raw_id
     except Exception as e:
         log.exception('Failed to fetch commit')
         raise JSONRPCError(safe_str(e))
 
     if isinstance(userid, Optional):
         userid = apiuser.user_id
 
     user = get_user_or_error(userid)
     status = Optional.extract(status)
     comment_type = Optional.extract(comment_type)
     resolves_comment_id = Optional.extract(resolves_comment_id)
     extra_recipients = Optional.extract(extra_recipients)
     send_email = Optional.extract(send_email, binary=True)
 
     allowed_statuses = [x[0] for x in ChangesetStatus.STATUSES]
     if status and status not in allowed_statuses:
         raise JSONRPCError('Bad status, must be on '
                            'of %s got %s' % (allowed_statuses, status,))
 
     if resolves_comment_id:
         comment = ChangesetComment.get(resolves_comment_id)
         if not comment:
             raise JSONRPCError(
                 'Invalid resolves_comment_id `%s` for this commit.'
                 % resolves_comment_id)
         if comment.comment_type != ChangesetComment.COMMENT_TYPE_TODO:
             raise JSONRPCError(
                 'Comment `%s` is wrong type for setting status to resolved.'
                 % resolves_comment_id)
 
     try:
         rc_config = SettingsModel().get_all_settings()
         renderer = rc_config.get('rhodecode_markup_renderer', 'rst')
         status_change_label = ChangesetStatus.get_status_lbl(status)
         comment = CommentsModel().create(
             message, repo, user, commit_id=commit_id,
             status_change=status_change_label,
             status_change_type=status,
             renderer=renderer,
             comment_type=comment_type,
             resolves_comment_id=resolves_comment_id,
             auth_user=apiuser,
             extra_recipients=extra_recipients,
             send_email=send_email
         )
         is_inline = comment.is_inline
 
         if status:
             # also do a status change
             try:
                 ChangesetStatusModel().set_status(
                     repo, status, user, comment, revision=commit_id,
                     dont_allow_on_closed_pull_request=True
                 )
             except StatusChangeOnClosedPullRequestError:
                 log.exception(
                     "Exception occurred while trying to change repo commit status")
                 msg = ('Changing status on a commit associated with '
                        'a closed pull request is not allowed')
                 raise JSONRPCError(msg)
 
         CommentsModel().trigger_commit_comment_hook(
             repo, apiuser, 'create',
             data={'comment': comment, 'commit': commit})
 
         Session().commit()
 
         comment_broadcast_channel = channelstream.comment_channel(
             db_repo_name, commit_obj=commit)
 
         comment_data = {'comment': comment, 'comment_id': comment.comment_id}
         comment_type = 'inline' if is_inline else 'general'
         channelstream.comment_channelstream_push(
             request, comment_broadcast_channel, apiuser,
             _('posted a new {} comment').format(comment_type),
             comment_data=comment_data)
 
         return {
             'msg': (
                 'Commented on commit `%s` for repository `%s`' % (
                     comment.revision, repo.repo_name)),
             'status_change': status,
             'success': True,
         }
     except JSONRPCError:
         # catch any inside errors, and re-raise them to prevent from
         # below global catch to silence them
         raise
     except Exception:
         log.exception("Exception occurred while trying to comment on commit")
         raise JSONRPCError(
             'failed to set comment on repository `%s`' % (repo.repo_name,)
         )
 
 
 @jsonrpc_method()
 def get_repo_comments(request, apiuser, repoid,
                       commit_id=Optional(None), comment_type=Optional(None),
                       userid=Optional(None)):
     """
     Get all comments for a repository
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository ID.
     :type repoid: str or int
     :param commit_id: Optionally filter the comments by the commit_id
     :type commit_id: Optional(str), default: None
     :param comment_type: Optionally filter the comments by the comment_type
         one of: 'note', 'todo'
     :type comment_type: Optional(str), default: None
     :param userid: Optionally filter the comments by the author of comment
     :type userid: Optional(str or int), Default: None
 
     Example error output:
 
     .. code-block:: bash
 
         {
             "id" : <id_given_in_input>,
             "result" : [
                 {
                   "comment_author": <USER_DETAILS>,
                   "comment_created_on": "2017-02-01T14:38:16.309",
                   "comment_f_path": "file.txt",
                   "comment_id": 282,
                   "comment_lineno": "n1",
                   "comment_resolved_by": null,
                   "comment_status": [],
                   "comment_text": "This file needs a header",
                   "comment_type": "todo",
                   "comment_last_version: 0
                 }
             ],
             "error" :  null
         }
 
     """
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.read', 'repository.write', 'repository.admin')
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     commit_id = Optional.extract(commit_id)
 
     userid = Optional.extract(userid)
     if userid:
         user = get_user_or_error(userid)
     else:
         user = None
 
     comment_type = Optional.extract(comment_type)
     if comment_type and comment_type not in ChangesetComment.COMMENT_TYPES:
         raise JSONRPCError(
                 'comment_type must be one of `{}` got {}'.format(
                     ChangesetComment.COMMENT_TYPES, comment_type)
             )
 
     comments = CommentsModel().get_repository_comments(
         repo=repo, comment_type=comment_type, user=user, commit_id=commit_id)
     return comments
 
 
 @jsonrpc_method()
 def get_comment(request, apiuser, comment_id):
     """
     Get single comment from repository or pull_request
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param comment_id: comment id found in the URL of comment
     :type comment_id: str or int
 
     Example error output:
 
     .. code-block:: bash
 
         {
             "id" : <id_given_in_input>,
             "result" : {
                   "comment_author": <USER_DETAILS>,
                   "comment_created_on": "2017-02-01T14:38:16.309",
                   "comment_f_path": "file.txt",
                   "comment_id": 282,
                   "comment_lineno": "n1",
                   "comment_resolved_by": null,
                   "comment_status": [],
                   "comment_text": "This file needs a header",
                   "comment_type": "todo",
                   "comment_last_version: 0
             },
             "error" :  null
         }
 
     """
 
     comment = ChangesetComment.get(comment_id)
     if not comment:
         raise JSONRPCError('comment `%s` does not exist' % (comment_id,))
 
     perms = ('repository.read', 'repository.write', 'repository.admin')
     has_comment_perm = HasRepoPermissionAnyApi(*perms)\
         (user=apiuser, repo_name=comment.repo.repo_name)
 
     if not has_comment_perm:
         raise JSONRPCError('comment `%s` does not exist' % (comment_id,))
 
     return comment
 
 
 @jsonrpc_method()
 def edit_comment(request, apiuser, message, comment_id, version,
                  userid=Optional(OAttr('apiuser'))):
     """
     Edit comment on the pull request or commit,
     specified by the `comment_id` and version. Initially version should be 0
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param comment_id: Specify the comment_id for editing
     :type comment_id: int
     :param version: version of the comment that will be created, starts from 0
     :type version: int
     :param message: The text content of the comment.
     :type message: str
     :param userid: Comment on the pull request as this user
     :type userid: Optional(str or int)
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result : {
             "comment": "<comment data>",
             "version": "<Integer>",
         },
         error :  null
     """
 
     auth_user = apiuser
     comment = ChangesetComment.get(comment_id)
     if not comment:
         raise JSONRPCError('comment `%s` does not exist' % (comment_id,))
 
     is_super_admin = has_superadmin_permission(apiuser)
     is_repo_admin = HasRepoPermissionAnyApi('repository.admin')\
         (user=apiuser, repo_name=comment.repo.repo_name)
 
     if not isinstance(userid, Optional):
         if is_super_admin or is_repo_admin:
             apiuser = get_user_or_error(userid)
             auth_user = apiuser.AuthUser()
         else:
             raise JSONRPCError('userid is not the same as your user')
 
     comment_author = comment.author.user_id == auth_user.user_id
     if not (comment.immutable is False and (is_super_admin or is_repo_admin) or comment_author):
         raise JSONRPCError("you don't have access to edit this comment")
 
     try:
         comment_history = CommentsModel().edit(
             comment_id=comment_id,
             text=message,
             auth_user=auth_user,
             version=version,
         )
         Session().commit()
     except CommentVersionMismatch:
         raise JSONRPCError(
             'comment ({}) version ({}) mismatch'.format(comment_id, version)
         )
     if not comment_history and not message:
         raise JSONRPCError(
             "comment ({}) can't be changed with empty string".format(comment_id)
         )
 
     if comment.pull_request:
         pull_request = comment.pull_request
         PullRequestModel().trigger_pull_request_hook(
             pull_request, apiuser, 'comment_edit',
             data={'comment': comment})
     else:
         db_repo = comment.repo
         commit_id = comment.revision
         commit = db_repo.get_commit(commit_id)
         CommentsModel().trigger_commit_comment_hook(
             db_repo, apiuser, 'edit',
             data={'comment': comment, 'commit': commit})
 
     data = {
         'comment': comment,
         'version': comment_history.version if comment_history else None,
     }
     return data
 
 
 # TODO(marcink): write this with all required logic for deleting a comments in PR or commits
 # @jsonrpc_method()
 # def delete_comment(request, apiuser, comment_id):
 #     auth_user = apiuser
 #
 #     comment = ChangesetComment.get(comment_id)
 #     if not comment:
 #         raise JSONRPCError('comment `%s` does not exist' % (comment_id,))
 #
 #     is_super_admin = has_superadmin_permission(apiuser)
 #     is_repo_admin = HasRepoPermissionAnyApi('repository.admin')\
 #         (user=apiuser, repo_name=comment.repo.repo_name)
 #
 #     comment_author = comment.author.user_id == auth_user.user_id
 #     if not (comment.immutable is False and (is_super_admin or is_repo_admin) or comment_author):
 #         raise JSONRPCError("you don't have access to edit this comment")
 
 @jsonrpc_method()
 def grant_user_permission(request, apiuser, repoid, userid, perm):
     """
     Grant permissions for the specified user on the given repository,
     or update existing permissions if found.
 
     This command can only be run using an |authtoken| with admin
     permissions on the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository ID.
     :type repoid: str or int
     :param userid: Set the user name.
     :type userid: str
     :param perm: Set the user permissions, using the following format
         ``(repository.(none|read|write|admin))``
     :type perm: str
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: {
                   "msg" : "Granted perm: `<perm>` for user: `<username>` in repo: `<reponame>`",
                   "success": true
                 }
         error:  null
     """
 
     repo = get_repo_or_error(repoid)
     user = get_user_or_error(userid)
     perm = get_perm_or_error(perm)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     perm_additions = [[user.user_id, perm.permission_name, "user"]]
     try:
         changes = RepoModel().update_permissions(
             repo=repo, perm_additions=perm_additions, cur_user=apiuser)
 
         action_data = {
             'added': changes['added'],
             'updated': changes['updated'],
             'deleted': changes['deleted'],
         }
         audit_logger.store_api(
             'repo.edit.permissions', action_data=action_data, user=apiuser, repo=repo)
         Session().commit()
         PermissionModel().flush_user_permission_caches(changes)
 
         return {
             'msg': 'Granted perm: `%s` for user: `%s` in repo: `%s`' % (
                 perm.permission_name, user.username, repo.repo_name
             ),
             'success': True
         }
     except Exception:
         log.exception("Exception occurred while trying edit permissions for repo")
         raise JSONRPCError(
             'failed to edit permission for user: `%s` in repo: `%s`' % (
                 userid, repoid
             )
         )
 
 
 @jsonrpc_method()
 def revoke_user_permission(request, apiuser, repoid, userid):
     """
     Revoke permission for a user on the specified repository.
 
     This command can only be run using an |authtoken| with admin
     permissions on the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository ID.
     :type repoid: str or int
     :param userid: Set the user name of revoked user.
     :type userid: str or int
 
     Example error output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: {
                   "msg" : "Revoked perm for user: `<username>` in repo: `<reponame>`",
                   "success": true
                 }
         error:  null
     """
 
     repo = get_repo_or_error(repoid)
     user = get_user_or_error(userid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     perm_deletions = [[user.user_id, None, "user"]]
     try:
         changes = RepoModel().update_permissions(
             repo=repo, perm_deletions=perm_deletions, cur_user=user)
 
         action_data = {
             'added': changes['added'],
             'updated': changes['updated'],
             'deleted': changes['deleted'],
         }
         audit_logger.store_api(
             'repo.edit.permissions', action_data=action_data, user=apiuser, repo=repo)
         Session().commit()
         PermissionModel().flush_user_permission_caches(changes)
 
         return {
             'msg': 'Revoked perm for user: `%s` in repo: `%s`' % (
                 user.username, repo.repo_name
             ),
             'success': True
         }
     except Exception:
         log.exception("Exception occurred while trying revoke permissions to repo")
         raise JSONRPCError(
             'failed to edit permission for user: `%s` in repo: `%s`' % (
                 userid, repoid
             )
         )
 
 
 @jsonrpc_method()
 def grant_user_group_permission(request, apiuser, repoid, usergroupid, perm):
     """
     Grant permission for a user group on the specified repository,
     or update existing permissions.
 
     This command can only be run using an |authtoken| with admin
     permissions on the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository ID.
     :type repoid: str or int
     :param usergroupid: Specify the ID of the user group.
     :type usergroupid: str or int
     :param perm: Set the user group permissions using the following
         format: (repository.(none|read|write|admin))
     :type perm: str
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         "msg" : "Granted perm: `<perm>` for group: `<usersgroupname>` in repo: `<reponame>`",
         "success": true
 
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         "failed to edit permission for user group: `<usergroup>` in repo `<repo>`'
       }
 
     """
 
     repo = get_repo_or_error(repoid)
     perm = get_perm_or_error(perm)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     user_group = get_user_group_or_error(usergroupid)
     if not has_superadmin_permission(apiuser):
         # check if we have at least read permission for this user group !
         _perms = ('usergroup.read', 'usergroup.write', 'usergroup.admin',)
         if not HasUserGroupPermissionAnyApi(*_perms)(
                 user=apiuser, user_group_name=user_group.users_group_name):
             raise JSONRPCError(
                 'user group `%s` does not exist' % (usergroupid,))
 
     perm_additions = [[user_group.users_group_id, perm.permission_name, "user_group"]]
     try:
         changes = RepoModel().update_permissions(
             repo=repo, perm_additions=perm_additions, cur_user=apiuser)
         action_data = {
             'added': changes['added'],
             'updated': changes['updated'],
             'deleted': changes['deleted'],
         }
         audit_logger.store_api(
             'repo.edit.permissions', action_data=action_data, user=apiuser, repo=repo)
         Session().commit()
         PermissionModel().flush_user_permission_caches(changes)
 
         return {
             'msg': 'Granted perm: `%s` for user group: `%s` in '
                    'repo: `%s`' % (
                        perm.permission_name, user_group.users_group_name,
                        repo.repo_name
                    ),
             'success': True
         }
     except Exception:
         log.exception(
             "Exception occurred while trying change permission on repo")
         raise JSONRPCError(
             'failed to edit permission for user group: `%s` in '
             'repo: `%s`' % (
                 usergroupid, repo.repo_name
             )
         )
 
 
 @jsonrpc_method()
 def revoke_user_group_permission(request, apiuser, repoid, usergroupid):
     """
     Revoke the permissions of a user group on a given repository.
 
     This command can only be run using an |authtoken| with admin
     permissions on the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository ID.
     :type repoid: str or int
     :param usergroupid: Specify the user group ID.
     :type usergroupid: str or int
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: {
                   "msg" : "Revoked perm for group: `<usersgroupname>` in repo: `<reponame>`",
                   "success": true
                 }
         error:  null
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     user_group = get_user_group_or_error(usergroupid)
     if not has_superadmin_permission(apiuser):
         # check if we have at least read permission for this user group !
         _perms = ('usergroup.read', 'usergroup.write', 'usergroup.admin',)
         if not HasUserGroupPermissionAnyApi(*_perms)(
                 user=apiuser, user_group_name=user_group.users_group_name):
             raise JSONRPCError(
                 'user group `%s` does not exist' % (usergroupid,))
 
     perm_deletions = [[user_group.users_group_id, None, "user_group"]]
     try:
         changes = RepoModel().update_permissions(
             repo=repo, perm_deletions=perm_deletions, cur_user=apiuser)
         action_data = {
             'added': changes['added'],
             'updated': changes['updated'],
             'deleted': changes['deleted'],
         }
         audit_logger.store_api(
             'repo.edit.permissions', action_data=action_data, user=apiuser, repo=repo)
         Session().commit()
         PermissionModel().flush_user_permission_caches(changes)
 
         return {
             'msg': 'Revoked perm for user group: `%s` in repo: `%s`' % (
                 user_group.users_group_name, repo.repo_name
             ),
             'success': True
         }
     except Exception:
         log.exception("Exception occurred while trying revoke "
                       "user group permission on repo")
         raise JSONRPCError(
             'failed to edit permission for user group: `%s` in '
             'repo: `%s`' % (
                 user_group.users_group_name, repo.repo_name
             )
         )
 
 
 @jsonrpc_method()
 def pull(request, apiuser, repoid, remote_uri=Optional(None)):
     """
     Triggers a pull on the given repository from a remote location. You
     can use this to keep remote repositories up-to-date.
 
     This command can only be run using an |authtoken| with admin
     rights to the specified repository. For more information,
     see :ref:`config-token-ref`.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository ID.
     :type repoid: str or int
     :param remote_uri: Optional remote URI to pass in for pull
     :type remote_uri: str
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         "msg": "Pulled from url `<remote_url>` on repo `<repository name>`"
         "repository": "<repository name>"
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         "Unable to push changes from `<remote_url>`"
       }
 
     """
 
     repo = get_repo_or_error(repoid)
     remote_uri = Optional.extract(remote_uri)
     remote_uri_display = remote_uri or repo.clone_uri_hidden
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     try:
         ScmModel().pull_changes(
             repo.repo_name, apiuser.username, remote_uri=remote_uri)
         return {
             'msg': 'Pulled from url `%s` on repo `%s`' % (
                 remote_uri_display, repo.repo_name),
             'repository': repo.repo_name
         }
     except Exception:
         log.exception("Exception occurred while trying to "
                       "pull changes from remote location")
         raise JSONRPCError(
             'Unable to pull changes from `%s`' % remote_uri_display
         )
 
 
 @jsonrpc_method()
 def strip(request, apiuser, repoid, revision, branch):
     """
     Strips the given revision from the specified repository.
 
     * This will remove the revision and all of its decendants.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository ID.
     :type repoid: str or int
     :param revision: The revision you wish to strip.
     :type revision: str
     :param branch: The branch from which to strip the revision.
     :type branch: str
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         "msg": "'Stripped commit <commit_hash> from repo `<repository name>`'"
         "repository": "<repository name>"
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         "Unable to strip commit <commit_hash> from repo `<repository name>`"
       }
 
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     try:
         ScmModel().strip(repo, revision, branch)
         audit_logger.store_api(
             'repo.commit.strip', action_data={'commit_id': revision},
             repo=repo,
             user=apiuser, commit=True)
 
         return {
             'msg': 'Stripped commit %s from repo `%s`' % (
                 revision, repo.repo_name),
             'repository': repo.repo_name
         }
     except Exception:
         log.exception("Exception while trying to strip")
         raise JSONRPCError(
             'Unable to strip commit %s from repo `%s`' % (
                 revision, repo.repo_name)
         )
 
 
 @jsonrpc_method()
 def get_repo_settings(request, apiuser, repoid, key=Optional(None)):
     """
     Returns all settings for a repository. If key is given it only returns the
     setting identified by the key or null.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository id.
     :type repoid: str or int
     :param key: Key of the setting to return.
     :type: key: Optional(str)
 
     Example output:
 
     .. code-block:: bash
 
         {
             "error": null,
             "id": 237,
             "result": {
                 "extensions_largefiles": true,
                 "extensions_evolve": true,
                 "hooks_changegroup_push_logger": true,
                 "hooks_changegroup_repo_size": false,
                 "hooks_outgoing_pull_logger": true,
                 "phases_publish": "True",
                 "rhodecode_hg_use_rebase_for_merging": true,
                 "rhodecode_pr_merge_enabled": true,
                 "rhodecode_use_outdated_comments": true
             }
         }
     """
 
     # Restrict access to this api method to super-admins, and repo admins only.
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     try:
         settings_model = VcsSettingsModel(repo=repo)
         settings = settings_model.get_global_settings()
         settings.update(settings_model.get_repo_settings())
 
         # If only a single setting is requested fetch it from all settings.
         key = Optional.extract(key)
         if key is not None:
             settings = settings.get(key, None)
     except Exception:
         msg = 'Failed to fetch settings for repository `{}`'.format(repoid)
         log.exception(msg)
         raise JSONRPCError(msg)
 
     return settings
 
 
 @jsonrpc_method()
 def set_repo_settings(request, apiuser, repoid, settings):
     """
     Update repository settings. Returns true on success.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository id.
     :type repoid: str or int
     :param settings: The new settings for the repository.
     :type: settings: dict
 
     Example output:
 
     .. code-block:: bash
 
         {
             "error": null,
             "id": 237,
             "result": true
         }
     """
     # Restrict access to this api method to super-admins, and repo admins only.
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     if type(settings) is not dict:
         raise JSONRPCError('Settings have to be a JSON Object.')
 
     try:
         settings_model = VcsSettingsModel(repo=repoid)
 
         # Merge global, repo and incoming settings.
         new_settings = settings_model.get_global_settings()
         new_settings.update(settings_model.get_repo_settings())
         new_settings.update(settings)
 
         # Update the settings.
         inherit_global_settings = new_settings.get(
             'inherit_global_settings', False)
         settings_model.create_or_update_repo_settings(
             new_settings, inherit_global_settings=inherit_global_settings)
         Session().commit()
     except Exception:
         msg = 'Failed to update settings for repository `{}`'.format(repoid)
         log.exception(msg)
         raise JSONRPCError(msg)
 
     # Indicate success.
     return True
 
 
 @jsonrpc_method()
 def maintenance(request, apiuser, repoid):
     """
     Triggers a maintenance on the given repository.
 
     This command can only be run using an |authtoken| with admin
     rights to the specified repository. For more information,
     see :ref:`config-token-ref`.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository ID.
     :type repoid: str or int
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         "msg": "executed maintenance command",
         "executed_actions": [
            <action_message>, <action_message2>...
         ],
         "repository": "<repository name>"
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         "Unable to execute maintenance on `<reponame>`"
       }
 
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     try:
         maintenance = repo_maintenance.RepoMaintenance()
         executed_actions = maintenance.execute(repo)
 
         return {
             'msg': 'executed maintenance command',
             'executed_actions': executed_actions,
             'repository': repo.repo_name
         }
     except Exception:
         log.exception("Exception occurred while trying to run maintenance")
         raise JSONRPCError(
             'Unable to execute maintenance on `%s`' % repo.repo_name)

rhodecode/apps/admin/views/users.py

@@ -1,1414 +1,1418 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import datetime
 import formencode
 import formencode.htmlfill
 
 from pyramid.httpexceptions import HTTPFound
 from pyramid.view import view_config
 from pyramid.renderers import render
 from pyramid.response import Response
 
 from rhodecode import events
 from rhodecode.apps._base import BaseAppView, DataGridAppView, UserAppView
 from rhodecode.apps.ssh_support import SshKeyFileChangeEvent
 from rhodecode.authentication.base import get_authn_registry, RhodeCodeExternalAuthPlugin
 from rhodecode.authentication.plugins import auth_rhodecode
 from rhodecode.events import trigger
 from rhodecode.model.db import true, UserNotice
 
-from rhodecode.lib import audit_logger, rc_cache
+from rhodecode.lib import audit_logger, rc_cache, auth
 from rhodecode.lib.exceptions import (
     UserCreationError, UserOwnsReposException, UserOwnsRepoGroupsException,
     UserOwnsUserGroupsException, UserOwnsPullRequestsException,
     UserOwnsArtifactsException, DefaultUserException)
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.auth import (
     LoginRequired, HasPermissionAllDecorator, CSRFRequired)
 from rhodecode.lib import helpers as h
 from rhodecode.lib.helpers import SqlPage
 from rhodecode.lib.utils2 import safe_int, safe_unicode, AttributeDict
 from rhodecode.model.auth_token import AuthTokenModel
 from rhodecode.model.forms import (
     UserForm, UserIndividualPermissionsForm, UserPermissionsForm,
     UserExtraEmailForm, UserExtraIpForm)
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.ssh_key import SshKeyModel
 from rhodecode.model.user import UserModel
 from rhodecode.model.user_group import UserGroupModel
 from rhodecode.model.db import (
     or_, coalesce,IntegrityError, User, UserGroup, UserIpMap, UserEmailMap,
     UserApiKeys, UserSshKeys, RepoGroup)
 from rhodecode.model.meta import Session
 
 log = logging.getLogger(__name__)
 
 
 class AdminUsersView(BaseAppView, DataGridAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         return c
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='users', request_method='GET',
         renderer='rhodecode:templates/admin/users/users.mako')
     def users_list(self):
         c = self.load_default_context()
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         # renderer defined below
         route_name='users_data', request_method='GET',
         renderer='json_ext', xhr=True)
     def users_list_data(self):
         self.load_default_context()
         column_map = {
             'first_name': 'name',
             'last_name': 'lastname',
         }
         draw, start, limit = self._extract_chunk(self.request)
         search_q, order_by, order_dir = self._extract_ordering(
             self.request, column_map=column_map)
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/data_table/_dt_elements.mako')
 
         def user_actions(user_id, username):
             return _render("user_actions", user_id, username)
 
         users_data_total_count = User.query()\
             .filter(User.username != User.DEFAULT_USER) \
             .count()
 
         users_data_total_inactive_count = User.query()\
             .filter(User.username != User.DEFAULT_USER) \
             .filter(User.active != true())\
             .count()
 
         # json generate
         base_q = User.query().filter(User.username != User.DEFAULT_USER)
         base_inactive_q = base_q.filter(User.active != true())
 
         if search_q:
             like_expression = u'%{}%'.format(safe_unicode(search_q))
             base_q = base_q.filter(or_(
                 User.username.ilike(like_expression),
                 User._email.ilike(like_expression),
                 User.name.ilike(like_expression),
                 User.lastname.ilike(like_expression),
             ))
             base_inactive_q = base_q.filter(User.active != true())
 
         users_data_total_filtered_count = base_q.count()
         users_data_total_filtered_inactive_count = base_inactive_q.count()
 
         sort_col = getattr(User, order_by, None)
         if sort_col:
             if order_dir == 'asc':
                 # handle null values properly to order by NULL last
                 if order_by in ['last_activity']:
                     sort_col = coalesce(sort_col, datetime.date.max)
                 sort_col = sort_col.asc()
             else:
                 # handle null values properly to order by NULL last
                 if order_by in ['last_activity']:
                     sort_col = coalesce(sort_col, datetime.date.min)
                 sort_col = sort_col.desc()
 
         base_q = base_q.order_by(sort_col)
         base_q = base_q.offset(start).limit(limit)
 
         users_list = base_q.all()
 
         users_data = []
         for user in users_list:
             users_data.append({
                 "username": h.gravatar_with_user(self.request, user.username),
                 "email": user.email,
                 "first_name": user.first_name,
                 "last_name": user.last_name,
                 "last_login": h.format_date(user.last_login),
                 "last_activity": h.format_date(user.last_activity),
                 "active": h.bool2icon(user.active),
                 "active_raw": user.active,
                 "admin": h.bool2icon(user.admin),
                 "extern_type": user.extern_type,
                 "extern_name": user.extern_name,
                 "action": user_actions(user.user_id, user.username),
             })
         data = ({
             'draw': draw,
             'data': users_data,
             'recordsTotal': users_data_total_count,
             'recordsFiltered': users_data_total_filtered_count,
             'recordsTotalInactive': users_data_total_inactive_count,
             'recordsFilteredInactive': users_data_total_filtered_inactive_count
         })
 
         return data
 
     def _set_personal_repo_group_template_vars(self, c_obj):
         DummyUser = AttributeDict({
             'username': '${username}',
             'user_id': '${user_id}',
         })
         c_obj.default_create_repo_group = RepoGroupModel() \
             .get_default_create_personal_repo_group()
         c_obj.personal_repo_group_name = RepoGroupModel() \
             .get_personal_group_name(DummyUser)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='users_new', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_add.mako')
     def users_new(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.default_extern_type = auth_rhodecode.RhodeCodeAuthPlugin.uid
         self._set_personal_repo_group_template_vars(c)
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='users_create', request_method='POST',
         renderer='rhodecode:templates/admin/users/user_add.mako')
     def users_create(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.default_extern_type = auth_rhodecode.RhodeCodeAuthPlugin.uid
         user_model = UserModel()
         user_form = UserForm(self.request.translate)()
         try:
             form_result = user_form.to_python(dict(self.request.POST))
             user = user_model.create(form_result)
             Session().flush()
             creation_data = user.get_api_data()
             username = form_result['username']
 
             audit_logger.store_web(
                 'user.create', action_data={'data': creation_data},
                 user=c.rhodecode_user)
 
             user_link = h.link_to(
                 h.escape(username),
                 h.route_path('user_edit', user_id=user.user_id))
             h.flash(h.literal(_('Created user %(user_link)s')
                               % {'user_link': user_link}), category='success')
             Session().commit()
         except formencode.Invalid as errors:
             self._set_personal_repo_group_template_vars(c)
             data = render(
                 'rhodecode:templates/admin/users/user_add.mako',
                 self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
         except UserCreationError as e:
             h.flash(e, 'error')
         except Exception:
             log.exception("Exception creation of user")
             h.flash(_('Error occurred during creation of user %s')
                     % self.request.POST.get('username'), category='error')
         raise HTTPFound(h.route_path('users'))
 
 
 class UsersView(UserAppView):
     ALLOW_SCOPED_TOKENS = False
     """
     This view has alternative version inside EE, if modified please take a look
     in there as well.
     """
 
     def get_auth_plugins(self):
         valid_plugins = []
         authn_registry = get_authn_registry(self.request.registry)
         for plugin in authn_registry.get_plugins_for_authentication():
             if isinstance(plugin, RhodeCodeExternalAuthPlugin):
                 valid_plugins.append(plugin)
             elif plugin.name == 'rhodecode':
                 valid_plugins.append(plugin)
 
         # extend our choices if user has set a bound plugin which isn't enabled at the
         # moment
         extern_type = self.db_user.extern_type
         if extern_type not in [x.uid for x in valid_plugins]:
             try:
                 plugin = authn_registry.get_plugin_by_uid(extern_type)
                 if plugin:
                     valid_plugins.append(plugin)
 
             except Exception:
                 log.exception(
                     'Could not extend user plugins with `{}`'.format(extern_type))
         return valid_plugins
 
     def load_default_context(self):
         req = self.request
 
         c = self._get_local_tmpl_context()
         c.allow_scoped_tokens = self.ALLOW_SCOPED_TOKENS
         c.allowed_languages = [
             ('en', 'English (en)'),
             ('de', 'German (de)'),
             ('fr', 'French (fr)'),
             ('it', 'Italian (it)'),
             ('ja', 'Japanese (ja)'),
             ('pl', 'Polish (pl)'),
             ('pt', 'Portuguese (pt)'),
             ('ru', 'Russian (ru)'),
             ('zh', 'Chinese (zh)'),
         ]
 
         c.allowed_extern_types = [
             (x.uid, x.get_display_name()) for x in self.get_auth_plugins()
         ]
+        perms = req.registry.settings.get('available_permissions')
+        if not perms:
+            # inject info about available permissions
+            auth.set_available_permissions(req.registry.settings)
 
         c.available_permissions = req.registry.settings['available_permissions']
         PermissionModel().set_global_permission_choices(
             c, gettext_translator=req.translate)
 
         return c
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='user_update', request_method='POST',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def user_update(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         c.active = 'profile'
         c.extern_type = c.user.extern_type
         c.extern_name = c.user.extern_name
         c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
         available_languages = [x[0] for x in c.allowed_languages]
         _form = UserForm(self.request.translate, edit=True,
                          available_languages=available_languages,
                          old_data={'user_id': user_id,
                                    'email': c.user.email})()
         form_result = {}
         old_values = c.user.get_api_data()
         try:
             form_result = _form.to_python(dict(self.request.POST))
             skip_attrs = ['extern_name']
             # TODO: plugin should define if username can be updated
             if c.extern_type != "rhodecode":
                 # forbid updating username for external accounts
                 skip_attrs.append('username')
 
             UserModel().update_user(
                 user_id, skip_attrs=skip_attrs, **form_result)
 
             audit_logger.store_web(
                 'user.edit', action_data={'old_data': old_values},
                 user=c.rhodecode_user)
 
             Session().commit()
             h.flash(_('User updated successfully'), category='success')
         except formencode.Invalid as errors:
             data = render(
                 'rhodecode:templates/admin/users/user_edit.mako',
                 self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
         except UserCreationError as e:
             h.flash(e, 'error')
         except Exception:
             log.exception("Exception updating user")
             h.flash(_('Error occurred during update of user %s')
                     % form_result.get('username'), category='error')
         raise HTTPFound(h.route_path('user_edit', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='user_delete', request_method='POST',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def user_delete(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         _repos = c.user.repositories
         _repo_groups = c.user.repository_groups
         _user_groups = c.user.user_groups
         _pull_requests = c.user.user_pull_requests
         _artifacts = c.user.artifacts
 
         handle_repos = None
         handle_repo_groups = None
         handle_user_groups = None
         handle_pull_requests = None
         handle_artifacts = None
 
         # calls for flash of handle based on handle case detach or delete
         def set_handle_flash_repos():
             handle = handle_repos
             if handle == 'detach':
                 h.flash(_('Detached %s repositories') % len(_repos),
                         category='success')
             elif handle == 'delete':
                 h.flash(_('Deleted %s repositories') % len(_repos),
                         category='success')
 
         def set_handle_flash_repo_groups():
             handle = handle_repo_groups
             if handle == 'detach':
                 h.flash(_('Detached %s repository groups') % len(_repo_groups),
                         category='success')
             elif handle == 'delete':
                 h.flash(_('Deleted %s repository groups') % len(_repo_groups),
                         category='success')
 
         def set_handle_flash_user_groups():
             handle = handle_user_groups
             if handle == 'detach':
                 h.flash(_('Detached %s user groups') % len(_user_groups),
                         category='success')
             elif handle == 'delete':
                 h.flash(_('Deleted %s user groups') % len(_user_groups),
                         category='success')
 
         def set_handle_flash_pull_requests():
             handle = handle_pull_requests
             if handle == 'detach':
                 h.flash(_('Detached %s pull requests') % len(_pull_requests),
                         category='success')
             elif handle == 'delete':
                 h.flash(_('Deleted %s pull requests') % len(_pull_requests),
                         category='success')
 
         def set_handle_flash_artifacts():
             handle = handle_artifacts
             if handle == 'detach':
                 h.flash(_('Detached %s artifacts') % len(_artifacts),
                         category='success')
             elif handle == 'delete':
                 h.flash(_('Deleted %s artifacts') % len(_artifacts),
                         category='success')
 
         handle_user = User.get_first_super_admin()
         handle_user_id = safe_int(self.request.POST.get('detach_user_id'))
         if handle_user_id:
             # NOTE(marcink): we get new owner for objects...
             handle_user = User.get_or_404(handle_user_id)
 
         if _repos and self.request.POST.get('user_repos'):
             handle_repos = self.request.POST['user_repos']
 
         if _repo_groups and self.request.POST.get('user_repo_groups'):
             handle_repo_groups = self.request.POST['user_repo_groups']
 
         if _user_groups and self.request.POST.get('user_user_groups'):
             handle_user_groups = self.request.POST['user_user_groups']
 
         if _pull_requests and self.request.POST.get('user_pull_requests'):
             handle_pull_requests = self.request.POST['user_pull_requests']
 
         if _artifacts and self.request.POST.get('user_artifacts'):
             handle_artifacts = self.request.POST['user_artifacts']
 
         old_values = c.user.get_api_data()
 
         try:
 
             UserModel().delete(
                 c.user,
                 handle_repos=handle_repos,
                 handle_repo_groups=handle_repo_groups,
                 handle_user_groups=handle_user_groups,
                 handle_pull_requests=handle_pull_requests,
                 handle_artifacts=handle_artifacts,
                 handle_new_owner=handle_user
             )
 
             audit_logger.store_web(
                 'user.delete', action_data={'old_data': old_values},
                 user=c.rhodecode_user)
 
             Session().commit()
             set_handle_flash_repos()
             set_handle_flash_repo_groups()
             set_handle_flash_user_groups()
             set_handle_flash_pull_requests()
             set_handle_flash_artifacts()
             username = h.escape(old_values['username'])
             h.flash(_('Successfully deleted user `{}`').format(username), category='success')
         except (UserOwnsReposException, UserOwnsRepoGroupsException,
                 UserOwnsUserGroupsException, UserOwnsPullRequestsException,
                 UserOwnsArtifactsException, DefaultUserException) as e:
             h.flash(e, category='warning')
         except Exception:
             log.exception("Exception during deletion of user")
             h.flash(_('An error occurred during deletion of user'),
                     category='error')
         raise HTTPFound(h.route_path('users'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='user_edit', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def user_edit(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'profile'
         c.extern_type = c.user.extern_type
         c.extern_name = c.user.extern_name
         c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
 
         defaults = c.user.get_dict()
         defaults.update({'language': c.user.user_data.get('language')})
 
         data = render(
             'rhodecode:templates/admin/users/user_edit.mako',
             self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='user_edit_advanced', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def user_edit_advanced(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         c.detach_user = User.get_first_super_admin()
         detach_user_id = safe_int(self.request.GET.get('detach_user_id'))
         if detach_user_id:
             c.detach_user = User.get_or_404(detach_user_id)
 
         c.active = 'advanced'
         c.personal_repo_group = RepoGroup.get_user_personal_repo_group(user_id)
         c.personal_repo_group_name = RepoGroupModel()\
             .get_personal_group_name(c.user)
 
         c.user_to_review_rules = sorted(
             (x.user for x in c.user.user_review_rules),
             key=lambda u: u.username.lower())
 
         defaults = c.user.get_dict()
 
         # Interim workaround if the user participated on any pull requests as a
         # reviewer.
         has_review = len(c.user.reviewer_pull_requests)
         c.can_delete_user = not has_review
         c.can_delete_user_message = ''
         inactive_link = h.link_to(
             'inactive', h.route_path('user_edit', user_id=user_id, _anchor='active'))
         if has_review == 1:
             c.can_delete_user_message = h.literal(_(
                 'The user participates as reviewer in {} pull request and '
                 'cannot be deleted. \nYou can set the user to '
                 '"{}" instead of deleting it.').format(
                 has_review, inactive_link))
         elif has_review:
             c.can_delete_user_message = h.literal(_(
                 'The user participates as reviewer in {} pull requests and '
                 'cannot be deleted. \nYou can set the user to '
                 '"{}" instead of deleting it.').format(
                 has_review, inactive_link))
 
         data = render(
             'rhodecode:templates/admin/users/user_edit.mako',
             self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='user_edit_global_perms', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def user_edit_global_perms(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'global_perms'
 
         c.default_user = User.get_default_user()
         defaults = c.user.get_dict()
         defaults.update(c.default_user.get_default_perms(suffix='_inherited'))
         defaults.update(c.default_user.get_default_perms())
         defaults.update(c.user.get_default_perms())
 
         data = render(
             'rhodecode:templates/admin/users/user_edit.mako',
             self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='user_edit_global_perms_update', request_method='POST',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def user_edit_global_perms_update(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         c.active = 'global_perms'
         try:
             # first stage that verifies the checkbox
             _form = UserIndividualPermissionsForm(self.request.translate)
             form_result = _form.to_python(dict(self.request.POST))
             inherit_perms = form_result['inherit_default_permissions']
             c.user.inherit_default_permissions = inherit_perms
             Session().add(c.user)
 
             if not inherit_perms:
                 # only update the individual ones if we un check the flag
                 _form = UserPermissionsForm(
                     self.request.translate,
                     [x[0] for x in c.repo_create_choices],
                     [x[0] for x in c.repo_create_on_write_choices],
                     [x[0] for x in c.repo_group_create_choices],
                     [x[0] for x in c.user_group_create_choices],
                     [x[0] for x in c.fork_choices],
                     [x[0] for x in c.inherit_default_permission_choices])()
 
                 form_result = _form.to_python(dict(self.request.POST))
                 form_result.update({'perm_user_id': c.user.user_id})
 
                 PermissionModel().update_user_permissions(form_result)
 
             # TODO(marcink): implement global permissions
             # audit_log.store_web('user.edit.permissions')
 
             Session().commit()
 
             h.flash(_('User global permissions updated successfully'),
                     category='success')
 
         except formencode.Invalid as errors:
             data = render(
                 'rhodecode:templates/admin/users/user_edit.mako',
                 self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
         except Exception:
             log.exception("Exception during permissions saving")
             h.flash(_('An error occurred during permissions saving'),
                     category='error')
 
         affected_user_ids = [user_id]
         PermissionModel().trigger_permission_flush(affected_user_ids)
         raise HTTPFound(h.route_path('user_edit_global_perms', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='user_enable_force_password_reset', request_method='POST',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def user_enable_force_password_reset(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         try:
             c.user.update_userdata(force_password_change=True)
 
             msg = _('Force password change enabled for user')
             audit_logger.store_web('user.edit.password_reset.enabled',
                                    user=c.rhodecode_user)
 
             Session().commit()
             h.flash(msg, category='success')
         except Exception:
             log.exception("Exception during password reset for user")
             h.flash(_('An error occurred during password reset for user'),
                     category='error')
 
         raise HTTPFound(h.route_path('user_edit_advanced', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='user_disable_force_password_reset', request_method='POST',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def user_disable_force_password_reset(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         try:
             c.user.update_userdata(force_password_change=False)
 
             msg = _('Force password change disabled for user')
             audit_logger.store_web(
                 'user.edit.password_reset.disabled',
                 user=c.rhodecode_user)
 
             Session().commit()
             h.flash(msg, category='success')
         except Exception:
             log.exception("Exception during password reset for user")
             h.flash(_('An error occurred during password reset for user'),
                     category='error')
 
         raise HTTPFound(h.route_path('user_edit_advanced', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='user_notice_dismiss', request_method='POST',
         renderer='json_ext', xhr=True)
     def user_notice_dismiss(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
         user_notice_id = safe_int(self.request.POST.get('notice_id'))
         notice = UserNotice().query()\
             .filter(UserNotice.user_id == user_id)\
             .filter(UserNotice.user_notice_id == user_notice_id)\
             .scalar()
         read = False
         if notice:
             notice.notice_read = True
             Session().add(notice)
             Session().commit()
             read = True
 
         return {'notice': user_notice_id, 'read': read}
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='user_create_personal_repo_group', request_method='POST',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def user_create_personal_repo_group(self):
         """
         Create personal repository group for this user
         """
         from rhodecode.model.repo_group import RepoGroupModel
 
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         personal_repo_group = RepoGroup.get_user_personal_repo_group(
             c.user.user_id)
         if personal_repo_group:
             raise HTTPFound(h.route_path('user_edit_advanced', user_id=user_id))
 
         personal_repo_group_name = RepoGroupModel().get_personal_group_name(c.user)
         named_personal_group = RepoGroup.get_by_group_name(
             personal_repo_group_name)
         try:
 
             if named_personal_group and named_personal_group.user_id == c.user.user_id:
                 # migrate the same named group, and mark it as personal
                 named_personal_group.personal = True
                 Session().add(named_personal_group)
                 Session().commit()
                 msg = _('Linked repository group `%s` as personal' % (
                     personal_repo_group_name,))
                 h.flash(msg, category='success')
             elif not named_personal_group:
                 RepoGroupModel().create_personal_repo_group(c.user)
 
                 msg = _('Created repository group `%s`' % (
                     personal_repo_group_name,))
                 h.flash(msg, category='success')
             else:
                 msg = _('Repository group `%s` is already taken' % (
                     personal_repo_group_name,))
                 h.flash(msg, category='warning')
         except Exception:
             log.exception("Exception during repository group creation")
             msg = _(
                 'An error occurred during repository group creation for user')
             h.flash(msg, category='error')
             Session().rollback()
 
         raise HTTPFound(h.route_path('user_edit_advanced', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='edit_user_auth_tokens', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def auth_tokens(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'auth_tokens'
 
         c.lifetime_values = AuthTokenModel.get_lifetime_values(translator=_)
         c.role_values = [
             (x, AuthTokenModel.cls._get_role_name(x))
             for x in AuthTokenModel.cls.ROLES]
         c.role_options = [(c.role_values, _("Role"))]
         c.user_auth_tokens = AuthTokenModel().get_auth_tokens(
             c.user.user_id, show_expired=True)
         c.role_vcs = AuthTokenModel.cls.ROLE_VCS
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='edit_user_auth_tokens_view', request_method='POST',
         renderer='json_ext', xhr=True)
     def auth_tokens_view(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         auth_token_id = self.request.POST.get('auth_token_id')
 
         if auth_token_id:
             token = UserApiKeys.get_or_404(auth_token_id)
 
             return {
                 'auth_token': token.api_key
             }
 
     def maybe_attach_token_scope(self, token):
         # implemented in EE edition
         pass
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='edit_user_auth_tokens_add', request_method='POST')
     def auth_tokens_add(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         user_data = c.user.get_api_data()
         lifetime = safe_int(self.request.POST.get('lifetime'), -1)
         description = self.request.POST.get('description')
         role = self.request.POST.get('role')
 
         token = UserModel().add_auth_token(
             user=c.user.user_id,
             lifetime_minutes=lifetime, role=role, description=description,
             scope_callback=self.maybe_attach_token_scope)
         token_data = token.get_api_data()
 
         audit_logger.store_web(
             'user.edit.token.add', action_data={
                 'data': {'token': token_data, 'user': user_data}},
             user=self._rhodecode_user, )
         Session().commit()
 
         h.flash(_("Auth token successfully created"), category='success')
         return HTTPFound(h.route_path('edit_user_auth_tokens', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='edit_user_auth_tokens_delete', request_method='POST')
     def auth_tokens_delete(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         user_data = c.user.get_api_data()
 
         del_auth_token = self.request.POST.get('del_auth_token')
 
         if del_auth_token:
             token = UserApiKeys.get_or_404(del_auth_token)
             token_data = token.get_api_data()
 
             AuthTokenModel().delete(del_auth_token, c.user.user_id)
             audit_logger.store_web(
                 'user.edit.token.delete', action_data={
                     'data': {'token': token_data, 'user': user_data}},
                 user=self._rhodecode_user,)
             Session().commit()
             h.flash(_("Auth token successfully deleted"), category='success')
 
         return HTTPFound(h.route_path('edit_user_auth_tokens', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='edit_user_ssh_keys', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def ssh_keys(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'ssh_keys'
         c.default_key = self.request.GET.get('default_key')
         c.user_ssh_keys = SshKeyModel().get_ssh_keys(c.user.user_id)
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='edit_user_ssh_keys_generate_keypair', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def ssh_keys_generate_keypair(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         c.user = self.db_user
 
         c.active = 'ssh_keys_generate'
         comment = 'RhodeCode-SSH {}'.format(c.user.email or '')
         private_format = self.request.GET.get('private_format') \
                          or SshKeyModel.DEFAULT_PRIVATE_KEY_FORMAT
         c.private, c.public = SshKeyModel().generate_keypair(
             comment=comment, private_format=private_format)
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='edit_user_ssh_keys_add', request_method='POST')
     def ssh_keys_add(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         user_data = c.user.get_api_data()
         key_data = self.request.POST.get('key_data')
         description = self.request.POST.get('description')
 
         fingerprint = 'unknown'
         try:
             if not key_data:
                 raise ValueError('Please add a valid public key')
 
             key = SshKeyModel().parse_key(key_data.strip())
             fingerprint = key.hash_md5()
 
             ssh_key = SshKeyModel().create(
                 c.user.user_id, fingerprint, key.keydata, description)
             ssh_key_data = ssh_key.get_api_data()
 
             audit_logger.store_web(
                 'user.edit.ssh_key.add', action_data={
                     'data': {'ssh_key': ssh_key_data, 'user': user_data}},
                 user=self._rhodecode_user, )
             Session().commit()
 
             # Trigger an event on change of keys.
             trigger(SshKeyFileChangeEvent(), self.request.registry)
 
             h.flash(_("Ssh Key successfully created"), category='success')
 
         except IntegrityError:
             log.exception("Exception during ssh key saving")
             err = 'Such key with fingerprint `{}` already exists, ' \
                   'please use a different one'.format(fingerprint)
             h.flash(_('An error occurred during ssh key saving: {}').format(err),
                     category='error')
         except Exception as e:
             log.exception("Exception during ssh key saving")
             h.flash(_('An error occurred during ssh key saving: {}').format(e),
                     category='error')
 
         return HTTPFound(
             h.route_path('edit_user_ssh_keys', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='edit_user_ssh_keys_delete', request_method='POST')
     def ssh_keys_delete(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         user_data = c.user.get_api_data()
 
         del_ssh_key = self.request.POST.get('del_ssh_key')
 
         if del_ssh_key:
             ssh_key = UserSshKeys.get_or_404(del_ssh_key)
             ssh_key_data = ssh_key.get_api_data()
 
             SshKeyModel().delete(del_ssh_key, c.user.user_id)
             audit_logger.store_web(
                 'user.edit.ssh_key.delete', action_data={
                     'data': {'ssh_key': ssh_key_data, 'user': user_data}},
                 user=self._rhodecode_user,)
             Session().commit()
             # Trigger an event on change of keys.
             trigger(SshKeyFileChangeEvent(), self.request.registry)
             h.flash(_("Ssh key successfully deleted"), category='success')
 
         return HTTPFound(h.route_path('edit_user_ssh_keys', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='edit_user_emails', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def emails(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'emails'
         c.user_email_map = UserEmailMap.query() \
             .filter(UserEmailMap.user == c.user).all()
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='edit_user_emails_add', request_method='POST')
     def emails_add(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         email = self.request.POST.get('new_email')
         user_data = c.user.get_api_data()
         try:
 
             form = UserExtraEmailForm(self.request.translate)()
             data = form.to_python({'email': email})
             email = data['email']
 
             UserModel().add_extra_email(c.user.user_id, email)
             audit_logger.store_web(
                 'user.edit.email.add',
                 action_data={'email': email, 'user': user_data},
                 user=self._rhodecode_user)
             Session().commit()
             h.flash(_("Added new email address `%s` for user account") % email,
                     category='success')
         except formencode.Invalid as error:
             h.flash(h.escape(error.error_dict['email']), category='error')
         except IntegrityError:
             log.warning("Email %s already exists", email)
             h.flash(_('Email `{}` is already registered for another user.').format(email),
                     category='error')
         except Exception:
             log.exception("Exception during email saving")
             h.flash(_('An error occurred during email saving'),
                     category='error')
         raise HTTPFound(h.route_path('edit_user_emails', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='edit_user_emails_delete', request_method='POST')
     def emails_delete(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         email_id = self.request.POST.get('del_email_id')
         user_model = UserModel()
 
         email = UserEmailMap.query().get(email_id).email
         user_data = c.user.get_api_data()
         user_model.delete_extra_email(c.user.user_id, email_id)
         audit_logger.store_web(
             'user.edit.email.delete',
             action_data={'email': email, 'user': user_data},
             user=self._rhodecode_user)
         Session().commit()
         h.flash(_("Removed email address from user account"),
                 category='success')
         raise HTTPFound(h.route_path('edit_user_emails', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='edit_user_ips', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def ips(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'ips'
         c.user_ip_map = UserIpMap.query() \
             .filter(UserIpMap.user == c.user).all()
 
         c.inherit_default_ips = c.user.inherit_default_permissions
         c.default_user_ip_map = UserIpMap.query() \
             .filter(UserIpMap.user == User.get_default_user()).all()
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='edit_user_ips_add', request_method='POST')
     # NOTE(marcink): this view is allowed for default users, as we can
     # edit their IP white list
     def ips_add(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         user_model = UserModel()
         desc = self.request.POST.get('description')
         try:
             ip_list = user_model.parse_ip_range(
                 self.request.POST.get('new_ip'))
         except Exception as e:
             ip_list = []
             log.exception("Exception during ip saving")
             h.flash(_('An error occurred during ip saving:%s' % (e,)),
                     category='error')
         added = []
         user_data = c.user.get_api_data()
         for ip in ip_list:
             try:
                 form = UserExtraIpForm(self.request.translate)()
                 data = form.to_python({'ip': ip})
                 ip = data['ip']
 
                 user_model.add_extra_ip(c.user.user_id, ip, desc)
                 audit_logger.store_web(
                     'user.edit.ip.add',
                     action_data={'ip': ip, 'user': user_data},
                     user=self._rhodecode_user)
                 Session().commit()
                 added.append(ip)
             except formencode.Invalid as error:
                 msg = error.error_dict['ip']
                 h.flash(msg, category='error')
             except Exception:
                 log.exception("Exception during ip saving")
                 h.flash(_('An error occurred during ip saving'),
                         category='error')
         if added:
             h.flash(
                 _("Added ips %s to user whitelist") % (', '.join(ip_list), ),
                 category='success')
         if 'default_user' in self.request.POST:
             # case for editing global IP list we do it for 'DEFAULT' user
             raise HTTPFound(h.route_path('admin_permissions_ips'))
         raise HTTPFound(h.route_path('edit_user_ips', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='edit_user_ips_delete', request_method='POST')
     # NOTE(marcink): this view is allowed for default users, as we can
     # edit their IP white list
     def ips_delete(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         ip_id = self.request.POST.get('del_ip_id')
         user_model = UserModel()
         user_data = c.user.get_api_data()
         ip = UserIpMap.query().get(ip_id).ip_addr
         user_model.delete_extra_ip(c.user.user_id, ip_id)
         audit_logger.store_web(
             'user.edit.ip.delete', action_data={'ip': ip, 'user': user_data},
             user=self._rhodecode_user)
         Session().commit()
         h.flash(_("Removed ip address from user whitelist"), category='success')
 
         if 'default_user' in self.request.POST:
             # case for editing global IP list we do it for 'DEFAULT' user
             raise HTTPFound(h.route_path('admin_permissions_ips'))
         raise HTTPFound(h.route_path('edit_user_ips', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='edit_user_groups_management', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def groups_management(self):
         c = self.load_default_context()
         c.user = self.db_user
         c.data = c.user.group_member
 
         groups = [UserGroupModel.get_user_groups_as_dict(group.users_group)
                   for group in c.user.group_member]
         c.groups = json.dumps(groups)
         c.active = 'groups'
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='edit_user_groups_management_updates', request_method='POST')
     def groups_management_updates(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         user_groups = set(self.request.POST.getall('users_group_id'))
         user_groups_objects = []
 
         for ugid in user_groups:
             user_groups_objects.append(
                 UserGroupModel().get_group(safe_int(ugid)))
         user_group_model = UserGroupModel()
         added_to_groups, removed_from_groups = \
             user_group_model.change_groups(c.user, user_groups_objects)
 
         user_data = c.user.get_api_data()
         for user_group_id in added_to_groups:
             user_group = UserGroup.get(user_group_id)
             old_values = user_group.get_api_data()
             audit_logger.store_web(
                 'user_group.edit.member.add',
                 action_data={'user': user_data, 'old_data': old_values},
                 user=self._rhodecode_user)
 
         for user_group_id in removed_from_groups:
             user_group = UserGroup.get(user_group_id)
             old_values = user_group.get_api_data()
             audit_logger.store_web(
                 'user_group.edit.member.delete',
                 action_data={'user': user_data, 'old_data': old_values},
                 user=self._rhodecode_user)
 
         Session().commit()
         c.active = 'user_groups_management'
         h.flash(_("Groups successfully changed"), category='success')
 
         return HTTPFound(h.route_path(
             'edit_user_groups_management', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='edit_user_audit_logs', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def user_audit_logs(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'audit'
 
         p = safe_int(self.request.GET.get('page', 1), 1)
 
         filter_term = self.request.GET.get('filter')
         user_log = UserModel().get_user_log(c.user, filter_term)
 
         def url_generator(page_num):
             query_params = {
                 'page': page_num
             }
             if filter_term:
                 query_params['filter'] = filter_term
             return self.request.current_route_path(_query=query_params)
 
         c.audit_logs = SqlPage(
             user_log, page=p, items_per_page=10, url_maker=url_generator)
         c.filter_term = filter_term
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='edit_user_audit_logs_download', request_method='GET',
         renderer='string')
     def user_audit_logs_download(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         user_log = UserModel().get_user_log(c.user, filter_term=None)
 
         audit_log_data = {}
         for entry in user_log:
             audit_log_data[entry.user_log_id] = entry.get_dict()
 
         response = Response(json.dumps(audit_log_data, indent=4))
         response.content_disposition = str(
             'attachment; filename=%s' % 'user_{}_audit_logs.json'.format(c.user.user_id))
         response.content_type = 'application/json'
 
         return response
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='edit_user_perms_summary', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def user_perms_summary(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'perms_summary'
         c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='edit_user_perms_summary_json', request_method='GET',
         renderer='json_ext')
     def user_perms_summary_json(self):
         self.load_default_context()
         perm_user = self.db_user.AuthUser(ip_addr=self.request.remote_addr)
 
         return perm_user.permissions
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='edit_user_caches', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def user_caches(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'caches'
         c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
 
         cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id)
         c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
         c.backend = c.region.backend
         c.user_keys = sorted(c.region.backend.list_keys(prefix=cache_namespace_uid))
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='edit_user_caches_update', request_method='POST')
     def user_caches_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'caches'
         c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
 
         cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id)
         del_keys = rc_cache.clear_cache_namespace('cache_perms', cache_namespace_uid)
 
         h.flash(_("Deleted {} cache keys").format(del_keys), category='success')
 
         return HTTPFound(h.route_path(
             'edit_user_caches', user_id=c.user.user_id))

rhodecode/apps/debug_style/views.py

@@ -1,486 +1,486 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import logging
 import datetime
 
 from pyramid.view import view_config
 from pyramid.renderers import render_to_response
 from rhodecode.apps._base import BaseAppView
 from rhodecode.lib.celerylib import run_task, tasks
 from rhodecode.lib.utils2 import AttributeDict
 from rhodecode.model.db import User
 from rhodecode.model.notification import EmailNotificationModel
 
 log = logging.getLogger(__name__)
 
 
 class DebugStyleView(BaseAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
 
         return c
 
     @view_config(
         route_name='debug_style_home', request_method='GET',
         renderer=None)
     def index(self):
         c = self.load_default_context()
         c.active = 'index'
 
         return render_to_response(
             'debug_style/index.html', self._get_template_context(c),
             request=self.request)
 
     @view_config(
         route_name='debug_style_email', request_method='GET',
         renderer=None)
     @view_config(
         route_name='debug_style_email_plain_rendered', request_method='GET',
         renderer=None)
     def render_email(self):
         c = self.load_default_context()
         email_id = self.request.matchdict['email_id']
         c.active = 'emails'
 
         pr = AttributeDict(
             pull_request_id=123,
             title='digital_ocean: fix redis, elastic search start on boot, '
                   'fix fd limits on supervisor, set postgres 11 version',
             description='''
 Check if we should use full-topic or mini-topic.
 
 - full topic produces some problems with merge states etc
 - server-mini-topic needs probably tweeks.            
             ''',
             repo_name='foobar',
             source_ref_parts=AttributeDict(type='branch', name='fix-ticket-2000'),
             target_ref_parts=AttributeDict(type='branch', name='master'),
         )
 
         target_repo = AttributeDict(repo_name='repo_group/target_repo')
         source_repo = AttributeDict(repo_name='repo_group/source_repo')
         user = User.get_by_username(self.request.GET.get('user')) or self._rhodecode_db_user
         # file/commit changes for PR update
         commit_changes = AttributeDict({
             'added': ['aaaaaaabbbbb', 'cccccccddddddd'],
             'removed': ['eeeeeeeeeee'],
         })
 
         file_changes = AttributeDict({
             'added': ['a/file1.md', 'file2.py'],
             'modified': ['b/modified_file.rst'],
             'removed': ['.idea'],
         })
 
         exc_traceback = {
             'exc_utc_date': '2020-03-26T12:54:50.683281',
             'exc_id': 139638856342656,
             'exc_timestamp': '1585227290.683288',
             'version': 'v1',
             'exc_message': 'Traceback (most recent call last):\n  File "/nix/store/s43k2r9rysfbzmsjdqnxgzvvb7zjhkxb-python2.7-pyramid-1.10.4/lib/python2.7/site-packages/pyramid/tweens.py", line 41, in excview_tween\n    response = handler(request)\n  File "/nix/store/s43k2r9rysfbzmsjdqnxgzvvb7zjhkxb-python2.7-pyramid-1.10.4/lib/python2.7/site-packages/pyramid/router.py", line 148, in handle_request\n    registry, request, context, context_iface, view_name\n  File "/nix/store/s43k2r9rysfbzmsjdqnxgzvvb7zjhkxb-python2.7-pyramid-1.10.4/lib/python2.7/site-packages/pyramid/view.py", line 667, in _call_view\n    response = view_callable(context, request)\n  File "/nix/store/s43k2r9rysfbzmsjdqnxgzvvb7zjhkxb-python2.7-pyramid-1.10.4/lib/python2.7/site-packages/pyramid/config/views.py", line 188, in attr_view\n    return view(context, request)\n  File "/nix/store/s43k2r9rysfbzmsjdqnxgzvvb7zjhkxb-python2.7-pyramid-1.10.4/lib/python2.7/site-packages/pyramid/config/views.py", line 214, in predicate_wrapper\n    return view(context, request)\n  File "/nix/store/s43k2r9rysfbzmsjdqnxgzvvb7zjhkxb-python2.7-pyramid-1.10.4/lib/python2.7/site-packages/pyramid/viewderivers.py", line 401, in viewresult_to_response\n    result = view(context, request)\n  File "/nix/store/s43k2r9rysfbzmsjdqnxgzvvb7zjhkxb-python2.7-pyramid-1.10.4/lib/python2.7/site-packages/pyramid/viewderivers.py", line 132, in _class_view\n    response = getattr(inst, attr)()\n  File "/mnt/hgfs/marcink/workspace/rhodecode-enterprise-ce/rhodecode/apps/debug_style/views.py", line 355, in render_email\n    template_type, **email_kwargs.get(email_id, {}))\n  File "/mnt/hgfs/marcink/workspace/rhodecode-enterprise-ce/rhodecode/model/notification.py", line 402, in render_email\n    body = email_template.render(None, **_kwargs)\n  File "/mnt/hgfs/marcink/workspace/rhodecode-enterprise-ce/rhodecode/lib/partial_renderer.py", line 95, in render\n    return self._render_with_exc(tmpl, args, kwargs)\n  File "/mnt/hgfs/marcink/workspace/rhodecode-enterprise-ce/rhodecode/lib/partial_renderer.py", line 79, in _render_with_exc\n    return render_func.render(*args, **kwargs)\n  File "/nix/store/dakh34sxz4yfr435c0cwjz0sd6hnd5g3-python2.7-mako-1.1.0/lib/python2.7/site-packages/mako/template.py", line 476, in render\n    return runtime._render(self, self.callable_, args, data)\n  File "/nix/store/dakh34sxz4yfr435c0cwjz0sd6hnd5g3-python2.7-mako-1.1.0/lib/python2.7/site-packages/mako/runtime.py", line 883, in _render\n    **_kwargs_for_callable(callable_, data)\n  File "/nix/store/dakh34sxz4yfr435c0cwjz0sd6hnd5g3-python2.7-mako-1.1.0/lib/python2.7/site-packages/mako/runtime.py", line 920, in _render_context\n    _exec_template(inherit, lclcontext, args=args, kwargs=kwargs)\n  File "/nix/store/dakh34sxz4yfr435c0cwjz0sd6hnd5g3-python2.7-mako-1.1.0/lib/python2.7/site-packages/mako/runtime.py", line 947, in _exec_template\n    callable_(context, *args, **kwargs)\n  File "rhodecode_templates_email_templates_base_mako", line 63, in render_body\n  File "rhodecode_templates_email_templates_exception_tracker_mako", line 43, in render_body\nAttributeError: \'str\' object has no attribute \'get\'\n',
             'exc_type': 'AttributeError'
         }
 
         email_kwargs = {
             'test': {},
 
             'message': {
                 'body': 'message body !'
             },
 
             'email_test': {
                 'user': user,
                 'date': datetime.datetime.now(),
             },
 
             'exception': {
                 'email_prefix': '[RHODECODE ERROR]',
                 'exc_id': exc_traceback['exc_id'],
                 'exc_url': 'http://server-url/{}'.format(exc_traceback['exc_id']),
                 'exc_type_name': 'NameError',
                 'exc_traceback': exc_traceback,
             },
 
             'password_reset': {
                 'password_reset_url': 'http://example.com/reset-rhodecode-password/token',
 
                 'user': user,
                 'date': datetime.datetime.now(),
                 'email': '[e-mail]',
                 'first_admin_email': User.get_first_super_admin().email
             },
 
             'password_reset_confirmation': {
                 'new_password': 'new-password-example',
                 'user': user,
                 'date': datetime.datetime.now(),
                 'email': '[e-mail]',
                 'first_admin_email': User.get_first_super_admin().email
             },
 
             'registration': {
                 'user': user,
                 'date': datetime.datetime.now(),
             },
 
             'pull_request_comment': {
                 'user': user,
 
                 'status_change': None,
                 'status_change_type': None,
 
                 'pull_request': pr,
                 'pull_request_commits': [],
 
                 'pull_request_target_repo': target_repo,
                 'pull_request_target_repo_url': 'http://target-repo/url',
 
                 'pull_request_source_repo': source_repo,
                 'pull_request_source_repo_url': 'http://source-repo/url',
 
                 'pull_request_url': 'http://localhost/pr1',
                 'pr_comment_url': 'http://comment-url',
                 'pr_comment_reply_url': 'http://comment-url#reply',
 
                 'comment_file': None,
                 'comment_line': None,
                 'comment_type': 'note',
                 'comment_body': 'This is my comment body. *I like !*',
                 'comment_id': 2048,
                 'renderer_type': 'markdown',
                 'mention': True,
 
             },
 
             'pull_request_comment+status': {
                 'user': user,
 
                 'status_change': 'approved',
                 'status_change_type': 'approved',
 
                 'pull_request': pr,
                 'pull_request_commits': [],
 
                 'pull_request_target_repo': target_repo,
                 'pull_request_target_repo_url': 'http://target-repo/url',
 
                 'pull_request_source_repo': source_repo,
                 'pull_request_source_repo_url': 'http://source-repo/url',
 
                 'pull_request_url': 'http://localhost/pr1',
                 'pr_comment_url': 'http://comment-url',
                 'pr_comment_reply_url': 'http://comment-url#reply',
 
                 'comment_type': 'todo',
                 'comment_file': None,
                 'comment_line': None,
                 'comment_body': '''
 I think something like this would be better
 
 ```py
 // markdown renderer
 
 def db():
    global connection
    return connection
 
 ```
                 
                 ''',
                 'comment_id': 2048,
                 'renderer_type': 'markdown',
                 'mention': True,
 
             },
 
             'pull_request_comment+file': {
                 'user': user,
 
                 'status_change': None,
                 'status_change_type': None,
 
                 'pull_request': pr,
                 'pull_request_commits': [],
 
                 'pull_request_target_repo': target_repo,
                 'pull_request_target_repo_url': 'http://target-repo/url',
 
                 'pull_request_source_repo': source_repo,
                 'pull_request_source_repo_url': 'http://source-repo/url',
 
                 'pull_request_url': 'http://localhost/pr1',
 
                 'pr_comment_url': 'http://comment-url',
                 'pr_comment_reply_url': 'http://comment-url#reply',
 
                 'comment_file': 'rhodecode/model/get_flow_commits',
                 'comment_line': 'o1210',
                 'comment_type': 'todo',
                 'comment_body': '''
 I like this !
 
 But please check this code
 
 .. code-block:: javascript
 
   // THIS IS RST CODE
 
   this.createResolutionComment = function(commentId) {
     // hide the trigger text
     $('#resolve-comment-{0}'.format(commentId)).hide();
 
     var comment = $('#comment-'+commentId);
     var commentData = comment.data();
     if (commentData.commentInline) {
-        this.createComment(comment, commentId)
+        this.createComment(comment, f_path, line_no, commentId)
     } else {
         Rhodecode.comments.createGeneralComment('general', "$placeholder", commentId)
     }
 
     return false;
   };
 
 This should work better !
                 ''',
                 'comment_id': 2048,
                 'renderer_type': 'rst',
                 'mention': True,
 
             },
 
             'pull_request_update': {
                 'updating_user': user,
 
                 'status_change': None,
                 'status_change_type': None,
 
                 'pull_request': pr,
                 'pull_request_commits': [],
 
                 'pull_request_target_repo': target_repo,
                 'pull_request_target_repo_url': 'http://target-repo/url',
 
                 'pull_request_source_repo': source_repo,
                 'pull_request_source_repo_url': 'http://source-repo/url',
 
                 'pull_request_url': 'http://localhost/pr1',
 
                 # update comment links
                 'pr_comment_url': 'http://comment-url',
                 'pr_comment_reply_url': 'http://comment-url#reply',
                 'ancestor_commit_id': 'f39bd443',
                 'added_commits': commit_changes.added,
                 'removed_commits': commit_changes.removed,
                 'changed_files': (file_changes.added + file_changes.modified + file_changes.removed),
                 'added_files': file_changes.added,
                 'modified_files': file_changes.modified,
                 'removed_files': file_changes.removed,
             },
 
             'cs_comment': {
                 'user': user,
                 'commit': AttributeDict(idx=123, raw_id='a'*40, message='Commit message'),
                 'status_change': None,
                 'status_change_type': None,
 
                 'commit_target_repo_url': 'http://foo.example.com/#comment1',
                 'repo_name': 'test-repo',
                 'comment_type': 'note',
                 'comment_file': None,
                 'comment_line': None,
                 'commit_comment_url': 'http://comment-url',
                 'commit_comment_reply_url': 'http://comment-url#reply',
                 'comment_body': 'This is my comment body. *I like !*',
                 'comment_id': 2048,
                 'renderer_type': 'markdown',
                 'mention': True,
             },
 
             'cs_comment+status': {
                 'user': user,
                 'commit': AttributeDict(idx=123, raw_id='a' * 40, message='Commit message'),
                 'status_change': 'approved',
                 'status_change_type': 'approved',
 
                 'commit_target_repo_url': 'http://foo.example.com/#comment1',
                 'repo_name': 'test-repo',
                 'comment_type': 'note',
                 'comment_file': None,
                 'comment_line': None,
                 'commit_comment_url': 'http://comment-url',
                 'commit_comment_reply_url': 'http://comment-url#reply',
                 'comment_body': '''
 Hello **world**
 
 This is a multiline comment :)
 
 - list
 - list2
                 ''',
                 'comment_id': 2048,
                 'renderer_type': 'markdown',
                 'mention': True,
             },
 
             'cs_comment+file': {
                 'user': user,
                 'commit': AttributeDict(idx=123, raw_id='a' * 40, message='Commit message'),
                 'status_change': None,
                 'status_change_type': None,
 
                 'commit_target_repo_url': 'http://foo.example.com/#comment1',
                 'repo_name': 'test-repo',
 
                 'comment_type': 'note',
                 'comment_file': 'test-file.py',
                 'comment_line': 'n100',
 
                 'commit_comment_url': 'http://comment-url',
                 'commit_comment_reply_url': 'http://comment-url#reply',
                 'comment_body': 'This is my comment body. *I like !*',
                 'comment_id': 2048,
                 'renderer_type': 'markdown',
                 'mention': True,
             },
 
             'pull_request': {
                 'user': user,
                 'pull_request': pr,
                 'pull_request_commits': [
                     ('472d1df03bf7206e278fcedc6ac92b46b01c4e21', '''\
         my-account: moved email closer to profile as it's similar data just moved outside.                    
                             '''),
                     ('cbfa3061b6de2696c7161ed15ba5c6a0045f90a7', '''\
         users: description edit fixes
 
         - tests
         - added metatags info                    
                             '''),
                 ],
 
                 'pull_request_target_repo': target_repo,
                 'pull_request_target_repo_url': 'http://target-repo/url',
 
                 'pull_request_source_repo': source_repo,
                 'pull_request_source_repo_url': 'http://source-repo/url',
 
                 'pull_request_url': 'http://code.rhodecode.com/_pull-request/123',
                 'user_role': 'reviewer',
             },
 
             'pull_request+reviewer_role': {
                 'user': user,
                 'pull_request': pr,
                 'pull_request_commits': [
                     ('472d1df03bf7206e278fcedc6ac92b46b01c4e21', '''\
 my-account: moved email closer to profile as it's similar data just moved outside.                    
                     '''),
                     ('cbfa3061b6de2696c7161ed15ba5c6a0045f90a7', '''\
 users: description edit fixes
 
 - tests
 - added metatags info                    
                     '''),
                 ],
 
                 'pull_request_target_repo': target_repo,
                 'pull_request_target_repo_url': 'http://target-repo/url',
 
                 'pull_request_source_repo': source_repo,
                 'pull_request_source_repo_url': 'http://source-repo/url',
 
                 'pull_request_url': 'http://code.rhodecode.com/_pull-request/123',
                 'user_role': 'reviewer',
             },
 
             'pull_request+observer_role': {
                 'user': user,
                 'pull_request': pr,
                 'pull_request_commits': [
                     ('472d1df03bf7206e278fcedc6ac92b46b01c4e21', '''\
         my-account: moved email closer to profile as it's similar data just moved outside.                    
                             '''),
                     ('cbfa3061b6de2696c7161ed15ba5c6a0045f90a7', '''\
         users: description edit fixes
 
         - tests
         - added metatags info                    
                             '''),
                 ],
 
                 'pull_request_target_repo': target_repo,
                 'pull_request_target_repo_url': 'http://target-repo/url',
 
                 'pull_request_source_repo': source_repo,
                 'pull_request_source_repo_url': 'http://source-repo/url',
 
                 'pull_request_url': 'http://code.rhodecode.com/_pull-request/123',
                 'user_role': 'observer'
             }
         }
 
         template_type = email_id.split('+')[0]
         (c.subject, c.email_body, c.email_body_plaintext) = EmailNotificationModel().render_email(
             template_type, **email_kwargs.get(email_id, {}))
 
         test_email = self.request.GET.get('email')
         if test_email:
             recipients = [test_email]
             run_task(tasks.send_email, recipients, c.subject,
                      c.email_body_plaintext, c.email_body)
 
         if self.request.matched_route.name == 'debug_style_email_plain_rendered':
             template = 'debug_style/email_plain_rendered.mako'
         else:
             template = 'debug_style/email.mako'
         return render_to_response(
             template, self._get_template_context(c),
             request=self.request)
 
     @view_config(
         route_name='debug_style_template', request_method='GET',
         renderer=None)
     def template(self):
         t_path = self.request.matchdict['t_path']
         c = self.load_default_context()
         c.active = os.path.splitext(t_path)[0]
         c.came_from = ''
         # NOTE(marcink): extend the email types with variations based on data sets
         c.email_types = {
             'cs_comment+file': {},
             'cs_comment+status': {},
 
             'pull_request_comment+file': {},
             'pull_request_comment+status': {},
 
             'pull_request_update': {},
 
             'pull_request+reviewer_role': {},
             'pull_request+observer_role': {},
         }
         c.email_types.update(EmailNotificationModel.email_types)
 
         return render_to_response(
             'debug_style/' + t_path, self._get_template_context(c),
             request=self.request)
 

rhodecode/apps/my_account/views/my_account.py

@@ -1,822 +1,826 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import datetime
 import string
 
 import formencode
 import formencode.htmlfill
 import peppercorn
 from pyramid.httpexceptions import HTTPFound, HTTPNotFound
 from pyramid.view import view_config
 
 from rhodecode.apps._base import BaseAppView, DataGridAppView
 from rhodecode import forms
 from rhodecode.lib import helpers as h
 from rhodecode.lib import audit_logger
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.auth import (
     LoginRequired, NotAnonymous, CSRFRequired,
     HasRepoPermissionAny, HasRepoGroupPermissionAny, AuthUser)
 from rhodecode.lib.channelstream import (
     channelstream_request, ChannelstreamException)
 from rhodecode.lib.utils2 import safe_int, md5, str2bool
 from rhodecode.model.auth_token import AuthTokenModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.db import (
     IntegrityError, or_, in_filter_generator,
     Repository, UserEmailMap, UserApiKeys, UserFollowing,
     PullRequest, UserBookmark, RepoGroup)
 from rhodecode.model.meta import Session
 from rhodecode.model.pull_request import PullRequestModel
 from rhodecode.model.user import UserModel
 from rhodecode.model.user_group import UserGroupModel
 from rhodecode.model.validation_schema.schemas import user_schema
 
 log = logging.getLogger(__name__)
 
 
 class MyAccountView(BaseAppView, DataGridAppView):
     ALLOW_SCOPED_TOKENS = False
     """
     This view has alternative version inside EE, if modified please take a look
     in there as well.
     """
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         c.user = c.auth_user.get_instance()
         c.allow_scoped_tokens = self.ALLOW_SCOPED_TOKENS
 
         return c
 
     @LoginRequired()
     @NotAnonymous()
     @view_config(
         route_name='my_account_profile', request_method='GET',
         renderer='rhodecode:templates/admin/my_account/my_account.mako')
     def my_account_profile(self):
         c = self.load_default_context()
         c.active = 'profile'
         c.extern_type = c.user.extern_type
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @view_config(
         route_name='my_account_password', request_method='GET',
         renderer='rhodecode:templates/admin/my_account/my_account.mako')
     def my_account_password(self):
         c = self.load_default_context()
         c.active = 'password'
         c.extern_type = c.user.extern_type
 
         schema = user_schema.ChangePasswordSchema().bind(
             username=c.user.username)
 
         form = forms.Form(
             schema,
             action=h.route_path('my_account_password_update'),
             buttons=(forms.buttons.save, forms.buttons.reset))
 
         c.form = form
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     @view_config(
         route_name='my_account_password_update', request_method='POST',
         renderer='rhodecode:templates/admin/my_account/my_account.mako')
     def my_account_password_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'password'
         c.extern_type = c.user.extern_type
 
         schema = user_schema.ChangePasswordSchema().bind(
             username=c.user.username)
 
         form = forms.Form(
             schema, buttons=(forms.buttons.save, forms.buttons.reset))
 
         if c.extern_type != 'rhodecode':
             raise HTTPFound(self.request.route_path('my_account_password'))
 
         controls = self.request.POST.items()
         try:
             valid_data = form.validate(controls)
             UserModel().update_user(c.user.user_id, **valid_data)
             c.user.update_userdata(force_password_change=False)
             Session().commit()
         except forms.ValidationFailure as e:
             c.form = e
             return self._get_template_context(c)
 
         except Exception:
             log.exception("Exception updating password")
             h.flash(_('Error occurred during update of user password'),
                     category='error')
         else:
             instance = c.auth_user.get_instance()
             self.session.setdefault('rhodecode_user', {}).update(
                 {'password': md5(instance.password)})
             self.session.save()
             h.flash(_("Successfully updated password"), category='success')
 
         raise HTTPFound(self.request.route_path('my_account_password'))
 
     @LoginRequired()
     @NotAnonymous()
     @view_config(
         route_name='my_account_auth_tokens', request_method='GET',
         renderer='rhodecode:templates/admin/my_account/my_account.mako')
     def my_account_auth_tokens(self):
         _ = self.request.translate
 
         c = self.load_default_context()
         c.active = 'auth_tokens'
         c.lifetime_values = AuthTokenModel.get_lifetime_values(translator=_)
         c.role_values = [
             (x, AuthTokenModel.cls._get_role_name(x))
             for x in AuthTokenModel.cls.ROLES]
         c.role_options = [(c.role_values, _("Role"))]
         c.user_auth_tokens = AuthTokenModel().get_auth_tokens(
             c.user.user_id, show_expired=True)
         c.role_vcs = AuthTokenModel.cls.ROLE_VCS
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     @view_config(
         route_name='my_account_auth_tokens_view', request_method='POST', xhr=True,
         renderer='json_ext')
     def my_account_auth_tokens_view(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         auth_token_id = self.request.POST.get('auth_token_id')
 
         if auth_token_id:
             token = UserApiKeys.get_or_404(auth_token_id)
             if token.user.user_id != c.user.user_id:
                 raise HTTPNotFound()
 
             return {
                 'auth_token': token.api_key
             }
 
     def maybe_attach_token_scope(self, token):
         # implemented in EE edition
         pass
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     @view_config(
         route_name='my_account_auth_tokens_add', request_method='POST',)
     def my_account_auth_tokens_add(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         lifetime = safe_int(self.request.POST.get('lifetime'), -1)
         description = self.request.POST.get('description')
         role = self.request.POST.get('role')
 
         token = UserModel().add_auth_token(
             user=c.user.user_id,
             lifetime_minutes=lifetime, role=role, description=description,
             scope_callback=self.maybe_attach_token_scope)
         token_data = token.get_api_data()
 
         audit_logger.store_web(
             'user.edit.token.add', action_data={
                 'data': {'token': token_data, 'user': 'self'}},
             user=self._rhodecode_user, )
         Session().commit()
 
         h.flash(_("Auth token successfully created"), category='success')
         return HTTPFound(h.route_path('my_account_auth_tokens'))
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     @view_config(
         route_name='my_account_auth_tokens_delete', request_method='POST')
     def my_account_auth_tokens_delete(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         del_auth_token = self.request.POST.get('del_auth_token')
 
         if del_auth_token:
             token = UserApiKeys.get_or_404(del_auth_token)
             token_data = token.get_api_data()
 
             AuthTokenModel().delete(del_auth_token, c.user.user_id)
             audit_logger.store_web(
                 'user.edit.token.delete', action_data={
                     'data': {'token': token_data, 'user': 'self'}},
                 user=self._rhodecode_user,)
             Session().commit()
             h.flash(_("Auth token successfully deleted"), category='success')
 
         return HTTPFound(h.route_path('my_account_auth_tokens'))
 
     @LoginRequired()
     @NotAnonymous()
     @view_config(
         route_name='my_account_emails', request_method='GET',
         renderer='rhodecode:templates/admin/my_account/my_account.mako')
     def my_account_emails(self):
         _ = self.request.translate
 
         c = self.load_default_context()
         c.active = 'emails'
 
         c.user_email_map = UserEmailMap.query()\
             .filter(UserEmailMap.user == c.user).all()
 
         schema = user_schema.AddEmailSchema().bind(
             username=c.user.username, user_emails=c.user.emails)
 
         form = forms.RcForm(schema,
                             action=h.route_path('my_account_emails_add'),
                             buttons=(forms.buttons.save, forms.buttons.reset))
 
         c.form = form
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     @view_config(
         route_name='my_account_emails_add', request_method='POST',
         renderer='rhodecode:templates/admin/my_account/my_account.mako')
     def my_account_emails_add(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'emails'
 
         schema = user_schema.AddEmailSchema().bind(
             username=c.user.username, user_emails=c.user.emails)
 
         form = forms.RcForm(
             schema, action=h.route_path('my_account_emails_add'),
             buttons=(forms.buttons.save, forms.buttons.reset))
 
         controls = self.request.POST.items()
         try:
             valid_data = form.validate(controls)
             UserModel().add_extra_email(c.user.user_id, valid_data['email'])
             audit_logger.store_web(
                 'user.edit.email.add', action_data={
                     'data': {'email': valid_data['email'], 'user': 'self'}},
                 user=self._rhodecode_user,)
             Session().commit()
         except formencode.Invalid as error:
             h.flash(h.escape(error.error_dict['email']), category='error')
         except forms.ValidationFailure as e:
             c.user_email_map = UserEmailMap.query() \
                 .filter(UserEmailMap.user == c.user).all()
             c.form = e
             return self._get_template_context(c)
         except Exception:
             log.exception("Exception adding email")
             h.flash(_('Error occurred during adding email'),
                     category='error')
         else:
             h.flash(_("Successfully added email"), category='success')
 
         raise HTTPFound(self.request.route_path('my_account_emails'))
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     @view_config(
         route_name='my_account_emails_delete', request_method='POST')
     def my_account_emails_delete(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         del_email_id = self.request.POST.get('del_email_id')
         if del_email_id:
             email = UserEmailMap.get_or_404(del_email_id).email
             UserModel().delete_extra_email(c.user.user_id, del_email_id)
             audit_logger.store_web(
                 'user.edit.email.delete', action_data={
                     'data': {'email': email, 'user': 'self'}},
                 user=self._rhodecode_user,)
             Session().commit()
             h.flash(_("Email successfully deleted"),
                     category='success')
         return HTTPFound(h.route_path('my_account_emails'))
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     @view_config(
         route_name='my_account_notifications_test_channelstream',
         request_method='POST', renderer='json_ext')
     def my_account_notifications_test_channelstream(self):
         message = 'Test message sent via Channelstream by user: {}, on {}'.format(
             self._rhodecode_user.username, datetime.datetime.now())
         payload = {
             # 'channel': 'broadcast',
             'type': 'message',
             'timestamp': datetime.datetime.utcnow(),
             'user': 'system',
             'pm_users': [self._rhodecode_user.username],
             'message': {
                 'message': message,
                 'level': 'info',
                 'topic': '/notifications'
             }
         }
 
         registry = self.request.registry
         rhodecode_plugins = getattr(registry, 'rhodecode_plugins', {})
         channelstream_config = rhodecode_plugins.get('channelstream', {})
 
         try:
             channelstream_request(channelstream_config, [payload], '/message')
         except ChannelstreamException as e:
             log.exception('Failed to send channelstream data')
             return {"response": 'ERROR: {}'.format(e.__class__.__name__)}
         return {"response": 'Channelstream data sent. '
                             'You should see a new live message now.'}
 
     def _load_my_repos_data(self, watched=False):
 
         allowed_ids = [-1] + self._rhodecode_user.repo_acl_ids_from_stack(AuthUser.repo_read_perms)
 
         if watched:
             # repos user watch
             repo_list = Session().query(
                     Repository
                 ) \
                 .join(
                     (UserFollowing, UserFollowing.follows_repo_id == Repository.repo_id)
                 ) \
                 .filter(
                     UserFollowing.user_id == self._rhodecode_user.user_id
                 ) \
                 .filter(or_(
                     # generate multiple IN to fix limitation problems
                     *in_filter_generator(Repository.repo_id, allowed_ids))
                 ) \
                 .order_by(Repository.repo_name) \
                 .all()
 
         else:
             # repos user is owner of
             repo_list = Session().query(
                     Repository
                 ) \
                 .filter(
                     Repository.user_id == self._rhodecode_user.user_id
                 ) \
                 .filter(or_(
                     # generate multiple IN to fix limitation problems
                     *in_filter_generator(Repository.repo_id, allowed_ids))
                 ) \
                 .order_by(Repository.repo_name) \
                 .all()
 
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/data_table/_dt_elements.mako')
 
         def repo_lnk(name, rtype, rstate, private, archived, fork_of):
             return _render('repo_name', name, rtype, rstate, private, archived, fork_of,
                            short_name=False, admin=False)
 
         repos_data = []
         for repo in repo_list:
             row = {
                 "name": repo_lnk(repo.repo_name, repo.repo_type, repo.repo_state,
                                  repo.private, repo.archived, repo.fork),
                 "name_raw": repo.repo_name.lower(),
             }
 
             repos_data.append(row)
 
         # json used to render the grid
         return json.dumps(repos_data)
 
     @LoginRequired()
     @NotAnonymous()
     @view_config(
         route_name='my_account_repos', request_method='GET',
         renderer='rhodecode:templates/admin/my_account/my_account.mako')
     def my_account_repos(self):
         c = self.load_default_context()
         c.active = 'repos'
 
         # json used to render the grid
         c.data = self._load_my_repos_data()
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @view_config(
         route_name='my_account_watched', request_method='GET',
         renderer='rhodecode:templates/admin/my_account/my_account.mako')
     def my_account_watched(self):
         c = self.load_default_context()
         c.active = 'watched'
 
         # json used to render the grid
         c.data = self._load_my_repos_data(watched=True)
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @view_config(
         route_name='my_account_bookmarks', request_method='GET',
         renderer='rhodecode:templates/admin/my_account/my_account.mako')
     def my_account_bookmarks(self):
         c = self.load_default_context()
         c.active = 'bookmarks'
         c.bookmark_items = UserBookmark.get_bookmarks_for_user(
             self._rhodecode_db_user.user_id, cache=False)
         return self._get_template_context(c)
 
     def _process_bookmark_entry(self, entry, user_id):
         position = safe_int(entry.get('position'))
         cur_position = safe_int(entry.get('cur_position'))
         if position is None:
             return
 
         # check if this is an existing entry
         is_new = False
         db_entry = UserBookmark().get_by_position_for_user(cur_position, user_id)
 
         if db_entry and str2bool(entry.get('remove')):
             log.debug('Marked bookmark %s for deletion', db_entry)
             Session().delete(db_entry)
             return
 
         if not db_entry:
             # new
             db_entry = UserBookmark()
             is_new = True
 
         should_save = False
         default_redirect_url = ''
 
         # save repo
         if entry.get('bookmark_repo') and safe_int(entry.get('bookmark_repo')):
             repo = Repository.get(entry['bookmark_repo'])
             perm_check = HasRepoPermissionAny(
                 'repository.read', 'repository.write', 'repository.admin')
             if repo and perm_check(repo_name=repo.repo_name):
                 db_entry.repository = repo
                 should_save = True
                 default_redirect_url = '${repo_url}'
         # save repo group
         elif entry.get('bookmark_repo_group') and safe_int(entry.get('bookmark_repo_group')):
             repo_group = RepoGroup.get(entry['bookmark_repo_group'])
             perm_check = HasRepoGroupPermissionAny(
                 'group.read', 'group.write', 'group.admin')
 
             if repo_group and perm_check(group_name=repo_group.group_name):
                 db_entry.repository_group = repo_group
                 should_save = True
                 default_redirect_url = '${repo_group_url}'
         # save generic info
         elif entry.get('title') and entry.get('redirect_url'):
             should_save = True
 
         if should_save:
             # mark user and position
             db_entry.user_id = user_id
             db_entry.position = position
             db_entry.title = entry.get('title')
             db_entry.redirect_url = entry.get('redirect_url') or default_redirect_url
             log.debug('Saving bookmark %s, new:%s', db_entry, is_new)
 
             Session().add(db_entry)
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     @view_config(
         route_name='my_account_bookmarks_update', request_method='POST')
     def my_account_bookmarks_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'bookmarks'
 
         controls = peppercorn.parse(self.request.POST.items())
         user_id = c.user.user_id
 
         # validate positions
         positions = {}
         for entry in controls.get('bookmarks', []):
             position = safe_int(entry['position'])
             if position is None:
                 continue
 
             if position in positions:
                 h.flash(_("Position {} is defined twice. "
                           "Please correct this error.").format(position), category='error')
                 return HTTPFound(h.route_path('my_account_bookmarks'))
 
             entry['position'] = position
             entry['cur_position'] = safe_int(entry.get('cur_position'))
             positions[position] = entry
 
         try:
             for entry in positions.values():
                 self._process_bookmark_entry(entry, user_id)
 
             Session().commit()
             h.flash(_("Update Bookmarks"), category='success')
         except IntegrityError:
             h.flash(_("Failed to update bookmarks. "
                       "Make sure an unique position is used."), category='error')
 
         return HTTPFound(h.route_path('my_account_bookmarks'))
 
     @LoginRequired()
     @NotAnonymous()
     @view_config(
         route_name='my_account_goto_bookmark', request_method='GET',
         renderer='rhodecode:templates/admin/my_account/my_account.mako')
     def my_account_goto_bookmark(self):
 
         bookmark_id = self.request.matchdict['bookmark_id']
         user_bookmark = UserBookmark().query()\
             .filter(UserBookmark.user_id == self.request.user.user_id) \
             .filter(UserBookmark.position == bookmark_id).scalar()
 
         redirect_url = h.route_path('my_account_bookmarks')
         if not user_bookmark:
             raise HTTPFound(redirect_url)
 
         # repository set
         if user_bookmark.repository:
             repo_name = user_bookmark.repository.repo_name
             base_redirect_url = h.route_path(
                 'repo_summary', repo_name=repo_name)
             if user_bookmark.redirect_url and \
                     '${repo_url}' in user_bookmark.redirect_url:
                 redirect_url = string.Template(user_bookmark.redirect_url)\
                     .safe_substitute({'repo_url': base_redirect_url})
             else:
                 redirect_url = base_redirect_url
         # repository group set
         elif user_bookmark.repository_group:
             repo_group_name = user_bookmark.repository_group.group_name
             base_redirect_url = h.route_path(
                 'repo_group_home', repo_group_name=repo_group_name)
             if user_bookmark.redirect_url and \
                     '${repo_group_url}' in user_bookmark.redirect_url:
                 redirect_url = string.Template(user_bookmark.redirect_url)\
                     .safe_substitute({'repo_group_url': base_redirect_url})
             else:
                 redirect_url = base_redirect_url
         # custom URL set
         elif user_bookmark.redirect_url:
             server_url = h.route_url('home').rstrip('/')
             redirect_url = string.Template(user_bookmark.redirect_url) \
                 .safe_substitute({'server_url': server_url})
 
         log.debug('Redirecting bookmark %s to %s', user_bookmark, redirect_url)
         raise HTTPFound(redirect_url)
 
     @LoginRequired()
     @NotAnonymous()
     @view_config(
         route_name='my_account_perms', request_method='GET',
         renderer='rhodecode:templates/admin/my_account/my_account.mako')
     def my_account_perms(self):
         c = self.load_default_context()
         c.active = 'perms'
 
         c.perm_user = c.auth_user
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @view_config(
         route_name='my_account_notifications', request_method='GET',
         renderer='rhodecode:templates/admin/my_account/my_account.mako')
     def my_notifications(self):
         c = self.load_default_context()
         c.active = 'notifications'
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     @view_config(
         route_name='my_account_notifications_toggle_visibility',
         request_method='POST', renderer='json_ext')
     def my_notifications_toggle_visibility(self):
         user = self._rhodecode_db_user
         new_status = not user.user_data.get('notification_status', True)
         user.update_userdata(notification_status=new_status)
         Session().commit()
         return user.user_data['notification_status']
 
     @LoginRequired()
     @NotAnonymous()
     @view_config(
         route_name='my_account_edit',
         request_method='GET',
         renderer='rhodecode:templates/admin/my_account/my_account.mako')
     def my_account_edit(self):
         c = self.load_default_context()
         c.active = 'profile_edit'
         c.extern_type = c.user.extern_type
         c.extern_name = c.user.extern_name
 
         schema = user_schema.UserProfileSchema().bind(
             username=c.user.username, user_emails=c.user.emails)
         appstruct = {
             'username': c.user.username,
             'email': c.user.email,
             'firstname': c.user.firstname,
             'lastname': c.user.lastname,
             'description': c.user.description,
         }
         c.form = forms.RcForm(
             schema, appstruct=appstruct,
             action=h.route_path('my_account_update'),
             buttons=(forms.buttons.save, forms.buttons.reset))
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     @view_config(
         route_name='my_account_update',
         request_method='POST',
         renderer='rhodecode:templates/admin/my_account/my_account.mako')
     def my_account_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'profile_edit'
         c.perm_user = c.auth_user
         c.extern_type = c.user.extern_type
         c.extern_name = c.user.extern_name
 
         schema = user_schema.UserProfileSchema().bind(
             username=c.user.username, user_emails=c.user.emails)
         form = forms.RcForm(
             schema, buttons=(forms.buttons.save, forms.buttons.reset))
 
         controls = self.request.POST.items()
         try:
             valid_data = form.validate(controls)
             skip_attrs = ['admin', 'active', 'extern_type', 'extern_name',
                           'new_password', 'password_confirmation']
             if c.extern_type != "rhodecode":
                 # forbid updating username for external accounts
                 skip_attrs.append('username')
             old_email = c.user.email
             UserModel().update_user(
                      self._rhodecode_user.user_id, skip_attrs=skip_attrs,
                      **valid_data)
             if old_email != valid_data['email']:
                 old = UserEmailMap.query() \
-                    .filter(UserEmailMap.user == c.user).filter(UserEmailMap.email == valid_data['email']).first()
+                    .filter(UserEmailMap.user == c.user)\
+                    .filter(UserEmailMap.email == valid_data['email'])\
+                    .first()
                 old.email = old_email
             h.flash(_('Your account was updated successfully'), category='success')
             Session().commit()
         except forms.ValidationFailure as e:
             c.form = e
             return self._get_template_context(c)
         except Exception:
             log.exception("Exception updating user")
             h.flash(_('Error occurred during update of user'),
                     category='error')
         raise HTTPFound(h.route_path('my_account_profile'))
 
     def _get_pull_requests_list(self, statuses):
         draw, start, limit = self._extract_chunk(self.request)
         search_q, order_by, order_dir = self._extract_ordering(self.request)
+
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/data_table/_dt_elements.mako')
 
         pull_requests = PullRequestModel().get_im_participating_in(
             user_id=self._rhodecode_user.user_id,
             statuses=statuses, query=search_q,
             offset=start, length=limit, order_by=order_by,
             order_dir=order_dir)
 
         pull_requests_total_count = PullRequestModel().count_im_participating_in(
             user_id=self._rhodecode_user.user_id, statuses=statuses, query=search_q)
 
         data = []
         comments_model = CommentsModel()
         for pr in pull_requests:
             repo_id = pr.target_repo_id
             comments_count = comments_model.get_all_comments(
-                repo_id, pull_request=pr, count_only=True)
+                repo_id, pull_request=pr, include_drafts=False, count_only=True)
             owned = pr.user_id == self._rhodecode_user.user_id
 
             data.append({
                 'target_repo': _render('pullrequest_target_repo',
                                        pr.target_repo.repo_name),
                 'name': _render('pullrequest_name',
                                 pr.pull_request_id, pr.pull_request_state,
                                 pr.work_in_progress, pr.target_repo.repo_name,
                                 short=True),
                 'name_raw': pr.pull_request_id,
                 'status': _render('pullrequest_status',
                                   pr.calculated_review_status()),
                 'title': _render('pullrequest_title', pr.title, pr.description),
                 'description': h.escape(pr.description),
                 'updated_on': _render('pullrequest_updated_on',
-                                      h.datetime_to_time(pr.updated_on)),
+                                      h.datetime_to_time(pr.updated_on),
+                                      pr.versions_count),
                 'updated_on_raw': h.datetime_to_time(pr.updated_on),
                 'created_on': _render('pullrequest_updated_on',
                                       h.datetime_to_time(pr.created_on)),
                 'created_on_raw': h.datetime_to_time(pr.created_on),
                 'state': pr.pull_request_state,
                 'author': _render('pullrequest_author',
                                   pr.author.full_contact, ),
                 'author_raw': pr.author.full_name,
                 'comments': _render('pullrequest_comments', comments_count),
                 'comments_raw': comments_count,
                 'closed': pr.is_closed(),
                 'owned': owned
             })
 
         # json used to render the grid
         data = ({
             'draw': draw,
             'data': data,
             'recordsTotal': pull_requests_total_count,
             'recordsFiltered': pull_requests_total_count,
         })
         return data
 
     @LoginRequired()
     @NotAnonymous()
     @view_config(
         route_name='my_account_pullrequests',
         request_method='GET',
         renderer='rhodecode:templates/admin/my_account/my_account.mako')
     def my_account_pullrequests(self):
         c = self.load_default_context()
         c.active = 'pullrequests'
         req_get = self.request.GET
 
         c.closed = str2bool(req_get.get('pr_show_closed'))
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @view_config(
         route_name='my_account_pullrequests_data',
         request_method='GET', renderer='json_ext')
     def my_account_pullrequests_data(self):
         self.load_default_context()
         req_get = self.request.GET
         closed = str2bool(req_get.get('closed'))
 
         statuses = [PullRequest.STATUS_NEW, PullRequest.STATUS_OPEN]
         if closed:
             statuses += [PullRequest.STATUS_CLOSED]
 
         data = self._get_pull_requests_list(statuses=statuses)
         return data
 
     @LoginRequired()
     @NotAnonymous()
     @view_config(
         route_name='my_account_user_group_membership',
         request_method='GET',
         renderer='rhodecode:templates/admin/my_account/my_account.mako')
     def my_account_user_group_membership(self):
         c = self.load_default_context()
         c.active = 'user_group_membership'
         groups = [UserGroupModel.get_user_groups_as_dict(group.users_group)
                   for group in self._rhodecode_db_user.group_member]
         c.user_groups = json.dumps(groups)
         return self._get_template_context(c)

rhodecode/apps/repository/__init__.py

@@ -1,543 +1,548 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 from rhodecode.apps._base import add_route_with_slash
 
 
 def includeme(config):
 
     # repo creating checks, special cases that aren't repo routes
     config.add_route(
         name='repo_creating',
         pattern='/{repo_name:.*?[^/]}/repo_creating')
 
     config.add_route(
         name='repo_creating_check',
         pattern='/{repo_name:.*?[^/]}/repo_creating_check')
 
     # Summary
     # NOTE(marcink): one additional route is defined in very bottom, catch
     # all pattern
     config.add_route(
         name='repo_summary_explicit',
         pattern='/{repo_name:.*?[^/]}/summary', repo_route=True)
     config.add_route(
         name='repo_summary_commits',
         pattern='/{repo_name:.*?[^/]}/summary-commits', repo_route=True)
 
     # Commits
     config.add_route(
         name='repo_commit',
         pattern='/{repo_name:.*?[^/]}/changeset/{commit_id}', repo_route=True)
 
     config.add_route(
         name='repo_commit_children',
         pattern='/{repo_name:.*?[^/]}/changeset_children/{commit_id}', repo_route=True)
 
     config.add_route(
         name='repo_commit_parents',
         pattern='/{repo_name:.*?[^/]}/changeset_parents/{commit_id}', repo_route=True)
 
     config.add_route(
         name='repo_commit_raw',
         pattern='/{repo_name:.*?[^/]}/changeset-diff/{commit_id}', repo_route=True)
 
     config.add_route(
         name='repo_commit_patch',
         pattern='/{repo_name:.*?[^/]}/changeset-patch/{commit_id}', repo_route=True)
 
     config.add_route(
         name='repo_commit_download',
         pattern='/{repo_name:.*?[^/]}/changeset-download/{commit_id}', repo_route=True)
 
     config.add_route(
         name='repo_commit_data',
         pattern='/{repo_name:.*?[^/]}/changeset-data/{commit_id}', repo_route=True)
 
     config.add_route(
         name='repo_commit_comment_create',
         pattern='/{repo_name:.*?[^/]}/changeset/{commit_id}/comment/create', repo_route=True)
 
     config.add_route(
         name='repo_commit_comment_preview',
         pattern='/{repo_name:.*?[^/]}/changeset/{commit_id}/comment/preview', repo_route=True)
 
     config.add_route(
         name='repo_commit_comment_history_view',
         pattern='/{repo_name:.*?[^/]}/changeset/{commit_id}/comment/{comment_history_id}/history_view', repo_route=True)
 
     config.add_route(
         name='repo_commit_comment_attachment_upload',
         pattern='/{repo_name:.*?[^/]}/changeset/{commit_id}/comment/attachment_upload', repo_route=True)
 
     config.add_route(
         name='repo_commit_comment_delete',
         pattern='/{repo_name:.*?[^/]}/changeset/{commit_id}/comment/{comment_id}/delete', repo_route=True)
 
     config.add_route(
         name='repo_commit_comment_edit',
         pattern='/{repo_name:.*?[^/]}/changeset/{commit_id}/comment/{comment_id}/edit', repo_route=True)
 
     # still working url for backward compat.
     config.add_route(
         name='repo_commit_raw_deprecated',
         pattern='/{repo_name:.*?[^/]}/raw-changeset/{commit_id}', repo_route=True)
 
     # Files
     config.add_route(
         name='repo_archivefile',
         pattern='/{repo_name:.*?[^/]}/archive/{fname:.*}', repo_route=True)
 
     config.add_route(
         name='repo_files_diff',
         pattern='/{repo_name:.*?[^/]}/diff/{f_path:.*}', repo_route=True)
     config.add_route(  # legacy route to make old links work
         name='repo_files_diff_2way_redirect',
         pattern='/{repo_name:.*?[^/]}/diff-2way/{f_path:.*}', repo_route=True)
 
     config.add_route(
         name='repo_files',
         pattern='/{repo_name:.*?[^/]}/files/{commit_id}/{f_path:.*}', repo_route=True)
     config.add_route(
         name='repo_files:default_path',
         pattern='/{repo_name:.*?[^/]}/files/{commit_id}/', repo_route=True)
     config.add_route(
         name='repo_files:default_commit',
         pattern='/{repo_name:.*?[^/]}/files', repo_route=True)
 
     config.add_route(
         name='repo_files:rendered',
         pattern='/{repo_name:.*?[^/]}/render/{commit_id}/{f_path:.*}', repo_route=True)
 
     config.add_route(
         name='repo_files:annotated',
         pattern='/{repo_name:.*?[^/]}/annotate/{commit_id}/{f_path:.*}', repo_route=True)
     config.add_route(
         name='repo_files:annotated_previous',
         pattern='/{repo_name:.*?[^/]}/annotate-previous/{commit_id}/{f_path:.*}', repo_route=True)
 
     config.add_route(
         name='repo_nodetree_full',
         pattern='/{repo_name:.*?[^/]}/nodetree_full/{commit_id}/{f_path:.*}', repo_route=True)
     config.add_route(
         name='repo_nodetree_full:default_path',
         pattern='/{repo_name:.*?[^/]}/nodetree_full/{commit_id}/', repo_route=True)
 
     config.add_route(
         name='repo_files_nodelist',
         pattern='/{repo_name:.*?[^/]}/nodelist/{commit_id}/{f_path:.*}', repo_route=True)
 
     config.add_route(
         name='repo_file_raw',
         pattern='/{repo_name:.*?[^/]}/raw/{commit_id}/{f_path:.*}', repo_route=True)
 
     config.add_route(
         name='repo_file_download',
         pattern='/{repo_name:.*?[^/]}/download/{commit_id}/{f_path:.*}', repo_route=True)
     config.add_route(  # backward compat to keep old links working
         name='repo_file_download:legacy',
         pattern='/{repo_name:.*?[^/]}/rawfile/{commit_id}/{f_path:.*}',
         repo_route=True)
 
     config.add_route(
         name='repo_file_history',
         pattern='/{repo_name:.*?[^/]}/history/{commit_id}/{f_path:.*}', repo_route=True)
 
     config.add_route(
         name='repo_file_authors',
         pattern='/{repo_name:.*?[^/]}/authors/{commit_id}/{f_path:.*}', repo_route=True)
 
     config.add_route(
         name='repo_files_check_head',
         pattern='/{repo_name:.*?[^/]}/check_head/{commit_id}/{f_path:.*}',
         repo_route=True)
     config.add_route(
         name='repo_files_remove_file',
         pattern='/{repo_name:.*?[^/]}/remove_file/{commit_id}/{f_path:.*}',
         repo_route=True)
     config.add_route(
         name='repo_files_delete_file',
         pattern='/{repo_name:.*?[^/]}/delete_file/{commit_id}/{f_path:.*}',
         repo_route=True)
     config.add_route(
         name='repo_files_edit_file',
         pattern='/{repo_name:.*?[^/]}/edit_file/{commit_id}/{f_path:.*}',
         repo_route=True)
     config.add_route(
         name='repo_files_update_file',
         pattern='/{repo_name:.*?[^/]}/update_file/{commit_id}/{f_path:.*}',
         repo_route=True)
     config.add_route(
         name='repo_files_add_file',
         pattern='/{repo_name:.*?[^/]}/add_file/{commit_id}/{f_path:.*}',
         repo_route=True)
     config.add_route(
         name='repo_files_upload_file',
         pattern='/{repo_name:.*?[^/]}/upload_file/{commit_id}/{f_path:.*}',
         repo_route=True)
     config.add_route(
         name='repo_files_create_file',
         pattern='/{repo_name:.*?[^/]}/create_file/{commit_id}/{f_path:.*}',
         repo_route=True)
 
     # Refs data
     config.add_route(
         name='repo_refs_data',
         pattern='/{repo_name:.*?[^/]}/refs-data', repo_route=True)
 
     config.add_route(
         name='repo_refs_changelog_data',
         pattern='/{repo_name:.*?[^/]}/refs-data-changelog', repo_route=True)
 
     config.add_route(
         name='repo_stats',
         pattern='/{repo_name:.*?[^/]}/repo_stats/{commit_id}', repo_route=True)
 
     # Commits
     config.add_route(
         name='repo_commits',
         pattern='/{repo_name:.*?[^/]}/commits', repo_route=True)
     config.add_route(
         name='repo_commits_file',
         pattern='/{repo_name:.*?[^/]}/commits/{commit_id}/{f_path:.*}', repo_route=True)
     config.add_route(
         name='repo_commits_elements',
         pattern='/{repo_name:.*?[^/]}/commits_elements', repo_route=True)
     config.add_route(
         name='repo_commits_elements_file',
         pattern='/{repo_name:.*?[^/]}/commits_elements/{commit_id}/{f_path:.*}', repo_route=True)
 
     # Changelog (old deprecated name for commits page)
     config.add_route(
         name='repo_changelog',
         pattern='/{repo_name:.*?[^/]}/changelog', repo_route=True)
     config.add_route(
         name='repo_changelog_file',
         pattern='/{repo_name:.*?[^/]}/changelog/{commit_id}/{f_path:.*}', repo_route=True)
 
     # Compare
     config.add_route(
         name='repo_compare_select',
         pattern='/{repo_name:.*?[^/]}/compare', repo_route=True)
 
     config.add_route(
         name='repo_compare',
         pattern='/{repo_name:.*?[^/]}/compare/{source_ref_type}@{source_ref:.*?}...{target_ref_type}@{target_ref:.*?}', repo_route=True)
 
     # Tags
     config.add_route(
         name='tags_home',
         pattern='/{repo_name:.*?[^/]}/tags', repo_route=True)
 
     # Branches
     config.add_route(
         name='branches_home',
         pattern='/{repo_name:.*?[^/]}/branches', repo_route=True)
 
     # Bookmarks
     config.add_route(
         name='bookmarks_home',
         pattern='/{repo_name:.*?[^/]}/bookmarks', repo_route=True)
 
     # Forks
     config.add_route(
         name='repo_fork_new',
         pattern='/{repo_name:.*?[^/]}/fork', repo_route=True,
         repo_forbid_when_archived=True,
         repo_accepted_types=['hg', 'git'])
 
     config.add_route(
         name='repo_fork_create',
         pattern='/{repo_name:.*?[^/]}/fork/create', repo_route=True,
         repo_forbid_when_archived=True,
         repo_accepted_types=['hg', 'git'])
 
     config.add_route(
         name='repo_forks_show_all',
         pattern='/{repo_name:.*?[^/]}/forks', repo_route=True,
         repo_accepted_types=['hg', 'git'])
     config.add_route(
         name='repo_forks_data',
         pattern='/{repo_name:.*?[^/]}/forks/data', repo_route=True,
         repo_accepted_types=['hg', 'git'])
 
     # Pull Requests
     config.add_route(
         name='pullrequest_show',
         pattern='/{repo_name:.*?[^/]}/pull-request/{pull_request_id:\d+}',
         repo_route=True)
 
     config.add_route(
         name='pullrequest_show_all',
         pattern='/{repo_name:.*?[^/]}/pull-request',
         repo_route=True, repo_accepted_types=['hg', 'git'])
 
     config.add_route(
         name='pullrequest_show_all_data',
         pattern='/{repo_name:.*?[^/]}/pull-request-data',
         repo_route=True, repo_accepted_types=['hg', 'git'])
 
     config.add_route(
         name='pullrequest_repo_refs',
         pattern='/{repo_name:.*?[^/]}/pull-request/refs/{target_repo_name:.*?[^/]}',
         repo_route=True)
 
     config.add_route(
         name='pullrequest_repo_targets',
         pattern='/{repo_name:.*?[^/]}/pull-request/repo-targets',
         repo_route=True)
 
     config.add_route(
         name='pullrequest_new',
         pattern='/{repo_name:.*?[^/]}/pull-request/new',
         repo_route=True, repo_accepted_types=['hg', 'git'],
         repo_forbid_when_archived=True)
 
     config.add_route(
         name='pullrequest_create',
         pattern='/{repo_name:.*?[^/]}/pull-request/create',
         repo_route=True, repo_accepted_types=['hg', 'git'],
         repo_forbid_when_archived=True)
 
     config.add_route(
         name='pullrequest_update',
         pattern='/{repo_name:.*?[^/]}/pull-request/{pull_request_id:\d+}/update',
         repo_route=True, repo_forbid_when_archived=True)
 
     config.add_route(
         name='pullrequest_merge',
         pattern='/{repo_name:.*?[^/]}/pull-request/{pull_request_id:\d+}/merge',
         repo_route=True, repo_forbid_when_archived=True)
 
     config.add_route(
         name='pullrequest_delete',
         pattern='/{repo_name:.*?[^/]}/pull-request/{pull_request_id:\d+}/delete',
         repo_route=True, repo_forbid_when_archived=True)
 
     config.add_route(
         name='pullrequest_comment_create',
         pattern='/{repo_name:.*?[^/]}/pull-request/{pull_request_id:\d+}/comment',
         repo_route=True)
 
     config.add_route(
         name='pullrequest_comment_edit',
         pattern='/{repo_name:.*?[^/]}/pull-request/{pull_request_id:\d+}/comment/{comment_id}/edit',
         repo_route=True, repo_accepted_types=['hg', 'git'])
 
     config.add_route(
         name='pullrequest_comment_delete',
         pattern='/{repo_name:.*?[^/]}/pull-request/{pull_request_id:\d+}/comment/{comment_id}/delete',
         repo_route=True, repo_accepted_types=['hg', 'git'])
 
     config.add_route(
         name='pullrequest_comments',
         pattern='/{repo_name:.*?[^/]}/pull-request/{pull_request_id:\d+}/comments',
         repo_route=True)
 
     config.add_route(
         name='pullrequest_todos',
         pattern='/{repo_name:.*?[^/]}/pull-request/{pull_request_id:\d+}/todos',
         repo_route=True)
 
+    config.add_route(
+        name='pullrequest_drafts',
+        pattern='/{repo_name:.*?[^/]}/pull-request/{pull_request_id:\d+}/drafts',
+        repo_route=True)
+
     # Artifacts, (EE feature)
     config.add_route(
         name='repo_artifacts_list',
         pattern='/{repo_name:.*?[^/]}/artifacts', repo_route=True)
 
     # Settings
     config.add_route(
         name='edit_repo',
         pattern='/{repo_name:.*?[^/]}/settings', repo_route=True)
     # update is POST on edit_repo
 
     # Settings advanced
     config.add_route(
         name='edit_repo_advanced',
         pattern='/{repo_name:.*?[^/]}/settings/advanced', repo_route=True)
     config.add_route(
         name='edit_repo_advanced_archive',
         pattern='/{repo_name:.*?[^/]}/settings/advanced/archive', repo_route=True)
     config.add_route(
         name='edit_repo_advanced_delete',
         pattern='/{repo_name:.*?[^/]}/settings/advanced/delete', repo_route=True)
     config.add_route(
         name='edit_repo_advanced_locking',
         pattern='/{repo_name:.*?[^/]}/settings/advanced/locking', repo_route=True)
     config.add_route(
         name='edit_repo_advanced_journal',
         pattern='/{repo_name:.*?[^/]}/settings/advanced/journal', repo_route=True)
     config.add_route(
         name='edit_repo_advanced_fork',
         pattern='/{repo_name:.*?[^/]}/settings/advanced/fork', repo_route=True)
 
     config.add_route(
         name='edit_repo_advanced_hooks',
         pattern='/{repo_name:.*?[^/]}/settings/advanced/hooks', repo_route=True)
 
     # Caches
     config.add_route(
         name='edit_repo_caches',
         pattern='/{repo_name:.*?[^/]}/settings/caches', repo_route=True)
 
     # Permissions
     config.add_route(
         name='edit_repo_perms',
         pattern='/{repo_name:.*?[^/]}/settings/permissions', repo_route=True)
 
     config.add_route(
         name='edit_repo_perms_set_private',
         pattern='/{repo_name:.*?[^/]}/settings/permissions/set_private', repo_route=True)
 
     # Permissions Branch (EE feature)
     config.add_route(
         name='edit_repo_perms_branch',
         pattern='/{repo_name:.*?[^/]}/settings/branch_permissions', repo_route=True)
     config.add_route(
         name='edit_repo_perms_branch_delete',
         pattern='/{repo_name:.*?[^/]}/settings/branch_permissions/{rule_id}/delete',
         repo_route=True)
 
     # Maintenance
     config.add_route(
         name='edit_repo_maintenance',
         pattern='/{repo_name:.*?[^/]}/settings/maintenance', repo_route=True)
 
     config.add_route(
         name='edit_repo_maintenance_execute',
         pattern='/{repo_name:.*?[^/]}/settings/maintenance/execute', repo_route=True)
 
     # Fields
     config.add_route(
         name='edit_repo_fields',
         pattern='/{repo_name:.*?[^/]}/settings/fields', repo_route=True)
     config.add_route(
         name='edit_repo_fields_create',
         pattern='/{repo_name:.*?[^/]}/settings/fields/create', repo_route=True)
     config.add_route(
         name='edit_repo_fields_delete',
         pattern='/{repo_name:.*?[^/]}/settings/fields/{field_id}/delete', repo_route=True)
 
     # Locking
     config.add_route(
         name='repo_edit_toggle_locking',
         pattern='/{repo_name:.*?[^/]}/settings/toggle_locking', repo_route=True)
 
     # Remote
     config.add_route(
         name='edit_repo_remote',
         pattern='/{repo_name:.*?[^/]}/settings/remote', repo_route=True)
     config.add_route(
         name='edit_repo_remote_pull',
         pattern='/{repo_name:.*?[^/]}/settings/remote/pull', repo_route=True)
     config.add_route(
         name='edit_repo_remote_push',
         pattern='/{repo_name:.*?[^/]}/settings/remote/push', repo_route=True)
 
     # Statistics
     config.add_route(
         name='edit_repo_statistics',
         pattern='/{repo_name:.*?[^/]}/settings/statistics', repo_route=True)
     config.add_route(
         name='edit_repo_statistics_reset',
         pattern='/{repo_name:.*?[^/]}/settings/statistics/update', repo_route=True)
 
     # Issue trackers
     config.add_route(
         name='edit_repo_issuetracker',
         pattern='/{repo_name:.*?[^/]}/settings/issue_trackers', repo_route=True)
     config.add_route(
         name='edit_repo_issuetracker_test',
         pattern='/{repo_name:.*?[^/]}/settings/issue_trackers/test', repo_route=True)
     config.add_route(
         name='edit_repo_issuetracker_delete',
         pattern='/{repo_name:.*?[^/]}/settings/issue_trackers/delete', repo_route=True)
     config.add_route(
         name='edit_repo_issuetracker_update',
         pattern='/{repo_name:.*?[^/]}/settings/issue_trackers/update', repo_route=True)
 
     # VCS Settings
     config.add_route(
         name='edit_repo_vcs',
         pattern='/{repo_name:.*?[^/]}/settings/vcs', repo_route=True)
     config.add_route(
         name='edit_repo_vcs_update',
         pattern='/{repo_name:.*?[^/]}/settings/vcs/update', repo_route=True)
 
     # svn pattern
     config.add_route(
         name='edit_repo_vcs_svn_pattern_delete',
         pattern='/{repo_name:.*?[^/]}/settings/vcs/svn_pattern/delete', repo_route=True)
 
     # Repo Review Rules (EE feature)
     config.add_route(
         name='repo_reviewers',
         pattern='/{repo_name:.*?[^/]}/settings/review/rules', repo_route=True)
 
     config.add_route(
         name='repo_default_reviewers_data',
         pattern='/{repo_name:.*?[^/]}/settings/review/default-reviewers', repo_route=True)
 
     # Repo Automation (EE feature)
     config.add_route(
         name='repo_automation',
         pattern='/{repo_name:.*?[^/]}/settings/automation', repo_route=True)
 
     # Strip
     config.add_route(
         name='edit_repo_strip',
         pattern='/{repo_name:.*?[^/]}/settings/strip', repo_route=True)
 
     config.add_route(
         name='strip_check',
         pattern='/{repo_name:.*?[^/]}/settings/strip_check', repo_route=True)
 
     config.add_route(
         name='strip_execute',
         pattern='/{repo_name:.*?[^/]}/settings/strip_execute', repo_route=True)
 
     # Audit logs
     config.add_route(
         name='edit_repo_audit_logs',
         pattern='/{repo_name:.*?[^/]}/settings/audit_logs', repo_route=True)
 
     # ATOM/RSS Feed, shouldn't contain slashes for outlook compatibility
     config.add_route(
         name='rss_feed_home',
         pattern='/{repo_name:.*?[^/]}/feed-rss', repo_route=True)
 
     config.add_route(
         name='atom_feed_home',
         pattern='/{repo_name:.*?[^/]}/feed-atom', repo_route=True)
 
     config.add_route(
         name='rss_feed_home_old',
         pattern='/{repo_name:.*?[^/]}/feed/rss', repo_route=True)
 
     config.add_route(
         name='atom_feed_home_old',
         pattern='/{repo_name:.*?[^/]}/feed/atom', repo_route=True)
 
     # NOTE(marcink): needs to be at the end for catch-all
     add_route_with_slash(
         config,
         name='repo_summary',
         pattern='/{repo_name:.*?[^/]}', repo_route=True)
 
     # Scan module for configuration decorators.
     config.scan('.views', ignore='.tests')

rhodecode/apps/repository/tests/test_repo_pullrequests.py

@@ -1,1661 +1,1658 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import mock
 import pytest
 
 import rhodecode
 from rhodecode.lib.vcs.backends.base import MergeResponse, MergeFailureReason
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.lib import helpers as h
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.db import (
     PullRequest, ChangesetStatus, UserLog, Notification, ChangesetComment, Repository)
 from rhodecode.model.meta import Session
 from rhodecode.model.pull_request import PullRequestModel
 from rhodecode.model.user import UserModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.tests import (
     assert_session_flash, TEST_USER_ADMIN_LOGIN, TEST_USER_REGULAR_LOGIN)
 
 
 def route_path(name, params=None, **kwargs):
     import urllib
 
     base_url = {
         'repo_changelog': '/{repo_name}/changelog',
         'repo_changelog_file': '/{repo_name}/changelog/{commit_id}/{f_path}',
         'repo_commits': '/{repo_name}/commits',
         'repo_commits_file': '/{repo_name}/commits/{commit_id}/{f_path}',
         'pullrequest_show': '/{repo_name}/pull-request/{pull_request_id}',
         'pullrequest_show_all': '/{repo_name}/pull-request',
         'pullrequest_show_all_data': '/{repo_name}/pull-request-data',
         'pullrequest_repo_refs': '/{repo_name}/pull-request/refs/{target_repo_name:.*?[^/]}',
         'pullrequest_repo_targets': '/{repo_name}/pull-request/repo-destinations',
         'pullrequest_new': '/{repo_name}/pull-request/new',
         'pullrequest_create': '/{repo_name}/pull-request/create',
         'pullrequest_update': '/{repo_name}/pull-request/{pull_request_id}/update',
         'pullrequest_merge': '/{repo_name}/pull-request/{pull_request_id}/merge',
         'pullrequest_delete': '/{repo_name}/pull-request/{pull_request_id}/delete',
         'pullrequest_comment_create': '/{repo_name}/pull-request/{pull_request_id}/comment',
         'pullrequest_comment_delete': '/{repo_name}/pull-request/{pull_request_id}/comment/{comment_id}/delete',
         'pullrequest_comment_edit': '/{repo_name}/pull-request/{pull_request_id}/comment/{comment_id}/edit',
     }[name].format(**kwargs)
 
     if params:
         base_url = '{}?{}'.format(base_url, urllib.urlencode(params))
     return base_url
 
 
 @pytest.mark.usefixtures('app', 'autologin_user')
 @pytest.mark.backends("git", "hg")
 class TestPullrequestsView(object):
 
     def test_index(self, backend):
         self.app.get(route_path(
             'pullrequest_new',
             repo_name=backend.repo_name))
 
     def test_option_menu_create_pull_request_exists(self, backend):
         repo_name = backend.repo_name
         response = self.app.get(h.route_path('repo_summary', repo_name=repo_name))
 
         create_pr_link = '<a href="%s">Create Pull Request</a>' % route_path(
             'pullrequest_new', repo_name=repo_name)
         response.mustcontain(create_pr_link)
 
     def test_create_pr_form_with_raw_commit_id(self, backend):
         repo = backend.repo
 
         self.app.get(
             route_path('pullrequest_new', repo_name=repo.repo_name,
                        commit=repo.get_commit().raw_id),
             status=200)
 
     @pytest.mark.parametrize('pr_merge_enabled', [True, False])
     @pytest.mark.parametrize('range_diff', ["0", "1"])
     def test_show(self, pr_util, pr_merge_enabled, range_diff):
         pull_request = pr_util.create_pull_request(
             mergeable=pr_merge_enabled, enable_notifications=False)
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id,
             params={'range-diff': range_diff}))
 
         for commit_id in pull_request.revisions:
             response.mustcontain(commit_id)
 
         response.mustcontain(pull_request.target_ref_parts.type)
         response.mustcontain(pull_request.target_ref_parts.name)
 
         response.mustcontain('class="pull-request-merge"')
 
         if pr_merge_enabled:
             response.mustcontain('Pull request reviewer approval is pending')
         else:
             response.mustcontain('Server-side pull request merging is disabled.')
 
         if range_diff == "1":
             response.mustcontain('Turn off: Show the diff as commit range')
 
     def test_show_versions_of_pr(self, backend, csrf_token):
         commits = [
             {'message': 'initial-commit',
              'added': [FileNode('test-file.txt', 'LINE1\n')]},
 
             {'message': 'commit-1',
              'changed': [FileNode('test-file.txt', 'LINE1\nLINE2\n')]},
             # Above is the initial version of PR that changes a single line
 
             # from now on we'll add 3x commit adding a nother line on each step
             {'message': 'commit-2',
              'changed': [FileNode('test-file.txt', 'LINE1\nLINE2\nLINE3\n')]},
 
             {'message': 'commit-3',
              'changed': [FileNode('test-file.txt', 'LINE1\nLINE2\nLINE3\nLINE4\n')]},
 
             {'message': 'commit-4',
              'changed': [FileNode('test-file.txt', 'LINE1\nLINE2\nLINE3\nLINE4\nLINE5\n')]},
         ]
 
         commit_ids = backend.create_master_repo(commits)
         target = backend.create_repo(heads=['initial-commit'])
         source = backend.create_repo(heads=['commit-1'])
         source_repo_name = source.repo_name
         target_repo_name = target.repo_name
 
         target_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name, commit_id=commit_ids['initial-commit'])
         source_ref = 'branch:{branch}:{commit_id}'.format(
              branch=backend.default_branch_name, commit_id=commit_ids['commit-1'])
 
         response = self.app.post(
             route_path('pullrequest_create', repo_name=source.repo_name),
             [
                 ('source_repo', source_repo_name),
                 ('source_ref', source_ref),
                 ('target_repo', target_repo_name),
                 ('target_ref',  target_ref),
                 ('common_ancestor', commit_ids['initial-commit']),
                 ('pullrequest_title', 'Title'),
                 ('pullrequest_desc', 'Description'),
                 ('description_renderer', 'markdown'),
                 ('__start__', 'review_members:sequence'),
                     ('__start__', 'reviewer:mapping'),
                         ('user_id', '1'),
                         ('__start__', 'reasons:sequence'),
                             ('reason', 'Some reason'),
                         ('__end__', 'reasons:sequence'),
                         ('__start__', 'rules:sequence'),
                         ('__end__', 'rules:sequence'),
                         ('mandatory', 'False'),
                     ('__end__', 'reviewer:mapping'),
                 ('__end__', 'review_members:sequence'),
                 ('__start__', 'revisions:sequence'),
                     ('revisions', commit_ids['commit-1']),
                 ('__end__', 'revisions:sequence'),
                 ('user', ''),
                 ('csrf_token', csrf_token),
             ],
             status=302)
 
         location = response.headers['Location']
 
         pull_request_id = location.rsplit('/', 1)[1]
         assert pull_request_id != 'new'
         pull_request = PullRequest.get(int(pull_request_id))
 
         pull_request_id = pull_request.pull_request_id
 
         # Show initial version of PR
         response = self.app.get(
             route_path('pullrequest_show',
                        repo_name=target_repo_name,
                        pull_request_id=pull_request_id))
 
         response.mustcontain('commit-1')
         response.mustcontain(no=['commit-2'])
         response.mustcontain(no=['commit-3'])
         response.mustcontain(no=['commit-4'])
 
         response.mustcontain('cb-addition"></span><span>LINE2</span>')
         response.mustcontain(no=['LINE3'])
         response.mustcontain(no=['LINE4'])
         response.mustcontain(no=['LINE5'])
 
         # update PR #1
         source_repo = Repository.get_by_repo_name(source_repo_name)
         backend.pull_heads(source_repo, heads=['commit-2'])
         response = self.app.post(
             route_path('pullrequest_update',
                        repo_name=target_repo_name, pull_request_id=pull_request_id),
             params={'update_commits': 'true', 'csrf_token': csrf_token})
 
         # update PR #2
         source_repo = Repository.get_by_repo_name(source_repo_name)
         backend.pull_heads(source_repo, heads=['commit-3'])
         response = self.app.post(
             route_path('pullrequest_update',
                        repo_name=target_repo_name, pull_request_id=pull_request_id),
             params={'update_commits': 'true', 'csrf_token': csrf_token})
 
         # update PR #3
         source_repo = Repository.get_by_repo_name(source_repo_name)
         backend.pull_heads(source_repo, heads=['commit-4'])
         response = self.app.post(
             route_path('pullrequest_update',
                        repo_name=target_repo_name, pull_request_id=pull_request_id),
             params={'update_commits': 'true', 'csrf_token': csrf_token})
 
         # Show final version !
         response = self.app.get(
             route_path('pullrequest_show',
                        repo_name=target_repo_name,
                        pull_request_id=pull_request_id))
 
         # 3 updates, and the latest == 4
         response.mustcontain('4 versions available for this pull request')
         response.mustcontain(no=['rhodecode diff rendering error'])
 
         # initial show must have 3 commits, and 3 adds
         response.mustcontain('commit-1')
         response.mustcontain('commit-2')
         response.mustcontain('commit-3')
         response.mustcontain('commit-4')
 
         response.mustcontain('cb-addition"></span><span>LINE2</span>')
         response.mustcontain('cb-addition"></span><span>LINE3</span>')
         response.mustcontain('cb-addition"></span><span>LINE4</span>')
         response.mustcontain('cb-addition"></span><span>LINE5</span>')
 
         # fetch versions
         pr = PullRequest.get(pull_request_id)
         versions = [x.pull_request_version_id for x in pr.versions.all()]
         assert len(versions) == 3
 
         # show v1,v2,v3,v4
         def cb_line(text):
             return 'cb-addition"></span><span>{}</span>'.format(text)
 
         def cb_context(text):
             return '<span class="cb-code"><span class="cb-action cb-context">' \
                    '</span><span>{}</span></span>'.format(text)
 
         commit_tests = {
             # in response, not in response
             1: (['commit-1'], ['commit-2', 'commit-3', 'commit-4']),
             2: (['commit-1', 'commit-2'], ['commit-3', 'commit-4']),
             3: (['commit-1', 'commit-2', 'commit-3'], ['commit-4']),
             4: (['commit-1', 'commit-2', 'commit-3', 'commit-4'], []),
         }
         diff_tests = {
             1: (['LINE2'], ['LINE3', 'LINE4', 'LINE5']),
             2: (['LINE2', 'LINE3'], ['LINE4', 'LINE5']),
             3: (['LINE2', 'LINE3', 'LINE4'], ['LINE5']),
             4: (['LINE2', 'LINE3', 'LINE4', 'LINE5'], []),
         }
         for idx, ver in enumerate(versions, 1):
 
             response = self.app.get(
                 route_path('pullrequest_show',
                            repo_name=target_repo_name,
                            pull_request_id=pull_request_id,
                            params={'version': ver}))
 
             response.mustcontain(no=['rhodecode diff rendering error'])
             response.mustcontain('Showing changes at v{}'.format(idx))
 
             yes, no = commit_tests[idx]
             for y in yes:
                 response.mustcontain(y)
             for n in no:
                 response.mustcontain(no=n)
 
             yes, no = diff_tests[idx]
             for y in yes:
                 response.mustcontain(cb_line(y))
             for n in no:
                 response.mustcontain(no=n)
 
         # show diff between versions
         diff_compare_tests = {
             1: (['LINE3'], ['LINE1', 'LINE2']),
             2: (['LINE3', 'LINE4'], ['LINE1', 'LINE2']),
             3: (['LINE3', 'LINE4', 'LINE5'], ['LINE1', 'LINE2']),
         }
         for idx, ver in enumerate(versions, 1):
             adds, context = diff_compare_tests[idx]
 
             to_ver = ver+1
             if idx == 3:
                 to_ver = 'latest'
 
             response = self.app.get(
                 route_path('pullrequest_show',
                            repo_name=target_repo_name,
                            pull_request_id=pull_request_id,
                            params={'from_version': versions[0], 'version': to_ver}))
 
             response.mustcontain(no=['rhodecode diff rendering error'])
 
             for a in adds:
                 response.mustcontain(cb_line(a))
             for c in context:
                 response.mustcontain(cb_context(c))
 
         # test version v2 -> v3
         response = self.app.get(
             route_path('pullrequest_show',
                        repo_name=target_repo_name,
                        pull_request_id=pull_request_id,
                        params={'from_version': versions[1], 'version': versions[2]}))
 
         response.mustcontain(cb_context('LINE1'))
         response.mustcontain(cb_context('LINE2'))
         response.mustcontain(cb_context('LINE3'))
         response.mustcontain(cb_line('LINE4'))
 
     def test_close_status_visibility(self, pr_util, user_util, csrf_token):
         # Logout
         response = self.app.post(
             h.route_path('logout'),
             params={'csrf_token': csrf_token})
         # Login as regular user
         response = self.app.post(h.route_path('login'),
                                  {'username': TEST_USER_REGULAR_LOGIN,
                                   'password': 'test12'})
 
         pull_request = pr_util.create_pull_request(
             author=TEST_USER_REGULAR_LOGIN)
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
 
         response.mustcontain('Server-side pull request merging is disabled.')
 
         assert_response = response.assert_response()
         # for regular user without a merge permissions, we don't see it
         assert_response.no_element_exists('#close-pull-request-action')
 
         user_util.grant_user_permission_to_repo(
             pull_request.target_repo,
             UserModel().get_by_username(TEST_USER_REGULAR_LOGIN),
             'repository.write')
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
 
         response.mustcontain('Server-side pull request merging is disabled.')
 
         assert_response = response.assert_response()
         # now regular user has a merge permissions, we have CLOSE button
         assert_response.one_element_exists('#close-pull-request-action')
 
     def test_show_invalid_commit_id(self, pr_util):
         # Simulating invalid revisions which will cause a lookup error
         pull_request = pr_util.create_pull_request()
         pull_request.revisions = ['invalid']
         Session().add(pull_request)
         Session().commit()
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
 
         for commit_id in pull_request.revisions:
             response.mustcontain(commit_id)
 
     def test_show_invalid_source_reference(self, pr_util):
         pull_request = pr_util.create_pull_request()
         pull_request.source_ref = 'branch:b:invalid'
         Session().add(pull_request)
         Session().commit()
 
         self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
 
     def test_edit_title_description(self, pr_util, csrf_token):
         pull_request = pr_util.create_pull_request()
         pull_request_id = pull_request.pull_request_id
 
         response = self.app.post(
             route_path('pullrequest_update',
                 repo_name=pull_request.target_repo.repo_name,
                 pull_request_id=pull_request_id),
             params={
                 'edit_pull_request': 'true',
                 'title': 'New title',
                 'description': 'New description',
                 'csrf_token': csrf_token})
 
         assert_session_flash(
             response, u'Pull request title & description updated.',
             category='success')
 
         pull_request = PullRequest.get(pull_request_id)
         assert pull_request.title == 'New title'
         assert pull_request.description == 'New description'
 
     def test_edit_title_description_closed(self, pr_util, csrf_token):
         pull_request = pr_util.create_pull_request()
         pull_request_id = pull_request.pull_request_id
         repo_name = pull_request.target_repo.repo_name
         pr_util.close()
 
         response = self.app.post(
             route_path('pullrequest_update',
                        repo_name=repo_name, pull_request_id=pull_request_id),
             params={
                 'edit_pull_request': 'true',
                 'title': 'New title',
                 'description': 'New description',
                 'csrf_token': csrf_token}, status=200)
         assert_session_flash(
             response, u'Cannot update closed pull requests.',
             category='error')
 
     def test_update_invalid_source_reference(self, pr_util, csrf_token):
         from rhodecode.lib.vcs.backends.base import UpdateFailureReason
 
         pull_request = pr_util.create_pull_request()
         pull_request.source_ref = 'branch:invalid-branch:invalid-commit-id'
         Session().add(pull_request)
         Session().commit()
 
         pull_request_id = pull_request.pull_request_id
 
         response = self.app.post(
             route_path('pullrequest_update',
                 repo_name=pull_request.target_repo.repo_name,
                 pull_request_id=pull_request_id),
             params={'update_commits': 'true', 'csrf_token': csrf_token})
 
         expected_msg = str(PullRequestModel.UPDATE_STATUS_MESSAGES[
             UpdateFailureReason.MISSING_SOURCE_REF])
         assert_session_flash(response, expected_msg, category='error')
 
     def test_missing_target_reference(self, pr_util, csrf_token):
         from rhodecode.lib.vcs.backends.base import MergeFailureReason
         pull_request = pr_util.create_pull_request(
             approved=True, mergeable=True)
         unicode_reference = u'branch:invalid-branch:invalid-commit-id'
         pull_request.target_ref = unicode_reference
         Session().add(pull_request)
         Session().commit()
 
         pull_request_id = pull_request.pull_request_id
         pull_request_url = route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.repo_name,
             pull_request_id=pull_request_id)
 
         response = self.app.get(pull_request_url)
         target_ref_id = 'invalid-branch'
         merge_resp = MergeResponse(
             True, True, '', MergeFailureReason.MISSING_TARGET_REF,
             metadata={'target_ref': PullRequest.unicode_to_reference(unicode_reference)})
         response.assert_response().element_contains(
             'div[data-role="merge-message"]', merge_resp.merge_status_message)
 
     def test_comment_and_close_pull_request_custom_message_approved(
             self, pr_util, csrf_token, xhr_header):
 
         pull_request = pr_util.create_pull_request(approved=True)
         pull_request_id = pull_request.pull_request_id
         author = pull_request.user_id
         repo = pull_request.target_repo.repo_id
 
         self.app.post(
             route_path('pullrequest_comment_create',
                        repo_name=pull_request.target_repo.scm_instance().name,
                        pull_request_id=pull_request_id),
             params={
                 'close_pull_request': '1',
                 'text': 'Closing a PR',
                 'csrf_token': csrf_token},
             extra_environ=xhr_header,)
 
         journal = UserLog.query()\
             .filter(UserLog.user_id == author)\
             .filter(UserLog.repository_id == repo) \
             .order_by(UserLog.user_log_id.asc()) \
             .all()
         assert journal[-1].action == 'repo.pull_request.close'
 
         pull_request = PullRequest.get(pull_request_id)
         assert pull_request.is_closed()
 
         status = ChangesetStatusModel().get_status(
             pull_request.source_repo, pull_request=pull_request)
         assert status == ChangesetStatus.STATUS_APPROVED
         comments = ChangesetComment().query() \
             .filter(ChangesetComment.pull_request == pull_request) \
             .order_by(ChangesetComment.comment_id.asc())\
             .all()
         assert comments[-1].text == 'Closing a PR'
 
     def test_comment_force_close_pull_request_rejected(
             self, pr_util, csrf_token, xhr_header):
         pull_request = pr_util.create_pull_request()
         pull_request_id = pull_request.pull_request_id
         PullRequestModel().update_reviewers(
             pull_request_id, [
                 (1, ['reason'], False, 'reviewer', []),
                 (2, ['reason2'], False, 'reviewer', [])],
             pull_request.author)
         author = pull_request.user_id
         repo = pull_request.target_repo.repo_id
 
         self.app.post(
             route_path('pullrequest_comment_create',
                 repo_name=pull_request.target_repo.scm_instance().name,
                 pull_request_id=pull_request_id),
             params={
                 'close_pull_request': '1',
                 'csrf_token': csrf_token},
             extra_environ=xhr_header)
 
         pull_request = PullRequest.get(pull_request_id)
 
         journal = UserLog.query()\
             .filter(UserLog.user_id == author, UserLog.repository_id == repo) \
             .order_by(UserLog.user_log_id.asc()) \
             .all()
         assert journal[-1].action == 'repo.pull_request.close'
 
         # check only the latest status, not the review status
         status = ChangesetStatusModel().get_status(
             pull_request.source_repo, pull_request=pull_request)
         assert status == ChangesetStatus.STATUS_REJECTED
 
     def test_comment_and_close_pull_request(
             self, pr_util, csrf_token, xhr_header):
         pull_request = pr_util.create_pull_request()
         pull_request_id = pull_request.pull_request_id
 
         response = self.app.post(
             route_path('pullrequest_comment_create',
                        repo_name=pull_request.target_repo.scm_instance().name,
                        pull_request_id=pull_request.pull_request_id),
             params={
                 'close_pull_request': 'true',
                 'csrf_token': csrf_token},
             extra_environ=xhr_header)
 
         assert response.json
 
         pull_request = PullRequest.get(pull_request_id)
         assert pull_request.is_closed()
 
         # check only the latest status, not the review status
         status = ChangesetStatusModel().get_status(
             pull_request.source_repo, pull_request=pull_request)
         assert status == ChangesetStatus.STATUS_REJECTED
 
     def test_comment_and_close_pull_request_try_edit_comment(
             self, pr_util, csrf_token, xhr_header
     ):
         pull_request = pr_util.create_pull_request()
         pull_request_id = pull_request.pull_request_id
         target_scm = pull_request.target_repo.scm_instance()
         target_scm_name = target_scm.name
 
         response = self.app.post(
             route_path(
                 'pullrequest_comment_create',
                 repo_name=target_scm_name,
                 pull_request_id=pull_request_id,
             ),
             params={
                 'close_pull_request': 'true',
                 'csrf_token': csrf_token,
             },
             extra_environ=xhr_header)
 
         assert response.json
 
         pull_request = PullRequest.get(pull_request_id)
         target_scm = pull_request.target_repo.scm_instance()
         target_scm_name = target_scm.name
         assert pull_request.is_closed()
 
         # check only the latest status, not the review status
         status = ChangesetStatusModel().get_status(
             pull_request.source_repo, pull_request=pull_request)
         assert status == ChangesetStatus.STATUS_REJECTED
 
-        comment_id = response.json.get('comment_id', None)
-        test_text = 'test'
-        response = self.app.post(
-            route_path(
-                'pullrequest_comment_edit',
-                repo_name=target_scm_name,
-                pull_request_id=pull_request_id,
-                comment_id=comment_id,
-            ),
-            extra_environ=xhr_header,
-            params={
-                'csrf_token': csrf_token,
-                'text': test_text,
-            },
-            status=403,
-        )
-        assert response.status_int == 403
+        for comment_id in response.json.keys():
+            test_text = 'test'
+            response = self.app.post(
+                route_path(
+                    'pullrequest_comment_edit',
+                    repo_name=target_scm_name,
+                    pull_request_id=pull_request_id,
+                    comment_id=comment_id,
+                ),
+                extra_environ=xhr_header,
+                params={
+                    'csrf_token': csrf_token,
+                    'text': test_text,
+                },
+                status=403,
+            )
+            assert response.status_int == 403
 
     def test_comment_and_comment_edit(self, pr_util, csrf_token, xhr_header):
         pull_request = pr_util.create_pull_request()
         target_scm = pull_request.target_repo.scm_instance()
         target_scm_name = target_scm.name
 
         response = self.app.post(
             route_path(
                 'pullrequest_comment_create',
                 repo_name=target_scm_name,
                 pull_request_id=pull_request.pull_request_id),
             params={
                 'csrf_token': csrf_token,
                 'text': 'init',
             },
             extra_environ=xhr_header,
         )
         assert response.json
 
-        comment_id = response.json.get('comment_id', None)
-        assert comment_id
-        test_text = 'test'
-        self.app.post(
-            route_path(
-                'pullrequest_comment_edit',
-                repo_name=target_scm_name,
-                pull_request_id=pull_request.pull_request_id,
-                comment_id=comment_id,
-            ),
-            extra_environ=xhr_header,
-            params={
-                'csrf_token': csrf_token,
-                'text': test_text,
-                'version': '0',
-            },
+        for comment_id in response.json.keys():
+            assert comment_id
+            test_text = 'test'
+            self.app.post(
+                route_path(
+                    'pullrequest_comment_edit',
+                    repo_name=target_scm_name,
+                    pull_request_id=pull_request.pull_request_id,
+                    comment_id=comment_id,
+                ),
+                extra_environ=xhr_header,
+                params={
+                    'csrf_token': csrf_token,
+                    'text': test_text,
+                    'version': '0',
+                },
 
-        )
-        text_form_db = ChangesetComment.query().filter(
-            ChangesetComment.comment_id == comment_id).first().text
-        assert test_text == text_form_db
+            )
+            text_form_db = ChangesetComment.query().filter(
+                ChangesetComment.comment_id == comment_id).first().text
+            assert test_text == text_form_db
 
     def test_comment_and_comment_edit(self, pr_util, csrf_token, xhr_header):
         pull_request = pr_util.create_pull_request()
         target_scm = pull_request.target_repo.scm_instance()
         target_scm_name = target_scm.name
 
         response = self.app.post(
             route_path(
                 'pullrequest_comment_create',
                 repo_name=target_scm_name,
                 pull_request_id=pull_request.pull_request_id),
             params={
                 'csrf_token': csrf_token,
                 'text': 'init',
             },
             extra_environ=xhr_header,
         )
         assert response.json
 
-        comment_id = response.json.get('comment_id', None)
-        assert comment_id
-        test_text = 'init'
-        response = self.app.post(
-            route_path(
-                'pullrequest_comment_edit',
-                repo_name=target_scm_name,
-                pull_request_id=pull_request.pull_request_id,
-                comment_id=comment_id,
-            ),
-            extra_environ=xhr_header,
-            params={
-                'csrf_token': csrf_token,
-                'text': test_text,
-                'version': '0',
-            },
-            status=404,
+        for comment_id in response.json.keys():
+            test_text = 'init'
+            response = self.app.post(
+                route_path(
+                    'pullrequest_comment_edit',
+                    repo_name=target_scm_name,
+                    pull_request_id=pull_request.pull_request_id,
+                    comment_id=comment_id,
+                ),
+                extra_environ=xhr_header,
+                params={
+                    'csrf_token': csrf_token,
+                    'text': test_text,
+                    'version': '0',
+                },
+                status=404,
 
-        )
-        assert response.status_int == 404
+            )
+            assert response.status_int == 404
 
     def test_comment_and_try_edit_already_edited(self, pr_util, csrf_token, xhr_header):
         pull_request = pr_util.create_pull_request()
         target_scm = pull_request.target_repo.scm_instance()
         target_scm_name = target_scm.name
 
         response = self.app.post(
             route_path(
                 'pullrequest_comment_create',
                 repo_name=target_scm_name,
                 pull_request_id=pull_request.pull_request_id),
             params={
                 'csrf_token': csrf_token,
                 'text': 'init',
             },
             extra_environ=xhr_header,
         )
         assert response.json
-        comment_id = response.json.get('comment_id', None)
-        assert comment_id
-
-        test_text = 'test'
-        self.app.post(
-            route_path(
-                'pullrequest_comment_edit',
-                repo_name=target_scm_name,
-                pull_request_id=pull_request.pull_request_id,
-                comment_id=comment_id,
-            ),
-            extra_environ=xhr_header,
-            params={
-                'csrf_token': csrf_token,
-                'text': test_text,
-                'version': '0',
-            },
+        for comment_id in response.json.keys():
+            test_text = 'test'
+            self.app.post(
+                route_path(
+                    'pullrequest_comment_edit',
+                    repo_name=target_scm_name,
+                    pull_request_id=pull_request.pull_request_id,
+                    comment_id=comment_id,
+                ),
+                extra_environ=xhr_header,
+                params={
+                    'csrf_token': csrf_token,
+                    'text': test_text,
+                    'version': '0',
+                },
 
-        )
-        test_text_v2 = 'test_v2'
-        response = self.app.post(
-            route_path(
-                'pullrequest_comment_edit',
-                repo_name=target_scm_name,
-                pull_request_id=pull_request.pull_request_id,
-                comment_id=comment_id,
-            ),
-            extra_environ=xhr_header,
-            params={
-                'csrf_token': csrf_token,
-                'text': test_text_v2,
-                'version': '0',
-            },
-            status=409,
-        )
-        assert response.status_int == 409
+            )
+            test_text_v2 = 'test_v2'
+            response = self.app.post(
+                route_path(
+                    'pullrequest_comment_edit',
+                    repo_name=target_scm_name,
+                    pull_request_id=pull_request.pull_request_id,
+                    comment_id=comment_id,
+                ),
+                extra_environ=xhr_header,
+                params={
+                    'csrf_token': csrf_token,
+                    'text': test_text_v2,
+                    'version': '0',
+                },
+                status=409,
+            )
+            assert response.status_int == 409
 
-        text_form_db = ChangesetComment.query().filter(
-            ChangesetComment.comment_id == comment_id).first().text
+            text_form_db = ChangesetComment.query().filter(
+                ChangesetComment.comment_id == comment_id).first().text
 
-        assert test_text == text_form_db
-        assert test_text_v2 != text_form_db
+            assert test_text == text_form_db
+            assert test_text_v2 != text_form_db
 
     def test_comment_and_comment_edit_permissions_forbidden(
             self, autologin_regular_user, user_regular, user_admin, pr_util,
             csrf_token, xhr_header):
         pull_request = pr_util.create_pull_request(
             author=user_admin.username, enable_notifications=False)
         comment = CommentsModel().create(
             text='test',
             repo=pull_request.target_repo.scm_instance().name,
             user=user_admin,
             pull_request=pull_request,
         )
         response = self.app.post(
             route_path(
                 'pullrequest_comment_edit',
                 repo_name=pull_request.target_repo.scm_instance().name,
                 pull_request_id=pull_request.pull_request_id,
                 comment_id=comment.comment_id,
             ),
             extra_environ=xhr_header,
             params={
                 'csrf_token': csrf_token,
                 'text': 'test_text',
             },
             status=403,
         )
         assert response.status_int == 403
 
     def test_create_pull_request(self, backend, csrf_token):
         commits = [
             {'message': 'ancestor'},
             {'message': 'change'},
             {'message': 'change2'},
         ]
         commit_ids = backend.create_master_repo(commits)
         target = backend.create_repo(heads=['ancestor'])
         source = backend.create_repo(heads=['change2'])
 
         response = self.app.post(
             route_path('pullrequest_create', repo_name=source.repo_name),
             [
                 ('source_repo', source.repo_name),
                 ('source_ref', 'branch:default:' + commit_ids['change2']),
                 ('target_repo', target.repo_name),
                 ('target_ref',  'branch:default:' + commit_ids['ancestor']),
                 ('common_ancestor', commit_ids['ancestor']),
                 ('pullrequest_title', 'Title'),
                 ('pullrequest_desc', 'Description'),
                 ('description_renderer', 'markdown'),
                 ('__start__', 'review_members:sequence'),
                     ('__start__', 'reviewer:mapping'),
                         ('user_id', '1'),
                         ('__start__', 'reasons:sequence'),
                             ('reason', 'Some reason'),
                         ('__end__', 'reasons:sequence'),
                         ('__start__', 'rules:sequence'),
                         ('__end__', 'rules:sequence'),
                         ('mandatory', 'False'),
                     ('__end__', 'reviewer:mapping'),
                 ('__end__', 'review_members:sequence'),
                 ('__start__', 'revisions:sequence'),
                     ('revisions', commit_ids['change']),
                     ('revisions', commit_ids['change2']),
                 ('__end__', 'revisions:sequence'),
                 ('user', ''),
                 ('csrf_token', csrf_token),
             ],
             status=302)
 
         location = response.headers['Location']
         pull_request_id = location.rsplit('/', 1)[1]
         assert pull_request_id != 'new'
         pull_request = PullRequest.get(int(pull_request_id))
 
         # check that we have now both revisions
         assert pull_request.revisions == [commit_ids['change2'], commit_ids['change']]
         assert pull_request.source_ref == 'branch:default:' + commit_ids['change2']
         expected_target_ref = 'branch:default:' + commit_ids['ancestor']
         assert pull_request.target_ref == expected_target_ref
 
     def test_reviewer_notifications(self, backend, csrf_token):
         # We have to use the app.post for this test so it will create the
         # notifications properly with the new PR
         commits = [
             {'message': 'ancestor',
              'added': [FileNode('file_A', content='content_of_ancestor')]},
             {'message': 'change',
              'added': [FileNode('file_a', content='content_of_change')]},
             {'message': 'change-child'},
             {'message': 'ancestor-child', 'parents': ['ancestor'],
              'added': [
                 FileNode('file_B', content='content_of_ancestor_child')]},
             {'message': 'ancestor-child-2'},
         ]
         commit_ids = backend.create_master_repo(commits)
         target = backend.create_repo(heads=['ancestor-child'])
         source = backend.create_repo(heads=['change'])
 
         response = self.app.post(
             route_path('pullrequest_create', repo_name=source.repo_name),
             [
                 ('source_repo', source.repo_name),
                 ('source_ref', 'branch:default:' + commit_ids['change']),
                 ('target_repo', target.repo_name),
                 ('target_ref',  'branch:default:' + commit_ids['ancestor-child']),
                 ('common_ancestor', commit_ids['ancestor']),
                 ('pullrequest_title', 'Title'),
                 ('pullrequest_desc', 'Description'),
                 ('description_renderer', 'markdown'),
                 ('__start__', 'review_members:sequence'),
                     ('__start__', 'reviewer:mapping'),
                         ('user_id', '2'),
                         ('__start__', 'reasons:sequence'),
                             ('reason', 'Some reason'),
                         ('__end__', 'reasons:sequence'),
                         ('__start__', 'rules:sequence'),
                         ('__end__', 'rules:sequence'),
                         ('mandatory', 'False'),
                     ('__end__', 'reviewer:mapping'),
                 ('__end__', 'review_members:sequence'),
                 ('__start__', 'revisions:sequence'),
                     ('revisions', commit_ids['change']),
                 ('__end__', 'revisions:sequence'),
                 ('user', ''),
                 ('csrf_token', csrf_token),
             ],
             status=302)
 
         location = response.headers['Location']
 
         pull_request_id = location.rsplit('/', 1)[1]
         assert pull_request_id != 'new'
         pull_request = PullRequest.get(int(pull_request_id))
 
         # Check that a notification was made
         notifications = Notification.query()\
             .filter(Notification.created_by == pull_request.author.user_id,
                     Notification.type_ == Notification.TYPE_PULL_REQUEST,
                     Notification.subject.contains(
                         "requested a pull request review. !%s" % pull_request_id))
         assert len(notifications.all()) == 1
 
         # Change reviewers and check that a notification was made
         PullRequestModel().update_reviewers(
             pull_request.pull_request_id, [
                 (1, [], False, 'reviewer', [])
             ],
             pull_request.author)
         assert len(notifications.all()) == 2
 
     def test_create_pull_request_stores_ancestor_commit_id(self, backend, csrf_token):
         commits = [
             {'message': 'ancestor',
              'added': [FileNode('file_A', content='content_of_ancestor')]},
             {'message': 'change',
              'added': [FileNode('file_a', content='content_of_change')]},
             {'message': 'change-child'},
             {'message': 'ancestor-child', 'parents': ['ancestor'],
              'added': [
                 FileNode('file_B', content='content_of_ancestor_child')]},
             {'message': 'ancestor-child-2'},
         ]
         commit_ids = backend.create_master_repo(commits)
         target = backend.create_repo(heads=['ancestor-child'])
         source = backend.create_repo(heads=['change'])
 
         response = self.app.post(
             route_path('pullrequest_create', repo_name=source.repo_name),
             [
                 ('source_repo', source.repo_name),
                 ('source_ref', 'branch:default:' + commit_ids['change']),
                 ('target_repo', target.repo_name),
                 ('target_ref',  'branch:default:' + commit_ids['ancestor-child']),
                 ('common_ancestor', commit_ids['ancestor']),
                 ('pullrequest_title', 'Title'),
                 ('pullrequest_desc', 'Description'),
                 ('description_renderer', 'markdown'),
                 ('__start__', 'review_members:sequence'),
                     ('__start__', 'reviewer:mapping'),
                         ('user_id', '1'),
                         ('__start__', 'reasons:sequence'),
                             ('reason', 'Some reason'),
                         ('__end__', 'reasons:sequence'),
                         ('__start__', 'rules:sequence'),
                         ('__end__', 'rules:sequence'),
                         ('mandatory', 'False'),
                     ('__end__', 'reviewer:mapping'),
                 ('__end__', 'review_members:sequence'),
                 ('__start__', 'revisions:sequence'),
                     ('revisions', commit_ids['change']),
                 ('__end__', 'revisions:sequence'),
                 ('user', ''),
                 ('csrf_token', csrf_token),
             ],
             status=302)
 
         location = response.headers['Location']
 
         pull_request_id = location.rsplit('/', 1)[1]
         assert pull_request_id != 'new'
         pull_request = PullRequest.get(int(pull_request_id))
 
         # target_ref has to point to the ancestor's commit_id in order to
         # show the correct diff
         expected_target_ref = 'branch:default:' + commit_ids['ancestor']
         assert pull_request.target_ref == expected_target_ref
 
         # Check generated diff contents
         response = response.follow()
         response.mustcontain(no=['content_of_ancestor'])
         response.mustcontain(no=['content_of_ancestor-child'])
         response.mustcontain('content_of_change')
 
     def test_merge_pull_request_enabled(self, pr_util, csrf_token):
         # Clear any previous calls to rcextensions
         rhodecode.EXTENSIONS.calls.clear()
 
         pull_request = pr_util.create_pull_request(
             approved=True, mergeable=True)
         pull_request_id = pull_request.pull_request_id
         repo_name = pull_request.target_repo.scm_instance().name,
 
         url = route_path('pullrequest_merge',
                          repo_name=str(repo_name[0]),
                          pull_request_id=pull_request_id)
         response = self.app.post(url, params={'csrf_token': csrf_token}).follow()
 
         pull_request = PullRequest.get(pull_request_id)
 
         assert response.status_int == 200
         assert pull_request.is_closed()
         assert_pull_request_status(
             pull_request, ChangesetStatus.STATUS_APPROVED)
 
         # Check the relevant log entries were added
         user_logs = UserLog.query().order_by(UserLog.user_log_id.desc()).limit(3)
         actions = [log.action for log in user_logs]
         pr_commit_ids = PullRequestModel()._get_commit_ids(pull_request)
         expected_actions = [
              u'repo.pull_request.close',
              u'repo.pull_request.merge',
              u'repo.pull_request.comment.create'
         ]
         assert actions == expected_actions
 
         user_logs = UserLog.query().order_by(UserLog.user_log_id.desc()).limit(4)
         actions = [log for log in user_logs]
         assert actions[-1].action == 'user.push'
         assert actions[-1].action_data['commit_ids'] == pr_commit_ids
 
         # Check post_push rcextension was really executed
         push_calls = rhodecode.EXTENSIONS.calls['_push_hook']
         assert len(push_calls) == 1
         unused_last_call_args, last_call_kwargs = push_calls[0]
         assert last_call_kwargs['action'] == 'push'
         assert last_call_kwargs['commit_ids'] == pr_commit_ids
 
     def test_merge_pull_request_disabled(self, pr_util, csrf_token):
         pull_request = pr_util.create_pull_request(mergeable=False)
         pull_request_id = pull_request.pull_request_id
         pull_request = PullRequest.get(pull_request_id)
 
         response = self.app.post(
             route_path('pullrequest_merge',
                 repo_name=pull_request.target_repo.scm_instance().name,
                 pull_request_id=pull_request.pull_request_id),
             params={'csrf_token': csrf_token}).follow()
 
         assert response.status_int == 200
         response.mustcontain(
             'Merge is not currently possible because of below failed checks.')
         response.mustcontain('Server-side pull request merging is disabled.')
 
     @pytest.mark.skip_backends('svn')
     def test_merge_pull_request_not_approved(self, pr_util, csrf_token):
         pull_request = pr_util.create_pull_request(mergeable=True)
         pull_request_id = pull_request.pull_request_id
         repo_name = pull_request.target_repo.scm_instance().name
 
         response = self.app.post(
             route_path('pullrequest_merge',
                        repo_name=repo_name, pull_request_id=pull_request_id),
             params={'csrf_token': csrf_token}).follow()
 
         assert response.status_int == 200
 
         response.mustcontain(
             'Merge is not currently possible because of below failed checks.')
         response.mustcontain('Pull request reviewer approval is pending.')
 
     def test_merge_pull_request_renders_failure_reason(
             self, user_regular, csrf_token, pr_util):
         pull_request = pr_util.create_pull_request(mergeable=True, approved=True)
         pull_request_id = pull_request.pull_request_id
         repo_name = pull_request.target_repo.scm_instance().name
 
         merge_resp = MergeResponse(True, False, 'STUB_COMMIT_ID',
                                    MergeFailureReason.PUSH_FAILED,
                                    metadata={'target': 'shadow repo',
                                              'merge_commit': 'xxx'})
         model_patcher = mock.patch.multiple(
             PullRequestModel,
             merge_repo=mock.Mock(return_value=merge_resp),
             merge_status=mock.Mock(return_value=(None, True, 'WRONG_MESSAGE')))
 
         with model_patcher:
             response = self.app.post(
                 route_path('pullrequest_merge',
                            repo_name=repo_name,
                            pull_request_id=pull_request_id),
                 params={'csrf_token': csrf_token}, status=302)
 
             merge_resp = MergeResponse(True, True, '', MergeFailureReason.PUSH_FAILED,
                                        metadata={'target': 'shadow repo',
                                                  'merge_commit': 'xxx'})
         assert_session_flash(response, merge_resp.merge_status_message)
 
     def test_update_source_revision(self, backend, csrf_token):
         commits = [
             {'message': 'ancestor'},
             {'message': 'change'},
             {'message': 'change-2'},
         ]
         commit_ids = backend.create_master_repo(commits)
         target = backend.create_repo(heads=['ancestor'])
         source = backend.create_repo(heads=['change'])
 
         # create pr from a in source to A in target
         pull_request = PullRequest()
 
         pull_request.source_repo = source
         pull_request.source_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name, commit_id=commit_ids['change'])
 
         pull_request.target_repo = target
         pull_request.target_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name, commit_id=commit_ids['ancestor'])
 
         pull_request.revisions = [commit_ids['change']]
         pull_request.title = u"Test"
         pull_request.description = u"Description"
         pull_request.author = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN)
         pull_request.pull_request_state = PullRequest.STATE_CREATED
         Session().add(pull_request)
         Session().commit()
         pull_request_id = pull_request.pull_request_id
 
         # source has ancestor - change - change-2
         backend.pull_heads(source, heads=['change-2'])
         target_repo_name = target.repo_name
 
         # update PR
         self.app.post(
             route_path('pullrequest_update',
                        repo_name=target_repo_name, pull_request_id=pull_request_id),
             params={'update_commits': 'true', 'csrf_token': csrf_token})
 
         response = self.app.get(
             route_path('pullrequest_show',
                        repo_name=target_repo_name,
                        pull_request_id=pull_request.pull_request_id))
 
         assert response.status_int == 200
         response.mustcontain('Pull request updated to')
         response.mustcontain('with 1 added, 0 removed commits.')
 
         # check that we have now both revisions
         pull_request = PullRequest.get(pull_request_id)
         assert pull_request.revisions == [commit_ids['change-2'], commit_ids['change']]
 
     def test_update_target_revision(self, backend, csrf_token):
         commits = [
             {'message': 'ancestor'},
             {'message': 'change'},
             {'message': 'ancestor-new', 'parents': ['ancestor']},
             {'message': 'change-rebased'},
         ]
         commit_ids = backend.create_master_repo(commits)
         target = backend.create_repo(heads=['ancestor'])
         source = backend.create_repo(heads=['change'])
 
         # create pr from a in source to A in target
         pull_request = PullRequest()
 
         pull_request.source_repo = source
         pull_request.source_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name, commit_id=commit_ids['change'])
 
         pull_request.target_repo = target
         pull_request.target_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name, commit_id=commit_ids['ancestor'])
 
         pull_request.revisions = [commit_ids['change']]
         pull_request.title = u"Test"
         pull_request.description = u"Description"
         pull_request.author = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN)
         pull_request.pull_request_state = PullRequest.STATE_CREATED
 
         Session().add(pull_request)
         Session().commit()
         pull_request_id = pull_request.pull_request_id
 
         # target has ancestor - ancestor-new
         # source has ancestor - ancestor-new - change-rebased
         backend.pull_heads(target, heads=['ancestor-new'])
         backend.pull_heads(source, heads=['change-rebased'])
         target_repo_name = target.repo_name
 
         # update PR
         url = route_path('pullrequest_update',
                          repo_name=target_repo_name,
                          pull_request_id=pull_request_id)
         self.app.post(url,
                       params={'update_commits': 'true', 'csrf_token': csrf_token},
                       status=200)
 
         # check that we have now both revisions
         pull_request = PullRequest.get(pull_request_id)
         assert pull_request.revisions == [commit_ids['change-rebased']]
         assert pull_request.target_ref == 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name, commit_id=commit_ids['ancestor-new'])
 
         response = self.app.get(
             route_path('pullrequest_show',
                        repo_name=target_repo_name,
                        pull_request_id=pull_request.pull_request_id))
         assert response.status_int == 200
         response.mustcontain('Pull request updated to')
         response.mustcontain('with 1 added, 1 removed commits.')
 
     def test_update_target_revision_with_removal_of_1_commit_git(self, backend_git, csrf_token):
         backend = backend_git
         commits = [
             {'message': 'master-commit-1'},
             {'message': 'master-commit-2-change-1'},
             {'message': 'master-commit-3-change-2'},
 
             {'message': 'feat-commit-1', 'parents': ['master-commit-1']},
             {'message': 'feat-commit-2'},
         ]
         commit_ids = backend.create_master_repo(commits)
         target = backend.create_repo(heads=['master-commit-3-change-2'])
         source = backend.create_repo(heads=['feat-commit-2'])
 
         # create pr from a in source to A in target
         pull_request = PullRequest()
         pull_request.source_repo = source
 
         pull_request.source_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name,
             commit_id=commit_ids['master-commit-3-change-2'])
 
         pull_request.target_repo = target
         pull_request.target_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name, commit_id=commit_ids['feat-commit-2'])
 
         pull_request.revisions = [
             commit_ids['feat-commit-1'],
             commit_ids['feat-commit-2']
         ]
         pull_request.title = u"Test"
         pull_request.description = u"Description"
         pull_request.author = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN)
         pull_request.pull_request_state = PullRequest.STATE_CREATED
         Session().add(pull_request)
         Session().commit()
         pull_request_id = pull_request.pull_request_id
 
         # PR is created, now we simulate a force-push into target,
         # that drops a 2 last commits
         vcsrepo = target.scm_instance()
         vcsrepo.config.clear_section('hooks')
         vcsrepo.run_git_command(['reset', '--soft', 'HEAD~2'])
         target_repo_name = target.repo_name
 
         # update PR
         url = route_path('pullrequest_update',
                          repo_name=target_repo_name,
                          pull_request_id=pull_request_id)
         self.app.post(url,
                       params={'update_commits': 'true', 'csrf_token': csrf_token},
                       status=200)
 
         response = self.app.get(route_path('pullrequest_new', repo_name=target_repo_name))
         assert response.status_int == 200
         response.mustcontain('Pull request updated to')
         response.mustcontain('with 0 added, 0 removed commits.')
 
     def test_update_of_ancestor_reference(self, backend, csrf_token):
         commits = [
             {'message': 'ancestor'},
             {'message': 'change'},
             {'message': 'change-2'},
             {'message': 'ancestor-new', 'parents': ['ancestor']},
             {'message': 'change-rebased'},
         ]
         commit_ids = backend.create_master_repo(commits)
         target = backend.create_repo(heads=['ancestor'])
         source = backend.create_repo(heads=['change'])
 
         # create pr from a in source to A in target
         pull_request = PullRequest()
         pull_request.source_repo = source
 
         pull_request.source_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name, commit_id=commit_ids['change'])
         pull_request.target_repo = target
         pull_request.target_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name, commit_id=commit_ids['ancestor'])
         pull_request.revisions = [commit_ids['change']]
         pull_request.title = u"Test"
         pull_request.description = u"Description"
         pull_request.author = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN)
         pull_request.pull_request_state = PullRequest.STATE_CREATED
         Session().add(pull_request)
         Session().commit()
         pull_request_id = pull_request.pull_request_id
 
         # target has ancestor - ancestor-new
         # source has ancestor - ancestor-new - change-rebased
         backend.pull_heads(target, heads=['ancestor-new'])
         backend.pull_heads(source, heads=['change-rebased'])
         target_repo_name = target.repo_name
 
         # update PR
         self.app.post(
             route_path('pullrequest_update',
                        repo_name=target_repo_name, pull_request_id=pull_request_id),
             params={'update_commits': 'true', 'csrf_token': csrf_token},
             status=200)
 
         # Expect the target reference to be updated correctly
         pull_request = PullRequest.get(pull_request_id)
         assert pull_request.revisions == [commit_ids['change-rebased']]
         expected_target_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name,
             commit_id=commit_ids['ancestor-new'])
         assert pull_request.target_ref == expected_target_ref
 
     def test_remove_pull_request_branch(self, backend_git, csrf_token):
         branch_name = 'development'
         commits = [
             {'message': 'initial-commit'},
             {'message': 'old-feature'},
             {'message': 'new-feature', 'branch': branch_name},
         ]
         repo = backend_git.create_repo(commits)
         repo_name = repo.repo_name
         commit_ids = backend_git.commit_ids
 
         pull_request = PullRequest()
         pull_request.source_repo = repo
         pull_request.target_repo = repo
         pull_request.source_ref = 'branch:{branch}:{commit_id}'.format(
             branch=branch_name, commit_id=commit_ids['new-feature'])
         pull_request.target_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend_git.default_branch_name, commit_id=commit_ids['old-feature'])
         pull_request.revisions = [commit_ids['new-feature']]
         pull_request.title = u"Test"
         pull_request.description = u"Description"
         pull_request.author = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN)
         pull_request.pull_request_state = PullRequest.STATE_CREATED
         Session().add(pull_request)
         Session().commit()
 
         pull_request_id = pull_request.pull_request_id
 
         vcs = repo.scm_instance()
         vcs.remove_ref('refs/heads/{}'.format(branch_name))
         # NOTE(marcink): run GC to ensure the commits are gone
         vcs.run_gc()
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=repo_name,
             pull_request_id=pull_request_id))
 
         assert response.status_int == 200
 
         response.assert_response().element_contains(
             '#changeset_compare_view_content .alert strong',
             'Missing commits')
         response.assert_response().element_contains(
             '#changeset_compare_view_content .alert',
             'This pull request cannot be displayed, because one or more'
             ' commits no longer exist in the source repository.')
 
     def test_strip_commits_from_pull_request(
             self, backend, pr_util, csrf_token):
         commits = [
             {'message': 'initial-commit'},
             {'message': 'old-feature'},
             {'message': 'new-feature', 'parents': ['initial-commit']},
         ]
         pull_request = pr_util.create_pull_request(
             commits, target_head='initial-commit', source_head='new-feature',
             revisions=['new-feature'])
 
         vcs = pr_util.source_repository.scm_instance()
         if backend.alias == 'git':
             vcs.strip(pr_util.commit_ids['new-feature'], branch_name='master')
         else:
             vcs.strip(pr_util.commit_ids['new-feature'])
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pr_util.target_repository.repo_name,
             pull_request_id=pull_request.pull_request_id))
 
         assert response.status_int == 200
 
         response.assert_response().element_contains(
             '#changeset_compare_view_content .alert strong',
             'Missing commits')
         response.assert_response().element_contains(
             '#changeset_compare_view_content .alert',
             'This pull request cannot be displayed, because one or more'
             ' commits no longer exist in the source repository.')
         response.assert_response().element_contains(
             '#update_commits',
             'Update commits')
 
     def test_strip_commits_and_update(
             self, backend, pr_util, csrf_token):
         commits = [
             {'message': 'initial-commit'},
             {'message': 'old-feature'},
             {'message': 'new-feature', 'parents': ['old-feature']},
         ]
         pull_request = pr_util.create_pull_request(
             commits, target_head='old-feature', source_head='new-feature',
             revisions=['new-feature'], mergeable=True)
         pr_id = pull_request.pull_request_id
         target_repo_name = pull_request.target_repo.repo_name
 
         vcs = pr_util.source_repository.scm_instance()
         if backend.alias == 'git':
             vcs.strip(pr_util.commit_ids['new-feature'], branch_name='master')
         else:
             vcs.strip(pr_util.commit_ids['new-feature'])
 
         url = route_path('pullrequest_update',
                          repo_name=target_repo_name,
                          pull_request_id=pr_id)
         response = self.app.post(url,
                                  params={'update_commits': 'true',
                                          'csrf_token': csrf_token})
 
         assert response.status_int == 200
         assert response.body == '{"response": true, "redirect_url": null}'
 
         # Make sure that after update, it won't raise 500 errors
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=target_repo_name,
             pull_request_id=pr_id))
 
         assert response.status_int == 200
         response.assert_response().element_contains(
             '#changeset_compare_view_content .alert strong',
             'Missing commits')
 
     def test_branch_is_a_link(self, pr_util):
         pull_request = pr_util.create_pull_request()
         pull_request.source_ref = 'branch:origin:1234567890abcdef'
         pull_request.target_ref = 'branch:target:abcdef1234567890'
         Session().add(pull_request)
         Session().commit()
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
         assert response.status_int == 200
 
         source = response.assert_response().get_element('.pr-source-info')
         source_parent = source.getparent()
         assert len(source_parent) == 1
 
         target = response.assert_response().get_element('.pr-target-info')
         target_parent = target.getparent()
         assert len(target_parent) == 1
 
         expected_origin_link = route_path(
             'repo_commits',
             repo_name=pull_request.source_repo.scm_instance().name,
             params=dict(branch='origin'))
         expected_target_link = route_path(
             'repo_commits',
             repo_name=pull_request.target_repo.scm_instance().name,
             params=dict(branch='target'))
         assert source_parent.attrib['href'] == expected_origin_link
         assert target_parent.attrib['href'] == expected_target_link
 
     def test_bookmark_is_not_a_link(self, pr_util):
         pull_request = pr_util.create_pull_request()
         pull_request.source_ref = 'bookmark:origin:1234567890abcdef'
         pull_request.target_ref = 'bookmark:target:abcdef1234567890'
         Session().add(pull_request)
         Session().commit()
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
         assert response.status_int == 200
 
         source = response.assert_response().get_element('.pr-source-info')
         assert source.text.strip() == 'bookmark:origin'
         assert source.getparent().attrib.get('href') is None
 
         target = response.assert_response().get_element('.pr-target-info')
         assert target.text.strip() == 'bookmark:target'
         assert target.getparent().attrib.get('href') is None
 
     def test_tag_is_not_a_link(self, pr_util):
         pull_request = pr_util.create_pull_request()
         pull_request.source_ref = 'tag:origin:1234567890abcdef'
         pull_request.target_ref = 'tag:target:abcdef1234567890'
         Session().add(pull_request)
         Session().commit()
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
         assert response.status_int == 200
 
         source = response.assert_response().get_element('.pr-source-info')
         assert source.text.strip() == 'tag:origin'
         assert source.getparent().attrib.get('href') is None
 
         target = response.assert_response().get_element('.pr-target-info')
         assert target.text.strip() == 'tag:target'
         assert target.getparent().attrib.get('href') is None
 
     @pytest.mark.parametrize('mergeable', [True, False])
     def test_shadow_repository_link(
             self, mergeable, pr_util, http_host_only_stub):
         """
         Check that the pull request summary page displays a link to the shadow
         repository if the pull request is mergeable. If it is not mergeable
         the link should not be displayed.
         """
         pull_request = pr_util.create_pull_request(
             mergeable=mergeable, enable_notifications=False)
         target_repo = pull_request.target_repo.scm_instance()
         pr_id = pull_request.pull_request_id
         shadow_url = '{host}/{repo}/pull-request/{pr_id}/repository'.format(
             host=http_host_only_stub, repo=target_repo.name, pr_id=pr_id)
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=target_repo.name,
             pull_request_id=pr_id))
 
         if mergeable:
             response.assert_response().element_value_contains(
                 'input.pr-mergeinfo', shadow_url)
             response.assert_response().element_value_contains(
                 'input.pr-mergeinfo ', 'pr-merge')
         else:
             response.assert_response().no_element_exists('.pr-mergeinfo')
 
 
 @pytest.mark.usefixtures('app')
 @pytest.mark.backends("git", "hg")
 class TestPullrequestsControllerDelete(object):
     def test_pull_request_delete_button_permissions_admin(
             self, autologin_user, user_admin, pr_util):
         pull_request = pr_util.create_pull_request(
             author=user_admin.username, enable_notifications=False)
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
 
         response.mustcontain('id="delete_pullrequest"')
         response.mustcontain('Confirm to delete this pull request')
 
     def test_pull_request_delete_button_permissions_owner(
             self, autologin_regular_user, user_regular, pr_util):
         pull_request = pr_util.create_pull_request(
             author=user_regular.username, enable_notifications=False)
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
 
         response.mustcontain('id="delete_pullrequest"')
         response.mustcontain('Confirm to delete this pull request')
 
     def test_pull_request_delete_button_permissions_forbidden(
             self, autologin_regular_user, user_regular, user_admin, pr_util):
         pull_request = pr_util.create_pull_request(
             author=user_admin.username, enable_notifications=False)
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
         response.mustcontain(no=['id="delete_pullrequest"'])
         response.mustcontain(no=['Confirm to delete this pull request'])
 
     def test_pull_request_delete_button_permissions_can_update_cannot_delete(
             self, autologin_regular_user, user_regular, user_admin, pr_util,
             user_util):
 
         pull_request = pr_util.create_pull_request(
             author=user_admin.username, enable_notifications=False)
 
         user_util.grant_user_permission_to_repo(
             pull_request.target_repo, user_regular,
             'repository.write')
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
 
         response.mustcontain('id="open_edit_pullrequest"')
         response.mustcontain('id="delete_pullrequest"')
         response.mustcontain(no=['Confirm to delete this pull request'])
 
     def test_delete_comment_returns_404_if_comment_does_not_exist(
             self, autologin_user, pr_util, user_admin, csrf_token, xhr_header):
 
         pull_request = pr_util.create_pull_request(
             author=user_admin.username, enable_notifications=False)
 
         self.app.post(
             route_path(
             'pullrequest_comment_delete',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id,
             comment_id=1024404),
             extra_environ=xhr_header,
             params={'csrf_token': csrf_token},
             status=404
         )
 
     def test_delete_comment(
             self, autologin_user, pr_util, user_admin, csrf_token, xhr_header):
 
         pull_request = pr_util.create_pull_request(
             author=user_admin.username, enable_notifications=False)
         comment = pr_util.create_comment()
         comment_id = comment.comment_id
 
         response = self.app.post(
             route_path(
             'pullrequest_comment_delete',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id,
             comment_id=comment_id),
             extra_environ=xhr_header,
             params={'csrf_token': csrf_token},
             status=200
         )
         assert response.body == 'true'
 
     @pytest.mark.parametrize('url_type', [
         'pullrequest_new',
         'pullrequest_create',
         'pullrequest_update',
         'pullrequest_merge',
     ])
     def test_pull_request_is_forbidden_on_archived_repo(
             self, autologin_user, backend, xhr_header, user_util, url_type):
 
         # create a temporary repo
         source = user_util.create_repo(repo_type=backend.alias)
         repo_name = source.repo_name
         repo = Repository.get_by_repo_name(repo_name)
         repo.archived = True
         Session().commit()
 
         response = self.app.get(
             route_path(url_type, repo_name=repo_name, pull_request_id=1), status=302)
 
         msg = 'Action not supported for archived repository.'
         assert_session_flash(response, msg)
 
 
 def assert_pull_request_status(pull_request, expected_status):
     status = ChangesetStatusModel().calculated_review_status(pull_request=pull_request)
     assert status == expected_status
 
 
 @pytest.mark.parametrize('route', ['pullrequest_new', 'pullrequest_create'])
 @pytest.mark.usefixtures("autologin_user")
 def test_forbidde_to_repo_summary_for_svn_repositories(backend_svn, app, route):
     app.get(route_path(route, repo_name=backend_svn.repo_name), status=404)

rhodecode/apps/repository/utils.py

@@ -1,111 +1,113 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 from rhodecode.lib import helpers as h, rc_cache
 from rhodecode.lib.utils2 import safe_int
 from rhodecode.model.pull_request import get_diff_info
 from rhodecode.model.db import PullRequestReviewers
 # V3 - Reviewers, with default rules data
 # v4 - Added observers metadata
-REVIEWER_API_VERSION = 'V4'
+# v5 - pr_author/commit_author include/exclude logic
+REVIEWER_API_VERSION = 'V5'
 
 
 def reviewer_as_json(user, reasons=None, role=None, mandatory=False, rules=None, user_group=None):
     """
     Returns json struct of a reviewer for frontend
 
     :param user: the reviewer
     :param reasons: list of strings of why they are reviewers
     :param mandatory: bool, to set user as mandatory
     """
     role = role or PullRequestReviewers.ROLE_REVIEWER
     if role not in PullRequestReviewers.ROLES:
         raise ValueError('role is not one of %s', PullRequestReviewers.ROLES)
 
     return {
         'user_id': user.user_id,
         'reasons': reasons or [],
         'rules': rules or [],
         'role': role,
         'mandatory': mandatory,
         'user_group': user_group,
         'username': user.username,
         'first_name': user.first_name,
         'last_name': user.last_name,
         'user_link': h.link_to_user(user),
         'gravatar_link': h.gravatar_url(user.email, 14),
     }
 
 
 def to_reviewers(e):
     if isinstance(e, (tuple, list)):
         return map(reviewer_as_json, e)
     else:
         return reviewer_as_json(e)
 
 
 def get_default_reviewers_data(current_user, source_repo, source_ref, target_repo, target_ref,
                                include_diff_info=True):
     """
     Return json for default reviewers of a repository
     """
 
     diff_info = {}
     if include_diff_info:
         diff_info = get_diff_info(
             source_repo, source_ref.commit_id, target_repo, target_ref.commit_id)
 
     reasons = ['Default reviewer', 'Repository owner']
     json_reviewers = [reviewer_as_json(
         user=target_repo.user, reasons=reasons, mandatory=False, rules=None, role=None)]
 
     compute_key = rc_cache.utils.compute_key_from_params(
         current_user.user_id, source_repo.repo_id, source_ref.type, source_ref.name,
         source_ref.commit_id, target_repo.repo_id, target_ref.type, target_ref.name,
         target_ref.commit_id)
 
     return {
         'api_ver': REVIEWER_API_VERSION,  # define version for later possible schema upgrade
         'compute_key': compute_key,
         'diff_info': diff_info,
         'reviewers': json_reviewers,
         'rules': {},
         'rules_data': {},
+        'rules_humanized': [],
     }
 
 
 def validate_default_reviewers(review_members, reviewer_rules):
     """
     Function to validate submitted reviewers against the saved rules
     """
     reviewers = []
     reviewer_by_id = {}
     for r in review_members:
         reviewer_user_id = safe_int(r['user_id'])
         entry = (reviewer_user_id, r['reasons'], r['mandatory'], r['role'], r['rules'])
 
         reviewer_by_id[reviewer_user_id] = entry
         reviewers.append(entry)
 
     return reviewers
 
 
 def validate_observers(observer_members, reviewer_rules):
     return {}

rhodecode/apps/repository/views/repo_commits.py

@@ -1,791 +1,852 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import collections
 
 from pyramid.httpexceptions import (
     HTTPNotFound, HTTPBadRequest, HTTPFound, HTTPForbidden, HTTPConflict)
 from pyramid.view import view_config
 from pyramid.renderers import render
 from pyramid.response import Response
 
 from rhodecode.apps._base import RepoAppView
 from rhodecode.apps.file_store import utils as store_utils
 from rhodecode.apps.file_store.exceptions import FileNotAllowedException, FileOverSizeException
 
 from rhodecode.lib import diffs, codeblocks, channelstream
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAnyDecorator, NotAnonymous, CSRFRequired)
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.compat import OrderedDict
 from rhodecode.lib.diffs import (
     cache_diff, load_cached_diff, diff_cache_exist, get_diff_context,
     get_diff_whitespace_flag)
 from rhodecode.lib.exceptions import StatusChangeOnClosedPullRequestError, CommentVersionMismatch
 import rhodecode.lib.helpers as h
 from rhodecode.lib.utils2 import safe_unicode, str2bool, StrictAttributeDict
 from rhodecode.lib.vcs.backends.base import EmptyCommit
 from rhodecode.lib.vcs.exceptions import (
     RepositoryError, CommitDoesNotExistError)
 from rhodecode.model.db import ChangesetComment, ChangesetStatus, FileStore, \
     ChangesetCommentHistory
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.meta import Session
 from rhodecode.model.settings import VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
 
 def _update_with_GET(params, request):
     for k in ['diff1', 'diff2', 'diff']:
         params[k] += request.GET.getall(k)
 
 
 class RepoCommitsView(RepoAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context(include_app_defaults=True)
         c.rhodecode_repo = self.rhodecode_vcs_repo
 
         return c
 
     def _is_diff_cache_enabled(self, target_repo):
         caching_enabled = self._get_general_setting(
             target_repo, 'rhodecode_diff_cache')
         log.debug('Diff caching enabled: %s', caching_enabled)
         return caching_enabled
 
     def _commit(self, commit_id_range, method):
         _ = self.request.translate
         c = self.load_default_context()
         c.fulldiff = self.request.GET.get('fulldiff')
+        redirect_to_combined = str2bool(self.request.GET.get('redirect_combined'))
 
         # fetch global flags of ignore ws or context lines
         diff_context = get_diff_context(self.request)
         hide_whitespace_changes = get_diff_whitespace_flag(self.request)
 
         # diff_limit will cut off the whole diff if the limit is applied
         # otherwise it will just hide the big files from the front-end
         diff_limit = c.visual.cut_off_limit_diff
         file_limit = c.visual.cut_off_limit_file
 
         # get ranges of commit ids if preset
         commit_range = commit_id_range.split('...')[:2]
 
         try:
             pre_load = ['affected_files', 'author', 'branch', 'date',
                         'message', 'parents']
             if self.rhodecode_vcs_repo.alias == 'hg':
                 pre_load += ['hidden', 'obsolete', 'phase']
 
             if len(commit_range) == 2:
                 commits = self.rhodecode_vcs_repo.get_commits(
                     start_id=commit_range[0], end_id=commit_range[1],
                     pre_load=pre_load, translate_tags=False)
                 commits = list(commits)
             else:
                 commits = [self.rhodecode_vcs_repo.get_commit(
                     commit_id=commit_id_range, pre_load=pre_load)]
 
             c.commit_ranges = commits
             if not c.commit_ranges:
                 raise RepositoryError('The commit range returned an empty result')
         except CommitDoesNotExistError as e:
             msg = _('No such commit exists. Org exception: `{}`').format(e)
             h.flash(msg, category='error')
             raise HTTPNotFound()
         except Exception:
             log.exception("General failure")
             raise HTTPNotFound()
         single_commit = len(c.commit_ranges) == 1
 
+        if redirect_to_combined and not single_commit:
+            source_ref = getattr(c.commit_ranges[0].parents[0]
+                                 if c.commit_ranges[0].parents else h.EmptyCommit(), 'raw_id')
+            target_ref = c.commit_ranges[-1].raw_id
+            next_url = h.route_path(
+                'repo_compare',
+                repo_name=c.repo_name,
+                source_ref_type='rev',
+                source_ref=source_ref,
+                target_ref_type='rev',
+                target_ref=target_ref)
+            raise HTTPFound(next_url)
+
         c.changes = OrderedDict()
         c.lines_added = 0
         c.lines_deleted = 0
 
         # auto collapse if we have more than limit
         collapse_limit = diffs.DiffProcessor._collapse_commits_over
         c.collapse_all_commits = len(c.commit_ranges) > collapse_limit
 
         c.commit_statuses = ChangesetStatus.STATUSES
         c.inline_comments = []
         c.files = []
 
         c.comments = []
         c.unresolved_comments = []
         c.resolved_comments = []
 
         # Single commit
         if single_commit:
             commit = c.commit_ranges[0]
             c.comments = CommentsModel().get_comments(
                 self.db_repo.repo_id,
                 revision=commit.raw_id)
 
             # comments from PR
             statuses = ChangesetStatusModel().get_statuses(
                 self.db_repo.repo_id, commit.raw_id,
                 with_revisions=True)
 
             prs = set()
             reviewers = list()
             reviewers_duplicates = set()  # to not have duplicates from multiple votes
             for c_status in statuses:
 
                 # extract associated pull-requests from votes
                 if c_status.pull_request:
                     prs.add(c_status.pull_request)
 
                 # extract reviewers
                 _user_id = c_status.author.user_id
                 if _user_id not in reviewers_duplicates:
                     reviewers.append(
                         StrictAttributeDict({
                             'user': c_status.author,
 
                             # fake attributed for commit, page that we don't have
                             # but we share the display with PR page
                             'mandatory': False,
                             'reasons': [],
                             'rule_user_group_data': lambda: None
                         })
                     )
                     reviewers_duplicates.add(_user_id)
 
             c.reviewers_count = len(reviewers)
             c.observers_count = 0
 
             # from associated statuses, check the pull requests, and
             # show comments from them
             for pr in prs:
                 c.comments.extend(pr.comments)
 
             c.unresolved_comments = CommentsModel()\
                 .get_commit_unresolved_todos(commit.raw_id)
             c.resolved_comments = CommentsModel()\
                 .get_commit_resolved_todos(commit.raw_id)
 
             c.inline_comments_flat = CommentsModel()\
                 .get_commit_inline_comments(commit.raw_id)
 
             review_statuses = ChangesetStatusModel().aggregate_votes_by_user(
                 statuses, reviewers)
 
             c.commit_review_status = ChangesetStatus.STATUS_NOT_REVIEWED
 
             c.commit_set_reviewers_data_json = collections.OrderedDict({'reviewers': []})
 
             for review_obj, member, reasons, mandatory, status in review_statuses:
                 member_reviewer = h.reviewer_as_json(
                     member, reasons=reasons, mandatory=mandatory, role=None,
                     user_group=None
                 )
 
                 current_review_status = status[0][1].status if status else ChangesetStatus.STATUS_NOT_REVIEWED
                 member_reviewer['review_status'] = current_review_status
                 member_reviewer['review_status_label'] = h.commit_status_lbl(current_review_status)
                 member_reviewer['allowed_to_update'] = False
                 c.commit_set_reviewers_data_json['reviewers'].append(member_reviewer)
 
             c.commit_set_reviewers_data_json = json.dumps(c.commit_set_reviewers_data_json)
 
             # NOTE(marcink): this uses the same voting logic as in pull-requests
             c.commit_review_status = ChangesetStatusModel().calculate_status(review_statuses)
             c.commit_broadcast_channel = channelstream.comment_channel(c.repo_name, commit_obj=commit)
 
         diff = None
         # Iterate over ranges (default commit view is always one commit)
         for commit in c.commit_ranges:
             c.changes[commit.raw_id] = []
 
             commit2 = commit
             commit1 = commit.first_parent
 
             if method == 'show':
                 inline_comments = CommentsModel().get_inline_comments(
                     self.db_repo.repo_id, revision=commit.raw_id)
                 c.inline_cnt = len(CommentsModel().get_inline_comments_as_list(
                     inline_comments))
                 c.inline_comments = inline_comments
 
                 cache_path = self.rhodecode_vcs_repo.get_create_shadow_cache_pr_path(
                     self.db_repo)
                 cache_file_path = diff_cache_exist(
                     cache_path, 'diff', commit.raw_id,
                     hide_whitespace_changes, diff_context, c.fulldiff)
 
                 caching_enabled = self._is_diff_cache_enabled(self.db_repo)
                 force_recache = str2bool(self.request.GET.get('force_recache'))
 
                 cached_diff = None
                 if caching_enabled:
                     cached_diff = load_cached_diff(cache_file_path)
 
                 has_proper_diff_cache = cached_diff and cached_diff.get('diff')
                 if not force_recache and has_proper_diff_cache:
                     diffset = cached_diff['diff']
                 else:
                     vcs_diff = self.rhodecode_vcs_repo.get_diff(
                         commit1, commit2,
                         ignore_whitespace=hide_whitespace_changes,
                         context=diff_context)
 
                     diff_processor = diffs.DiffProcessor(
                         vcs_diff, format='newdiff', diff_limit=diff_limit,
                         file_limit=file_limit, show_full_diff=c.fulldiff)
 
                     _parsed = diff_processor.prepare()
 
                     diffset = codeblocks.DiffSet(
                         repo_name=self.db_repo_name,
                         source_node_getter=codeblocks.diffset_node_getter(commit1),
                         target_node_getter=codeblocks.diffset_node_getter(commit2))
 
                     diffset = self.path_filter.render_patchset_filtered(
                         diffset, _parsed, commit1.raw_id, commit2.raw_id)
 
                     # save cached diff
                     if caching_enabled:
                         cache_diff(cache_file_path, diffset, None)
 
                 c.limited_diff = diffset.limited_diff
                 c.changes[commit.raw_id] = diffset
             else:
                 # TODO(marcink): no cache usage here...
                 _diff = self.rhodecode_vcs_repo.get_diff(
                     commit1, commit2,
                     ignore_whitespace=hide_whitespace_changes, context=diff_context)
                 diff_processor = diffs.DiffProcessor(
                     _diff, format='newdiff', diff_limit=diff_limit,
                     file_limit=file_limit, show_full_diff=c.fulldiff)
                 # downloads/raw we only need RAW diff nothing else
                 diff = self.path_filter.get_raw_patch(diff_processor)
                 c.changes[commit.raw_id] = [None, None, None, None, diff, None, None]
 
         # sort comments by how they were generated
         c.comments = sorted(c.comments, key=lambda x: x.comment_id)
         c.at_version_num = None
 
         if len(c.commit_ranges) == 1:
             c.commit = c.commit_ranges[0]
             c.parent_tmpl = ''.join(
                 '# Parent  %s\n' % x.raw_id for x in c.commit.parents)
 
         if method == 'download':
             response = Response(diff)
             response.content_type = 'text/plain'
             response.content_disposition = (
                 'attachment; filename=%s.diff' % commit_id_range[:12])
             return response
         elif method == 'patch':
             c.diff = safe_unicode(diff)
             patch = render(
                 'rhodecode:templates/changeset/patch_changeset.mako',
                 self._get_template_context(c), self.request)
             response = Response(patch)
             response.content_type = 'text/plain'
             return response
         elif method == 'raw':
             response = Response(diff)
             response.content_type = 'text/plain'
             return response
         elif method == 'show':
             if len(c.commit_ranges) == 1:
                 html = render(
                     'rhodecode:templates/changeset/changeset.mako',
                     self._get_template_context(c), self.request)
                 return Response(html)
             else:
                 c.ancestor = None
                 c.target_repo = self.db_repo
                 html = render(
                     'rhodecode:templates/changeset/changeset_range.mako',
                     self._get_template_context(c), self.request)
                 return Response(html)
 
         raise HTTPBadRequest()
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_commit', request_method='GET',
         renderer=None)
     def repo_commit_show(self):
         commit_id = self.request.matchdict['commit_id']
         return self._commit(commit_id, method='show')
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_commit_raw', request_method='GET',
         renderer=None)
     @view_config(
         route_name='repo_commit_raw_deprecated', request_method='GET',
         renderer=None)
     def repo_commit_raw(self):
         commit_id = self.request.matchdict['commit_id']
         return self._commit(commit_id, method='raw')
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_commit_patch', request_method='GET',
         renderer=None)
     def repo_commit_patch(self):
         commit_id = self.request.matchdict['commit_id']
         return self._commit(commit_id, method='patch')
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_commit_download', request_method='GET',
         renderer=None)
     def repo_commit_download(self):
         commit_id = self.request.matchdict['commit_id']
         return self._commit(commit_id, method='download')
 
+    def _commit_comments_create(self, commit_id, comments):
+        _ = self.request.translate
+        data = {}
+        if not comments:
+            return
+
+        commit = self.db_repo.get_commit(commit_id)
+
+        all_drafts = len([x for x in comments if str2bool(x['is_draft'])]) == len(comments)
+        for entry in comments:
+            c = self.load_default_context()
+            comment_type = entry['comment_type']
+            text = entry['text']
+            status = entry['status']
+            is_draft = str2bool(entry['is_draft'])
+            resolves_comment_id = entry['resolves_comment_id']
+            f_path = entry['f_path']
+            line_no = entry['line']
+            target_elem_id = 'file-{}'.format(h.safeid(h.safe_unicode(f_path)))
+
+            if status:
+                text = text or (_('Status change %(transition_icon)s %(status)s')
+                                % {'transition_icon': '>',
+                                   'status': ChangesetStatus.get_status_lbl(status)})
+
+            comment = CommentsModel().create(
+                text=text,
+                repo=self.db_repo.repo_id,
+                user=self._rhodecode_db_user.user_id,
+                commit_id=commit_id,
+                f_path=f_path,
+                line_no=line_no,
+                status_change=(ChangesetStatus.get_status_lbl(status)
+                               if status else None),
+                status_change_type=status,
+                comment_type=comment_type,
+                is_draft=is_draft,
+                resolves_comment_id=resolves_comment_id,
+                auth_user=self._rhodecode_user,
+                send_email=not is_draft,  # skip notification for draft comments
+            )
+            is_inline = comment.is_inline
+
+            # get status if set !
+            if status:
+                # `dont_allow_on_closed_pull_request = True` means
+                # if latest status was from pull request and it's closed
+                # disallow changing status !
+
+                try:
+                    ChangesetStatusModel().set_status(
+                        self.db_repo.repo_id,
+                        status,
+                        self._rhodecode_db_user.user_id,
+                        comment,
+                        revision=commit_id,
+                        dont_allow_on_closed_pull_request=True
+                    )
+                except StatusChangeOnClosedPullRequestError:
+                    msg = _('Changing the status of a commit associated with '
+                            'a closed pull request is not allowed')
+                    log.exception(msg)
+                    h.flash(msg, category='warning')
+                    raise HTTPFound(h.route_path(
+                        'repo_commit', repo_name=self.db_repo_name,
+                        commit_id=commit_id))
+
+            Session().flush()
+            # this is somehow required to get access to some relationship
+            # loaded on comment
+            Session().refresh(comment)
+
+            # skip notifications for drafts
+            if not is_draft:
+                CommentsModel().trigger_commit_comment_hook(
+                    self.db_repo, self._rhodecode_user, 'create',
+                    data={'comment': comment, 'commit': commit})
+
+            comment_id = comment.comment_id
+            data[comment_id] = {
+                'target_id': target_elem_id
+            }
+            Session().flush()
+
+            c.co = comment
+            c.at_version_num = 0
+            c.is_new = True
+            rendered_comment = render(
+                'rhodecode:templates/changeset/changeset_comment_block.mako',
+                self._get_template_context(c), self.request)
+
+            data[comment_id].update(comment.get_dict())
+            data[comment_id].update({'rendered_text': rendered_comment})
+
+        # finalize, commit and redirect
+        Session().commit()
+
+        # skip channelstream for draft comments
+        if not all_drafts:
+            comment_broadcast_channel = channelstream.comment_channel(
+                self.db_repo_name, commit_obj=commit)
+
+            comment_data = data
+            posted_comment_type = 'inline' if is_inline else 'general'
+            if len(data) == 1:
+                msg = _('posted {} new {} comment').format(len(data), posted_comment_type)
+            else:
+                msg = _('posted {} new {} comments').format(len(data), posted_comment_type)
+
+            channelstream.comment_channelstream_push(
+                self.request, comment_broadcast_channel, self._rhodecode_user, msg,
+                comment_data=comment_data)
+
+        return data
+
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='repo_commit_comment_create', request_method='POST',
         renderer='json_ext')
     def repo_commit_comment_create(self):
         _ = self.request.translate
         commit_id = self.request.matchdict['commit_id']
 
-        c = self.load_default_context()
-        status = self.request.POST.get('changeset_status', None)
-        text = self.request.POST.get('text')
-        comment_type = self.request.POST.get('comment_type')
-        resolves_comment_id = self.request.POST.get('resolves_comment_id', None)
-
-        if status:
-            text = text or (_('Status change %(transition_icon)s %(status)s')
-                            % {'transition_icon': '>',
-                               'status': ChangesetStatus.get_status_lbl(status)})
-
         multi_commit_ids = []
         for _commit_id in self.request.POST.get('commit_ids', '').split(','):
             if _commit_id not in ['', None, EmptyCommit.raw_id]:
                 if _commit_id not in multi_commit_ids:
                     multi_commit_ids.append(_commit_id)
 
         commit_ids = multi_commit_ids or [commit_id]
 
-        comment = None
+        data = []
+        # Multiple comments for each passed commit id
         for current_id in filter(None, commit_ids):
-            comment = CommentsModel().create(
-                text=text,
-                repo=self.db_repo.repo_id,
-                user=self._rhodecode_db_user.user_id,
-                commit_id=current_id,
-                f_path=self.request.POST.get('f_path'),
-                line_no=self.request.POST.get('line'),
-                status_change=(ChangesetStatus.get_status_lbl(status)
-                               if status else None),
-                status_change_type=status,
-                comment_type=comment_type,
-                resolves_comment_id=resolves_comment_id,
-                auth_user=self._rhodecode_user
-            )
-            is_inline = comment.is_inline
-
-            # get status if set !
-            if status:
-                # if latest status was from pull request and it's closed
-                # disallow changing status !
-                # dont_allow_on_closed_pull_request = True !
+            comment_data = {
+                'comment_type': self.request.POST.get('comment_type'),
+                'text': self.request.POST.get('text'),
+                'status': self.request.POST.get('changeset_status', None),
+                'is_draft': self.request.POST.get('draft'),
+                'resolves_comment_id': self.request.POST.get('resolves_comment_id', None),
+                'close_pull_request': self.request.POST.get('close_pull_request'),
+                'f_path': self.request.POST.get('f_path'),
+                'line': self.request.POST.get('line'),
+            }
+            comment = self._commit_comments_create(commit_id=current_id, comments=[comment_data])
+            data.append(comment)
 
-                try:
-                    ChangesetStatusModel().set_status(
-                        self.db_repo.repo_id,
-                        status,
-                        self._rhodecode_db_user.user_id,
-                        comment,
-                        revision=current_id,
-                        dont_allow_on_closed_pull_request=True
-                    )
-                except StatusChangeOnClosedPullRequestError:
-                    msg = _('Changing the status of a commit associated with '
-                            'a closed pull request is not allowed')
-                    log.exception(msg)
-                    h.flash(msg, category='warning')
-                    raise HTTPFound(h.route_path(
-                        'repo_commit', repo_name=self.db_repo_name,
-                        commit_id=current_id))
-
-            commit = self.db_repo.get_commit(current_id)
-            CommentsModel().trigger_commit_comment_hook(
-                self.db_repo, self._rhodecode_user, 'create',
-                data={'comment': comment, 'commit': commit})
-
-        # finalize, commit and redirect
-        Session().commit()
-
-        data = {
-            'target_id': h.safeid(h.safe_unicode(
-                self.request.POST.get('f_path'))),
-        }
-        if comment:
-            c.co = comment
-            c.at_version_num = 0
-            rendered_comment = render(
-                'rhodecode:templates/changeset/changeset_comment_block.mako',
-                self._get_template_context(c), self.request)
-
-            data.update(comment.get_dict())
-            data.update({'rendered_text': rendered_comment})
-
-            comment_broadcast_channel = channelstream.comment_channel(
-                self.db_repo_name, commit_obj=commit)
-
-            comment_data = data
-            comment_type = 'inline' if is_inline else 'general'
-            channelstream.comment_channelstream_push(
-                self.request, comment_broadcast_channel, self._rhodecode_user,
-                _('posted a new {} comment').format(comment_type),
-                comment_data=comment_data)
-
-        return data
+        return data if len(data) > 1 else data[0]
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='repo_commit_comment_preview', request_method='POST',
         renderer='string', xhr=True)
     def repo_commit_comment_preview(self):
         # Technically a CSRF token is not needed as no state changes with this
         # call. However, as this is a POST is better to have it, so automated
         # tools don't flag it as potential CSRF.
         # Post is required because the payload could be bigger than the maximum
         # allowed by GET.
 
         text = self.request.POST.get('text')
         renderer = self.request.POST.get('renderer') or 'rst'
         if text:
             return h.render(text, renderer=renderer, mentions=True,
                             repo_name=self.db_repo_name)
         return ''
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='repo_commit_comment_history_view', request_method='POST',
         renderer='string', xhr=True)
     def repo_commit_comment_history_view(self):
         c = self.load_default_context()
 
         comment_history_id = self.request.matchdict['comment_history_id']
         comment_history = ChangesetCommentHistory.get_or_404(comment_history_id)
         is_repo_comment = comment_history.comment.repo.repo_id == self.db_repo.repo_id
 
         if is_repo_comment:
             c.comment_history = comment_history
 
             rendered_comment = render(
                 'rhodecode:templates/changeset/comment_history.mako',
                 self._get_template_context(c)
                 , self.request)
             return rendered_comment
         else:
             log.warning('No permissions for user %s to show comment_history_id: %s',
                         self._rhodecode_db_user, comment_history_id)
             raise HTTPNotFound()
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='repo_commit_comment_attachment_upload', request_method='POST',
         renderer='json_ext', xhr=True)
     def repo_commit_comment_attachment_upload(self):
         c = self.load_default_context()
         upload_key = 'attachment'
 
         file_obj = self.request.POST.get(upload_key)
 
         if file_obj is None:
             self.request.response.status = 400
             return {'store_fid': None,
                     'access_path': None,
                     'error': '{} data field is missing'.format(upload_key)}
 
         if not hasattr(file_obj, 'filename'):
             self.request.response.status = 400
             return {'store_fid': None,
                     'access_path': None,
                     'error': 'filename cannot be read from the data field'}
 
         filename = file_obj.filename
         file_display_name = filename
 
         metadata = {
             'user_uploaded': {'username': self._rhodecode_user.username,
                               'user_id': self._rhodecode_user.user_id,
                               'ip': self._rhodecode_user.ip_addr}}
 
         # TODO(marcink): allow .ini configuration for allowed_extensions, and file-size
         allowed_extensions = [
             'gif', '.jpeg', '.jpg', '.png', '.docx', '.gz', '.log', '.pdf',
             '.pptx', '.txt', '.xlsx', '.zip']
         max_file_size = 10 * 1024 * 1024  # 10MB, also validated via dropzone.js
 
         try:
             storage = store_utils.get_file_storage(self.request.registry.settings)
             store_uid, metadata = storage.save_file(
                 file_obj.file, filename, extra_metadata=metadata,
                 extensions=allowed_extensions, max_filesize=max_file_size)
         except FileNotAllowedException:
             self.request.response.status = 400
             permitted_extensions = ', '.join(allowed_extensions)
             error_msg = 'File `{}` is not allowed. ' \
                         'Only following extensions are permitted: {}'.format(
                             filename, permitted_extensions)
             return {'store_fid': None,
                     'access_path': None,
                     'error': error_msg}
         except FileOverSizeException:
             self.request.response.status = 400
             limit_mb = h.format_byte_size_binary(max_file_size)
             return {'store_fid': None,
                     'access_path': None,
                     'error': 'File {} is exceeding allowed limit of {}.'.format(
                         filename, limit_mb)}
 
         try:
             entry = FileStore.create(
                 file_uid=store_uid, filename=metadata["filename"],
                 file_hash=metadata["sha256"], file_size=metadata["size"],
                 file_display_name=file_display_name,
                 file_description=u'comment attachment `{}`'.format(safe_unicode(filename)),
                 hidden=True, check_acl=True, user_id=self._rhodecode_user.user_id,
                 scope_repo_id=self.db_repo.repo_id
             )
             Session().add(entry)
             Session().commit()
             log.debug('Stored upload in DB as %s', entry)
         except Exception:
             log.exception('Failed to store file %s', filename)
             self.request.response.status = 400
             return {'store_fid': None,
                     'access_path': None,
                     'error': 'File {} failed to store in DB.'.format(filename)}
 
         Session().commit()
 
         return {
             'store_fid': store_uid,
             'access_path': h.route_path(
                 'download_file', fid=store_uid),
             'fqn_access_path': h.route_url(
                 'download_file', fid=store_uid),
             'repo_access_path': h.route_path(
                 'repo_artifacts_get', repo_name=self.db_repo_name, uid=store_uid),
             'repo_fqn_access_path': h.route_url(
                 'repo_artifacts_get', repo_name=self.db_repo_name, uid=store_uid),
         }
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='repo_commit_comment_delete', request_method='POST',
         renderer='json_ext')
     def repo_commit_comment_delete(self):
         commit_id = self.request.matchdict['commit_id']
         comment_id = self.request.matchdict['comment_id']
 
         comment = ChangesetComment.get_or_404(comment_id)
         if not comment:
             log.debug('Comment with id:%s not found, skipping', comment_id)
             # comment already deleted in another call probably
             return True
 
         if comment.immutable:
             # don't allow deleting comments that are immutable
             raise HTTPForbidden()
 
         is_repo_admin = h.HasRepoPermissionAny('repository.admin')(self.db_repo_name)
         super_admin = h.HasPermissionAny('hg.admin')()
         comment_owner = (comment.author.user_id == self._rhodecode_db_user.user_id)
         is_repo_comment = comment.repo.repo_id == self.db_repo.repo_id
         comment_repo_admin = is_repo_admin and is_repo_comment
 
         if super_admin or comment_owner or comment_repo_admin:
             CommentsModel().delete(comment=comment, auth_user=self._rhodecode_user)
             Session().commit()
             return True
         else:
             log.warning('No permissions for user %s to delete comment_id: %s',
                         self._rhodecode_db_user, comment_id)
             raise HTTPNotFound()
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='repo_commit_comment_edit', request_method='POST',
         renderer='json_ext')
     def repo_commit_comment_edit(self):
         self.load_default_context()
 
+        commit_id = self.request.matchdict['commit_id']
         comment_id = self.request.matchdict['comment_id']
         comment = ChangesetComment.get_or_404(comment_id)
 
         if comment.immutable:
             # don't allow deleting comments that are immutable
             raise HTTPForbidden()
 
         is_repo_admin = h.HasRepoPermissionAny('repository.admin')(self.db_repo_name)
         super_admin = h.HasPermissionAny('hg.admin')()
         comment_owner = (comment.author.user_id == self._rhodecode_db_user.user_id)
         is_repo_comment = comment.repo.repo_id == self.db_repo.repo_id
         comment_repo_admin = is_repo_admin and is_repo_comment
 
         if super_admin or comment_owner or comment_repo_admin:
             text = self.request.POST.get('text')
             version = self.request.POST.get('version')
             if text == comment.text:
                 log.warning(
                     'Comment(repo): '
                     'Trying to create new version '
                     'with the same comment body {}'.format(
                         comment_id,
                     )
                 )
                 raise HTTPNotFound()
 
             if version.isdigit():
                 version = int(version)
             else:
                 log.warning(
                     'Comment(repo): Wrong version type {} {} '
                     'for comment {}'.format(
                         version,
                         type(version),
                         comment_id,
                     )
                 )
                 raise HTTPNotFound()
 
             try:
                 comment_history = CommentsModel().edit(
                     comment_id=comment_id,
                     text=text,
                     auth_user=self._rhodecode_user,
                     version=version,
                 )
             except CommentVersionMismatch:
                 raise HTTPConflict()
 
             if not comment_history:
                 raise HTTPNotFound()
 
-            commit_id = self.request.matchdict['commit_id']
-            commit = self.db_repo.get_commit(commit_id)
-            CommentsModel().trigger_commit_comment_hook(
-                self.db_repo, self._rhodecode_user, 'edit',
-                data={'comment': comment, 'commit': commit})
+            if not comment.draft:
+                commit = self.db_repo.get_commit(commit_id)
+                CommentsModel().trigger_commit_comment_hook(
+                    self.db_repo, self._rhodecode_user, 'edit',
+                    data={'comment': comment, 'commit': commit})
 
             Session().commit()
             return {
                 'comment_history_id': comment_history.comment_history_id,
                 'comment_id': comment.comment_id,
                 'comment_version': comment_history.version,
                 'comment_author_username': comment_history.author.username,
                 'comment_author_gravatar': h.gravatar_url(comment_history.author.email, 16),
                 'comment_created_on': h.age_component(comment_history.created_on,
                                                       time_is_local=True),
             }
         else:
             log.warning('No permissions for user %s to edit comment_id: %s',
                         self._rhodecode_db_user, comment_id)
             raise HTTPNotFound()
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_commit_data', request_method='GET',
         renderer='json_ext', xhr=True)
     def repo_commit_data(self):
         commit_id = self.request.matchdict['commit_id']
         self.load_default_context()
 
         try:
             return self.rhodecode_vcs_repo.get_commit(commit_id=commit_id)
         except CommitDoesNotExistError as e:
             return EmptyCommit(message=str(e))
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_commit_children', request_method='GET',
         renderer='json_ext', xhr=True)
     def repo_commit_children(self):
         commit_id = self.request.matchdict['commit_id']
         self.load_default_context()
 
         try:
             commit = self.rhodecode_vcs_repo.get_commit(commit_id=commit_id)
             children = commit.children
         except CommitDoesNotExistError:
             children = []
 
         result = {"results": children}
         return result
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_commit_parents', request_method='GET',
         renderer='json_ext')
     def repo_commit_parents(self):
         commit_id = self.request.matchdict['commit_id']
         self.load_default_context()
 
         try:
             commit = self.rhodecode_vcs_repo.get_commit(commit_id=commit_id)
             parents = commit.parents
         except CommitDoesNotExistError:
             parents = []
         result = {"results": parents}
         return result