release: Merge default into stable for release preparation

pkgs/patches/beaker/patch-beaker-improved-redis-2.diff

@@ -0,0 +1,18 b''
+diff -rup Beaker-1.9.1-orig/beaker/session.py Beaker-1.9.1/beaker/session.py
+--- Beaker-1.9.1-orig/beaker/session.py	2020-04-10 10:23:04.000000000 +0200
++++ Beaker-1.9.1/beaker/session.py	2020-04-10 10:23:34.000000000 +0200
+@@ -156,6 +156,14 @@ def __init__(self, request, id=None, invalidate_corrupt=False,
+         if timeout and not save_accessed_time:
+             raise BeakerException("timeout requires save_accessed_time")
+         self.timeout = timeout
++        # We want to pass timeout param to redis backend to support expiration of keys
++        # In future, I believe, we can use this param for memcached and mongo as well
++        if self.timeout is not None and self.type == 'ext:redis':
++            # The backend expiration should always be a bit longer (I decied to use 2 minutes) than the
++            # session expiration itself to prevent the case where the backend data expires while
++            # the session is being read (PR#153)
++            self.namespace_args['timeout'] = self.timeout + 60 * 2
++
+         self.save_atime = save_accessed_time
+         self.use_cookies = use_cookies
+         self.cookie_expires = cookie_expires
  No newline at end of file

pkgs/patches/beaker/patch-beaker-improved-redis.diff

@@ -0,0 +1,26 b''
+diff -rup Beaker-1.9.1-orig/beaker/ext/redisnm.py Beaker-1.9.1/beaker/ext/redisnm.py
+--- Beaker-1.9.1-orig/beaker/ext/redisnm.py	2018-04-10 10:23:04.000000000 +0200
++++ Beaker-1.9.1/beaker/ext/redisnm.py	2018-04-10 10:23:34.000000000 +0200
+@@ -30,9 +30,10 @@ class RedisNamespaceManager(NamespaceManager):
+
+     clients = SyncDict()
+
+-    def __init__(self, namespace, url, **kw):
++    def __init__(self, namespace, url, timeout=None, **kw):
+         super(RedisNamespaceManager, self).__init__(namespace)
+         self.lock_dir = None  # Redis uses redis itself for locking.
++        self.timeout = timeout
+
+         if redis is None:
+             raise RuntimeError('redis is not available')
+@@ -68,6 +69,8 @@ def has_key(self, key):
+
+     def set_value(self, key, value, expiretime=None):
+         value = pickle.dumps(value)
++        if expiretime is None and self.timeout is not None:
++            expiretime = self.timeout
+         if expiretime is not None:
+             self.client.setex(self._format_key(key), int(expiretime), value)
+         else:
+
+

.bumpversion.cfg

@@ -1,6 +1,5 b''
 [bumpversion]
-current_version = 4.20.1
+current_version = 4.21.0
 message = release: Bump version {current_version} to {new_version}
 
 [bumpversion:file:rhodecode/VERSION]
-

.release.cfg

@@ -1,33 +1,28 b''
 [DEFAULT]
 done = false
 
 [task:bump_version]
 done = true
 
 [task:rc_tools_pinned]
-done = true
 
 [task:fixes_on_stable]
-done = true
 
 [task:pip2nix_generated]
-done = true
 
 [task:changelog_updated]
-done = true
 
 [task:generate_api_docs]
-done = true
+
+[task:updated_translation]
 
 [release]
-state = prepared
-version = 4.20.1
-
-[task:updated_translation]
+state = in_progress
+version = 4.21.0
 
 [task:generate_js_routes]
 
 [task:updated_trial_license]
 
 [task:generate_oss_licenses]
 

docs/admin/system_admin/nginx/nginx-config-example.rst

@@ -1,193 +1,194 b''
 Nginx Configuration Example
 ---------------------------
 
 Use the following example to configure Nginx as a your web server.
 
 
 .. code-block:: nginx
 
     ## Rate limiter for certain pages to prevent brute force attacks
     limit_req_zone  $binary_remote_addr  zone=req_limit:10m   rate=1r/s;
 
     ## cache zone
     proxy_cache_path /etc/nginx/nginx_cache levels=1:2 use_temp_path=off keys_zone=cache_zone:10m inactive=720h max_size=10g;
 
     ## Custom log format
     log_format log_custom '$remote_addr - $remote_user [$time_local] '
                           '"$request" $status $body_bytes_sent '
                           '"$http_referer" "$http_user_agent" '
                           '$request_time $upstream_response_time $pipe';
 
     ## Define one or more upstreams (local RhodeCode instance) to connect to
     upstream rc {
         # Url to running RhodeCode instance.
         # This is shown as `- URL: <host>` in output from rccontrol status.
         server 127.0.0.1:10002;
 
         # add more instances for load balancing
         # server 127.0.0.1:10003;
         # server 127.0.0.1:10004;
     }
 
     ## HTTP to HTTPS rewrite
     server {
         listen          80;
         server_name     rhodecode.myserver.com;
 
         if ($http_host = rhodecode.myserver.com) {
             rewrite  (.*)  https://rhodecode.myserver.com$1 permanent;
         }
     }
 
     ## Optional gist alias server, for serving nicer GIST urls.
     server {
         listen          443;
         server_name     gist.myserver.com;
         access_log      /var/log/nginx/gist.access.log log_custom;
         error_log       /var/log/nginx/gist.error.log;
 
         ssl on;
         ssl_certificate     gist.rhodecode.myserver.com.crt;
         ssl_certificate_key gist.rhodecode.myserver.com.key;
 
         ssl_session_timeout 5m;
 
         ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
         ssl_prefer_server_ciphers on;
         ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA';
 
         ## Strict http prevents from https -> http downgrade
         add_header Strict-Transport-Security "max-age=31536000; includeSubdomains;";
 
         ## Diffie-Hellman parameter for DHE ciphersuites, recommended 2048 bits
         #ssl_dhparam /etc/nginx/ssl/dhparam.pem;
 
         rewrite ^/(.+)$ https://rhodecode.myserver.com/_admin/gists/$1;
         rewrite (.*)    https://rhodecode.myserver.com/_admin/gists;
     }
 
 
     ## MAIN SSL enabled server
     server {
         listen       443 ssl http2;
         server_name  rhodecode.myserver.com;
 
         access_log   /var/log/nginx/rhodecode.access.log log_custom;
         error_log    /var/log/nginx/rhodecode.error.log;
 
         ssl_certificate     rhodecode.myserver.com.crt;
         ssl_certificate_key rhodecode.myserver.com.key;
 
         # enable session resumption to improve https performance
         # http://vincent.bernat.im/en/blog/2011-ssl-session-reuse-rfc5077.html
         ssl_session_cache shared:SSL:50m;
         ssl_session_timeout 5m;
 
         ## Diffie-Hellman parameter for DHE ciphersuites, recommended 2048 bits
         #ssl_dhparam /etc/nginx/ssl/dhparam.pem;
 
         # enables server-side protection from BEAST attacks
         # http://blog.ivanristic.com/2013/09/is-beast-still-a-threat.html
         ssl_prefer_server_ciphers on;
 
         # disable SSLv3(enabled by default since nginx 0.8.19) since it's less secure then TLS http://en.wikipedia.org/wiki/Secure_Sockets_Layer#SSL_3.0
         ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
 
         # ciphers chosen for forward secrecy and compatibility
         # http://blog.ivanristic.com/2013/08/configuring-apache-nginx-and-openssl-for-forward-secrecy.html
         ssl_ciphers "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4";
 
         client_body_buffer_size     128k;
         # maximum number and size of buffers for large headers to read from client request
         large_client_header_buffers 16 256k;
 
         ## uncomment to serve static files by Nginx, recommended for performance
         # location /_static/rhodecode {
         #    gzip on;
         #    gzip_min_length  500;
         #    gzip_proxied     any;
         #    gzip_comp_level 4;
         #    gzip_types  text/css text/javascript text/xml text/plain text/x-component application/javascript application/json application/xml application/rss+xml font/truetype font/opentype application/vnd.ms-fontobject image/svg+xml;
         #    gzip_vary on;
         #    gzip_disable     "msie6";
         #    expires 60d;
         #    alias /path/to/.rccontrol/community-1/static;
         #    alias /path/to/.rccontrol/enterprise-1/static;
         # }
 
         ## channelstream location handler, if channelstream live chat and notifications
         ## are enable this will proxy the requests to channelstream websocket server
         location /_channelstream {
             rewrite /_channelstream/(.*) /$1 break;
             gzip                         off;
             tcp_nodelay                  off;
 
             proxy_connect_timeout        10;
             proxy_send_timeout           10m;
             proxy_read_timeout           10m;
 
             proxy_set_header             Host $host;
             proxy_set_header             X-Real-IP $remote_addr;
             proxy_set_header             X-Url-Scheme $scheme;
             proxy_set_header             X-Forwarded-Proto $scheme;
             proxy_set_header             X-Forwarded-For $proxy_add_x_forwarded_for;
 
             proxy_http_version           1.1;
             proxy_set_header Upgrade     $http_upgrade;
             proxy_set_header Connection  "upgrade";
 
             proxy_pass                  http://127.0.0.1:9800;
         }
 
         ## rate limit this endpoint to prevent login page brute-force attacks
         location /_admin/login {
             limit_req  zone=req_limit  burst=10  nodelay;
             try_files $uri @rhodecode_http;
         }
 
         ## Special Cache for file store, make sure you enable this intentionally as
         ## it could bypass upload files permissions
-        # location /_file_store/download {
+        # location /_file_store/download/gravatars {
         #
         #    proxy_cache cache_zone;
         #    # ignore Set-Cookie
         #    proxy_ignore_headers Set-Cookie;
-        #    proxy_ignore_headers Cookie;
+        #    # ignore cache-control
+        #    proxy_ignore_headers Cache-Control;
         #
         #    proxy_cache_key $host$uri$is_args$args;
         #    proxy_cache_methods GET;
         #
         #    proxy_cache_bypass  $http_cache_control;
         #    proxy_cache_valid 200 302 720h;
         #
         #    proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504;
         #
         #    # returns cache status in headers
         #    add_header X-Proxy-Cache $upstream_cache_status;
         #    add_header Cache-Control "public";
         #
         #    proxy_cache_lock on;
         #    proxy_cache_lock_age 5m;
         #
         #    proxy_pass  http://rc;
         #
         # }
 
         location / {
             try_files $uri @rhodecode_http;
         }
 
         location @rhodecode_http {
             # example of proxy.conf can be found in our docs.
             include     /etc/nginx/proxy.conf;
             proxy_pass  http://rc;
         }
 
         ## Custom 502 error page.
         ## Will be displayed while RhodeCode server is turned off
         error_page 502 /502.html;
         location = /502.html {
            #root  /path/to/.rccontrol/community-1/static;
            root  /path/to/.rccontrol/enterprise-1/static;
         }
     }
  No newline at end of file

pkgs/python-packages-overrides.nix

@@ -1,279 +1,281 b''
 # Overrides for the generated python-packages.nix
 #
 # This function is intended to be used as an extension to the generated file
 # python-packages.nix. The main objective is to add needed dependencies of C
 # libraries and tweak the build instructions where needed.
 
 { pkgs
 , basePythonPackages
 }:
 
 let
   sed = "sed -i";
 
   localLicenses = {
     repoze = {
       fullName = "Repoze License";
       url =  http://www.repoze.org/LICENSE.txt;
     };
   };
 
 in
 
 self: super: {
 
   "appenlight-client" = super."appenlight-client".override (attrs: {
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   });
 
   "beaker" = super."beaker".override (attrs: {
     patches = [
       ./patches/beaker/patch-beaker-lock-func-debug.diff
       ./patches/beaker/patch-beaker-metadata-reuse.diff
+      ./patches/beaker/patch-beaker-improved-redis.diff
+      ./patches/beaker/patch-beaker-improved-redis-2.diff
     ];
   });
 
   "cffi" = super."cffi".override (attrs: {
     buildInputs = [
       pkgs.libffi
     ];
   });
 
   "cryptography" = super."cryptography".override (attrs: {
     buildInputs = [
       pkgs.openssl
     ];
   });
 
   "gevent" = super."gevent".override (attrs: {
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
       # NOTE: (marcink) odd requirements from gevent aren't set properly,
       # thus we need to inject psutil manually
       self."psutil"
     ];
   });
 
   "future" = super."future".override (attrs: {
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   });
 
   "testpath" = super."testpath".override (attrs: {
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   });
 
   "gnureadline" = super."gnureadline".override (attrs: {
     buildInputs = [
       pkgs.ncurses
     ];
     patchPhase = ''
       substituteInPlace setup.py --replace "/bin/bash" "${pkgs.bash}/bin/bash"
     '';
   });
 
   "gunicorn" = super."gunicorn".override (attrs: {
     propagatedBuildInputs = [
       # johbo: futures is needed as long as we are on Python 2, otherwise
       # gunicorn explodes if used with multiple threads per worker.
       self."futures"
     ];
   });
 
   "nbconvert" = super."nbconvert".override (attrs: {
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
       # marcink: plug in jupyter-client for notebook rendering
       self."jupyter-client"
     ];
   });
 
   "ipython" = super."ipython".override (attrs: {
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
       self."gnureadline"
     ];
   });
 
   "lxml" = super."lxml".override (attrs: {
     buildInputs = [
       pkgs.libxml2
       pkgs.libxslt
     ];
     propagatedBuildInputs = [
       # Needed, so that "setup.py bdist_wheel" does work
       self."wheel"
     ];
   });
 
   "mysql-python" = super."mysql-python".override (attrs: {
     buildInputs = [
       pkgs.openssl
     ];
     propagatedBuildInputs = [
       pkgs.libmysql
       pkgs.zlib
     ];
   });
 
   "psycopg2" = super."psycopg2".override (attrs: {
     propagatedBuildInputs = [
       pkgs.postgresql
     ];
     meta = {
       license = pkgs.lib.licenses.lgpl3Plus;
     };
   });
 
   "pycurl" = super."pycurl".override (attrs: {
     propagatedBuildInputs = [
       pkgs.curl
       pkgs.openssl
     ];
 
     preConfigure = ''
       substituteInPlace setup.py --replace '--static-libs' '--libs'
       export PYCURL_SSL_LIBRARY=openssl
     '';
 
     meta = {
       license = pkgs.lib.licenses.mit;
     };
   });
 
   "pyramid" = super."pyramid".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
   "pyramid-debugtoolbar" = super."pyramid-debugtoolbar".override (attrs: {
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal localLicenses.repoze ];
     };
   });
 
   "pysqlite" = super."pysqlite".override (attrs: {
     propagatedBuildInputs = [
       pkgs.sqlite
     ];
     meta = {
       license = [ pkgs.lib.licenses.zlib pkgs.lib.licenses.libpng ];
     };
   });
 
   "python-ldap" = super."python-ldap".override (attrs: {
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
       pkgs.openldap
       pkgs.cyrus_sasl
       pkgs.openssl
     ];
   });
 
   "python-pam" = super."python-pam".override (attrs: {
     propagatedBuildInputs = [
         pkgs.pam
     ];
 
     # TODO: johbo: Check if this can be avoided, or transform into
     # a real patch
     patchPhase = ''
       substituteInPlace pam.py \
         --replace 'find_library("pam")' '"${pkgs.pam}/lib/libpam.so.0"'
     '';
 
     });
 
   "python-saml" = super."python-saml".override (attrs: {
     buildInputs = [
       pkgs.libxml2
       pkgs.libxslt
     ];
   });
 
   "dm.xmlsec.binding" = super."dm.xmlsec.binding".override (attrs: {
     buildInputs = [
       pkgs.libxml2
       pkgs.libxslt
       pkgs.xmlsec
       pkgs.libtool
     ];
   });
 
   "pyzmq" = super."pyzmq".override (attrs: {
     buildInputs = [
       pkgs.czmq
     ];
   });
 
   "urlobject" = super."urlobject".override (attrs: {
     meta = {
       license = {
         spdxId = "Unlicense";
         fullName = "The Unlicense";
         url = http://unlicense.org/;
       };
     };
   });
 
   "docutils" = super."docutils".override (attrs: {
     meta = {
       license = pkgs.lib.licenses.bsd2;
     };
   });
 
   "colander" = super."colander".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
   "pyramid-beaker"  = super."pyramid-beaker".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
   "pyramid-mako" = super."pyramid-mako".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
   "repoze.lru" = super."repoze.lru".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
   "python-editor" = super."python-editor".override (attrs: {
     meta = {
       license = pkgs.lib.licenses.asl20;
     };
   });
 
   "translationstring" = super."translationstring".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
   "venusian" = super."venusian".override (attrs: {
     meta = {
       license = localLicenses.repoze;
     };
   });
 
   "supervisor" = super."supervisor".override (attrs: {
     patches = [
       ./patches/supervisor/patch-rlimits-old-kernel.diff
     ];
   });
 
   # Avoid that base packages screw up the build process
   inherit (basePythonPackages)
     setuptools;
 
 }

pkgs/python-packages.nix

@@ -1,2444 +1,2497 b''
 # Generated by pip2nix 0.8.0.dev1
 # See https://github.com/johbo/pip2nix
 
 { pkgs, fetchurl, fetchgit, fetchhg }:
 
 self: super: {
   "alembic" = super.buildPythonPackage {
     name = "alembic-1.4.2";
     doCheck = false;
     propagatedBuildInputs = [
       self."sqlalchemy"
       self."mako"
       self."python-editor"
       self."python-dateutil"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/60/1e/cabc75a189de0fbb2841d0975243e59bde8b7822bacbb95008ac6fe9ad47/alembic-1.4.2.tar.gz";
       sha256 = "1gsdrzx9h7wfva200qvvsc9sn4w79mk2vs0bbnzjhxi1jw2b0nh3";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "amqp" = super.buildPythonPackage {
     name = "amqp-2.5.2";
     doCheck = false;
     propagatedBuildInputs = [
       self."vine"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/92/1d/433541994a5a69f4ad2fff39746ddbb0bdedb0ea0d85673eb0db68a7edd9/amqp-2.5.2.tar.gz";
       sha256 = "13dhhfxjrqcjybnq4zahg92mydhpg2l76nxcmq7d560687wsxwbp";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
+  "apispec" = super.buildPythonPackage {
+    name = "apispec-1.0.0";
+    doCheck = false;
+    propagatedBuildInputs = [
+      self."PyYAML"
+    ];
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/67/15/346c04988dd67d36007e28145504c520491930c878b1f484a97b27a8f497/apispec-1.0.0.tar.gz";
+      sha256 = "1712w1anvqrvadjjpvai84vbaygaxabd3zz5lxihdzwzs4gvi9sp";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.mit ];
+    };
+  };
   "appenlight-client" = super.buildPythonPackage {
     name = "appenlight-client-0.6.26";
     doCheck = false;
     propagatedBuildInputs = [
       self."webob"
       self."requests"
       self."six"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/2e/56/418fc10379b96e795ee39a15e69a730c222818af04c3821fa354eaa859ec/appenlight_client-0.6.26.tar.gz";
       sha256 = "0s9xw3sb8s3pk73k78nnq4jil3q4mk6bczfa1fmgfx61kdxl2712";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "asn1crypto" = super.buildPythonPackage {
     name = "asn1crypto-0.24.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/fc/f1/8db7daa71f414ddabfa056c4ef792e1461ff655c2ae2928a2b675bfed6b4/asn1crypto-0.24.0.tar.gz";
       sha256 = "0jaf8rf9dx1lf23xfv2cdd5h52f1qr3w8k63985bc35g3d220p4x";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "atomicwrites" = super.buildPythonPackage {
     name = "atomicwrites-1.3.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ec/0f/cd484ac8820fed363b374af30049adc8fd13065720fd4f4c6be8a2309da7/atomicwrites-1.3.0.tar.gz";
       sha256 = "19ngcscdf3jsqmpcxn6zl5b6anmsajb6izp1smcd1n02midl9abm";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "attrs" = super.buildPythonPackage {
     name = "attrs-19.3.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/98/c3/2c227e66b5e896e15ccdae2e00bbc69aa46e9a8ce8869cc5fa96310bf612/attrs-19.3.0.tar.gz";
       sha256 = "0wky4h28n7xnr6xv69p9z6kv8bzn50d10c3drmd9ds8gawbcxdzp";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "babel" = super.buildPythonPackage {
     name = "babel-1.3";
     doCheck = false;
     propagatedBuildInputs = [
       self."pytz"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/33/27/e3978243a03a76398c384c83f7ca879bc6e8f1511233a621fcada135606e/Babel-1.3.tar.gz";
       sha256 = "0bnin777lc53nxd1hp3apq410jj5wx92n08h7h4izpl4f4sx00lz";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "backports.shutil-get-terminal-size" = super.buildPythonPackage {
     name = "backports.shutil-get-terminal-size-1.0.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ec/9c/368086faa9c016efce5da3e0e13ba392c9db79e3ab740b763fe28620b18b/backports.shutil_get_terminal_size-1.0.0.tar.gz";
       sha256 = "107cmn7g3jnbkp826zlj8rrj19fam301qvaqf0f3905f5217lgki";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "beaker" = super.buildPythonPackage {
     name = "beaker-1.9.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."funcsigs"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ca/14/a626188d0d0c7b55dd7cf1902046c2743bd392a7078bb53073e13280eb1e/Beaker-1.9.1.tar.gz";
       sha256 = "08arsn61r255lhz6hcpn2lsiqpg30clla805ysx06wmbhvb6w9rj";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "beautifulsoup4" = super.buildPythonPackage {
     name = "beautifulsoup4-4.6.3";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/88/df/86bffad6309f74f3ff85ea69344a078fc30003270c8df6894fca7a3c72ff/beautifulsoup4-4.6.3.tar.gz";
       sha256 = "041dhalzjciw6qyzzq7a2k4h1yvyk76xigp35hv5ibnn448ydy4h";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "billiard" = super.buildPythonPackage {
     name = "billiard-3.6.1.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/68/1d/2aea8fbb0b1e1260a8a2e77352de2983d36d7ac01207cf14c2b9c6cc860e/billiard-3.6.1.0.tar.gz";
       sha256 = "09hzy3aqi7visy4vmf4xiish61n0rq5nd3iwjydydps8yrs9r05q";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "bleach" = super.buildPythonPackage {
     name = "bleach-3.1.3";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."webencodings"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/de/09/5267f8577a92487ed43bc694476c4629c6eca2e3c93fcf690a26bfe39e1d/bleach-3.1.3.tar.gz";
       sha256 = "0al437aw4p2xp83az5hhlrp913nsf0cg6kg4qj3fjhv4wakxipzq";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "bumpversion" = super.buildPythonPackage {
     name = "bumpversion-0.5.3";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/14/41/8c9da3549f8e00c84f0432c3a8cf8ed6898374714676aab91501d48760db/bumpversion-0.5.3.tar.gz";
       sha256 = "0zn7694yfipxg35ikkfh7kvgl2fissha3dnqad2c5bvsvmrwhi37";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "cachetools" = super.buildPythonPackage {
     name = "cachetools-3.1.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ae/37/7fd45996b19200e0cb2027a0b6bef4636951c4ea111bfad36c71287247f6/cachetools-3.1.1.tar.gz";
       sha256 = "16m69l6n6y1r1y7cklm92rr7v69ldig2n3lbl3j323w5jz7d78lf";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "celery" = super.buildPythonPackage {
     name = "celery-4.3.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."pytz"
       self."billiard"
       self."kombu"
       self."vine"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/a2/4b/d020836f751617e907e84753a41c92231cd4b673ff991b8ee9da52361323/celery-4.3.0.tar.gz";
       sha256 = "1y8y0gbgkwimpxqnxq2rm5qz2vy01fvjiybnpm00y5rzd2m34iac";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "certifi" = super.buildPythonPackage {
     name = "certifi-2020.4.5.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/b8/e2/a3a86a67c3fc8249ed305fc7b7d290ebe5e4d46ad45573884761ef4dea7b/certifi-2020.4.5.1.tar.gz";
       sha256 = "06b5gfs7wmmipln8f3z928d2mmx2j4b3x7pnqmj6cvmyfh8v7z2i";
     };
     meta = {
       license = [ pkgs.lib.licenses.mpl20 { fullName = "Mozilla Public License 2.0 (MPL 2.0)"; } ];
     };
   };
   "cffi" = super.buildPythonPackage {
     name = "cffi-1.12.3";
     doCheck = false;
     propagatedBuildInputs = [
       self."pycparser"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/93/1a/ab8c62b5838722f29f3daffcc8d4bd61844aa9b5f437341cc890ceee483b/cffi-1.12.3.tar.gz";
       sha256 = "0x075521fxwv0mfp4cqzk7lvmw4n94bjw601qkcv314z5s182704";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "chameleon" = super.buildPythonPackage {
     name = "chameleon-2.24";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/5a/9e/637379ffa13c5172b5c0e704833ffea6bf51cec7567f93fd6e903d53ed74/Chameleon-2.24.tar.gz";
       sha256 = "0ykqr7syxfa6h9adjfnsv1gdsca2xzm22vmic8859n0f0j09abj5";
     };
     meta = {
       license = [ { fullName = "BSD-like (http://repoze.org/license.html)"; } ];
     };
   };
   "channelstream" = super.buildPythonPackage {
-    name = "channelstream-0.5.2";
+    name = "channelstream-0.6.14";
     doCheck = false;
     propagatedBuildInputs = [
       self."gevent"
       self."ws4py"
+      self."marshmallow"
+      self."python-dateutil"
       self."pyramid"
       self."pyramid-jinja2"
+      self."pyramid-apispec"
       self."itsdangerous"
       self."requests"
       self."six"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/2b/31/29a8e085cf5bf97fa88e7b947adabfc581a18a3463adf77fb6dada34a65f/channelstream-0.5.2.tar.gz";
-      sha256 = "1qbm4xdl5hfkja683x546bncg3rqq8qv79w1m1a1wd48cqqzb6rm";
+      url = "https://files.pythonhosted.org/packages/d4/2d/86d6757ccd06ce673ee224123471da3d45251d061da7c580bfc259bad853/channelstream-0.6.14.tar.gz";
+      sha256 = "0qgy5j3rj6c8cslzidh32glhkrhbbdxjc008y69v8a0y3zyaz2d3";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "chardet" = super.buildPythonPackage {
     name = "chardet-3.0.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/fc/bb/a5768c230f9ddb03acc9ef3f0d4a3cf93462473795d18e9535498c8f929d/chardet-3.0.4.tar.gz";
       sha256 = "1bpalpia6r5x1kknbk11p1fzph56fmmnp405ds8icksd3knr5aw4";
     };
     meta = {
       license = [ { fullName = "LGPL"; } { fullName = "GNU Library or Lesser General Public License (LGPL)"; } ];
     };
   };
   "click" = super.buildPythonPackage {
     name = "click-7.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/f8/5c/f60e9d8a1e77005f664b76ff8aeaee5bc05d0a91798afd7f53fc998dbc47/Click-7.0.tar.gz";
       sha256 = "1mzjixd4vjbjvzb6vylki9w1556a9qmdh35kzmq6cign46av952v";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "colander" = super.buildPythonPackage {
     name = "colander-1.7.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."translationstring"
       self."iso8601"
       self."enum34"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/db/e4/74ab06f54211917b41865cafc987ce511e35503de48da9bfe9358a1bdc3e/colander-1.7.0.tar.gz";
       sha256 = "1wl1bqab307lbbcjx81i28s3yl6dlm4rf15fxawkjb6j48x1cn6p";
     };
     meta = {
       license = [ { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "configobj" = super.buildPythonPackage {
     name = "configobj-5.0.6";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
     ];
     src = fetchurl {
       url = "https://code.rhodecode.com/upstream/configobj/artifacts/download/0-012de99a-b1e1-4f64-a5c0-07a98a41b324.tar.gz?md5=6a513f51fe04b2c18cf84c1395a7c626";
       sha256 = "0kqfrdfr14mw8yd8qwq14dv2xghpkjmd3yjsy8dfcbvpcc17xnxp";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "configparser" = super.buildPythonPackage {
     name = "configparser-4.0.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/16/4f/48975536bd488d3a272549eb795ac4a13a5f7fcdc8995def77fbef3532ee/configparser-4.0.2.tar.gz";
       sha256 = "1priacxym85yjcf68hh38w55nqswaxp71ryjyfdk222kg9l85ln7";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "contextlib2" = super.buildPythonPackage {
     name = "contextlib2-0.6.0.post1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/02/54/669207eb72e3d8ae8b38aa1f0703ee87a0e9f88f30d3c0a47bebdb6de242/contextlib2-0.6.0.post1.tar.gz";
       sha256 = "0bhnr2ac7wy5l85ji909gyljyk85n92w8pdvslmrvc8qih4r1x01";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   "cov-core" = super.buildPythonPackage {
     name = "cov-core-1.15.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."coverage"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/4b/87/13e75a47b4ba1be06f29f6d807ca99638bedc6b57fa491cd3de891ca2923/cov-core-1.15.0.tar.gz";
       sha256 = "0k3np9ymh06yv1ib96sb6wfsxjkqhmik8qfsn119vnhga9ywc52a";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "coverage" = super.buildPythonPackage {
     name = "coverage-4.5.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/85/d5/818d0e603685c4a613d56f065a721013e942088047ff1027a632948bdae6/coverage-4.5.4.tar.gz";
       sha256 = "0p0j4di6h8k6ica7jwwj09azdcg4ycxq60i9qsskmsg94cd9yzg0";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "cryptography" = super.buildPythonPackage {
     name = "cryptography-2.6.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."asn1crypto"
       self."six"
       self."cffi"
       self."enum34"
       self."ipaddress"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/07/ca/bc827c5e55918ad223d59d299fff92f3563476c3b00d0a9157d9c0217449/cryptography-2.6.1.tar.gz";
       sha256 = "19iwz5avym5zl6jrrrkym1rdaa9h61j20ph4cswsqgv8xg5j3j16";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "BSD or Apache License, Version 2.0"; } pkgs.lib.licenses.asl20 ];
     };
   };
   "cssselect" = super.buildPythonPackage {
     name = "cssselect-1.0.3";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/52/ea/f31e1d2e9eb130fda2a631e22eac369dc644e8807345fbed5113f2d6f92b/cssselect-1.0.3.tar.gz";
       sha256 = "011jqa2jhmydhi0iz4v1w3cr540z5zas8g2bw8brdw4s4b2qnv86";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "cssutils" = super.buildPythonPackage {
     name = "cssutils-1.0.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/5c/0b/c5f29d29c037e97043770b5e7c740b6252993e4b57f029b3cd03c78ddfec/cssutils-1.0.2.tar.gz";
       sha256 = "1bxchrbqzapwijap0yhlxdil1w9bmwvgx77aizlkhc2mcxjg1z52";
     };
     meta = {
       license = [ { fullName = "GNU Library or Lesser General Public License (LGPL)"; } { fullName = "LGPL 2.1 or later, see also http://cthedot.de/cssutils/"; } ];
     };
   };
   "decorator" = super.buildPythonPackage {
     name = "decorator-4.1.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/bb/e0/f6e41e9091e130bf16d4437dabbac3993908e4d6485ecbc985ef1352db94/decorator-4.1.2.tar.gz";
       sha256 = "1d8npb11kxyi36mrvjdpcjij76l5zfyrz2f820brf0l0rcw4vdkw";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "new BSD License"; } ];
     };
   };
   "deform" = super.buildPythonPackage {
     name = "deform-2.0.8";
     doCheck = false;
     propagatedBuildInputs = [
       self."chameleon"
       self."colander"
       self."iso8601"
       self."peppercorn"
       self."translationstring"
       self."zope.deprecation"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/21/d0/45fdf891a82722c02fc2da319cf2d1ae6b5abf9e470ad3762135a895a868/deform-2.0.8.tar.gz";
       sha256 = "0wbjv98sib96649aqaygzxnrkclyy50qij2rha6fn1i4c86bfdl9";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "defusedxml" = super.buildPythonPackage {
     name = "defusedxml-0.6.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/a4/5f/f8aa58ca0cf01cbcee728abc9d88bfeb74e95e6cb4334cfd5bed5673ea77/defusedxml-0.6.0.tar.gz";
       sha256 = "1xbp8fivl3wlbyg2jrvs4lalaqv1xp9a9f29p75wdx2s2d6h717n";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   "dm.xmlsec.binding" = super.buildPythonPackage {
     name = "dm.xmlsec.binding-1.3.7";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
       self."lxml"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/2c/9e/7651982d50252692991acdae614af821fd6c79bc8dcd598ad71d55be8fc7/dm.xmlsec.binding-1.3.7.tar.gz";
       sha256 = "03jjjscx1pz2nc0dwiw9nia02qbz1c6f0f9zkyr8fmvys2n5jkb3";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "docutils" = super.buildPythonPackage {
     name = "docutils-0.16";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/2f/e0/3d435b34abd2d62e8206171892f174b180cd37b09d57b924ca5c2ef2219d/docutils-0.16.tar.gz";
       sha256 = "1z3qliszqca9m719q3qhdkh0ghh90g500avzdgi7pl77x5h3mpn2";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal pkgs.lib.licenses.publicDomain pkgs.lib.licenses.gpl1 { fullName = "public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt)"; } pkgs.lib.licenses.psfl ];
     };
   };
   "dogpile.cache" = super.buildPythonPackage {
     name = "dogpile.cache-0.9.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."decorator"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ac/6a/9ac405686a94b7f009a20a50070a5786b0e1aedc707b88d40d0c4b51a82e/dogpile.cache-0.9.0.tar.gz";
       sha256 = "0sr1fn6b4k5bh0cscd9yi8csqxvj4ngzildav58x5p694mc86j5k";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "dogpile.core" = super.buildPythonPackage {
     name = "dogpile.core-0.4.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/0e/77/e72abc04c22aedf874301861e5c1e761231c288b5de369c18be8f4b5c9bb/dogpile.core-0.4.1.tar.gz";
       sha256 = "0xpdvg4kr1isfkrh1rfsh7za4q5a5s6l2kf9wpvndbwf3aqjyrdy";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "ecdsa" = super.buildPythonPackage {
     name = "ecdsa-0.13.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/51/76/139bf6e9b7b6684d5891212cdbd9e0739f2bfc03f380a1a6ffa700f392ac/ecdsa-0.13.2.tar.gz";
       sha256 = "116qaq7bh4lcynzi613960jhsnn19v0kmsqwahiwjfj14gx4y0sw";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "elasticsearch" = super.buildPythonPackage {
     name = "elasticsearch-6.3.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."urllib3"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/9d/ce/c4664e8380e379a9402ecfbaf158e56396da90d520daba21cfa840e0eb71/elasticsearch-6.3.1.tar.gz";
       sha256 = "12y93v0yn7a4xmf969239g8gb3l4cdkclfpbk1qc8hx5qkymrnma";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "elasticsearch-dsl" = super.buildPythonPackage {
     name = "elasticsearch-dsl-6.3.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."python-dateutil"
       self."elasticsearch"
       self."ipaddress"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/4c/0d/1549f50c591db6bb4e66cbcc8d34a6e537c3d89aa426b167c244fd46420a/elasticsearch-dsl-6.3.1.tar.gz";
       sha256 = "1gh8a0shqi105k325hgwb9avrpdjh0mc6mxwfg9ba7g6lssb702z";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "elasticsearch1" = super.buildPythonPackage {
     name = "elasticsearch1-1.10.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."urllib3"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/a6/eb/73e75f9681fa71e3157b8ee878534235d57f24ee64f0e77f8d995fb57076/elasticsearch1-1.10.0.tar.gz";
       sha256 = "0g89444kd5zwql4vbvyrmi2m6l6dcj6ga98j4hqxyyyz6z20aki2";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "elasticsearch1-dsl" = super.buildPythonPackage {
     name = "elasticsearch1-dsl-0.0.12";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."python-dateutil"
       self."elasticsearch1"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/eb/9d/785342775cb10eddc9b8d7457d618a423b4f0b89d8b2b2d1bc27190d71db/elasticsearch1-dsl-0.0.12.tar.gz";
       sha256 = "0ig1ly39v93hba0z975wnhbmzwj28w6w1sqlr2g7cn5spp732bhk";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "elasticsearch2" = super.buildPythonPackage {
     name = "elasticsearch2-2.5.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."urllib3"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/f6/09/f9b24aa6b1120bea371cd57ef6f57c7694cf16660469456a8be6c2bdbe22/elasticsearch2-2.5.1.tar.gz";
       sha256 = "19k2znpjfyp0hrq73cz7pjyj289040xpsxsm0xhh4jfh6y551g7k";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "entrypoints" = super.buildPythonPackage {
     name = "entrypoints-0.2.2";
     doCheck = false;
     propagatedBuildInputs = [
       self."configparser"
     ];
     src = fetchurl {
       url = "https://code.rhodecode.com/upstream/entrypoints/artifacts/download/0-8e9ee9e4-c4db-409c-b07e-81568fd1832d.tar.gz?md5=3a027b8ff1d257b91fe257de6c43357d";
       sha256 = "0qih72n2myclanplqipqxpgpj9d2yhff1pz5d02zq1cfqyd173w5";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "enum34" = super.buildPythonPackage {
     name = "enum34-1.1.10";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/11/c4/2da1f4952ba476677a42f25cd32ab8aaf0e1c0d0e00b89822b835c7e654c/enum34-1.1.10.tar.gz";
       sha256 = "0j7ji699fwswm4vg6w1v07fkbf8dkzdm6gfh88jvs5nqgr3sgrnc";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "formencode" = super.buildPythonPackage {
     name = "formencode-1.2.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/8e/59/0174271a6f004512e0201188593e6d319db139d14cb7490e488bbb078015/FormEncode-1.2.4.tar.gz";
       sha256 = "1fgy04sdy4yry5xcjls3x3xy30dqwj58ycnkndim819jx0788w42";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   "funcsigs" = super.buildPythonPackage {
     name = "funcsigs-1.0.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/94/4a/db842e7a0545de1cdb0439bb80e6e42dfe82aaeaadd4072f2263a4fbed23/funcsigs-1.0.2.tar.gz";
       sha256 = "0l4g5818ffyfmfs1a924811azhjj8ax9xd1cffr1mzd3ycn0zfx7";
     };
     meta = {
       license = [ { fullName = "ASL"; } pkgs.lib.licenses.asl20 ];
     };
   };
   "functools32" = super.buildPythonPackage {
     name = "functools32-3.2.3.post2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/c5/60/6ac26ad05857c601308d8fb9e87fa36d0ebf889423f47c3502ef034365db/functools32-3.2.3-2.tar.gz";
       sha256 = "0v8ya0b58x47wp216n1zamimv4iw57cxz3xxhzix52jkw3xks9gn";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   "future" = super.buildPythonPackage {
     name = "future-0.14.3";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/83/80/8ef3a11a15f8eaafafa0937b20c1b3f73527e69ab6b3fa1cf94a5a96aabb/future-0.14.3.tar.gz";
       sha256 = "1savk7jx7hal032f522c5ajhh8fra6gmnadrj9adv5qxi18pv1b2";
     };
     meta = {
       license = [ { fullName = "OSI Approved"; } pkgs.lib.licenses.mit ];
     };
   };
   "futures" = super.buildPythonPackage {
     name = "futures-3.0.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/f8/e7/fc0fcbeb9193ba2d4de00b065e7fd5aecd0679e93ce95a07322b2b1434f4/futures-3.0.2.tar.gz";
       sha256 = "0mz2pbgxbc2nbib1szifi07whjbfs4r02pv2z390z7p410awjgyw";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "gevent" = super.buildPythonPackage {
     name = "gevent-1.5.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."greenlet"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/5a/79/2c63d385d017b5dd7d70983a463dfd25befae70c824fedb857df6e72eff2/gevent-1.5.0.tar.gz";
       sha256 = "0aac3d4vhv5n4rsb6cqzq0d1xx9immqz4fmpddw35yxkwdc450dj";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "gnureadline" = super.buildPythonPackage {
     name = "gnureadline-6.3.8";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/50/64/86085c823cd78f9df9d8e33dce0baa71618016f8860460b82cf6610e1eb3/gnureadline-6.3.8.tar.gz";
       sha256 = "0ddhj98x2nv45iz4aadk4b9m0b1kpsn1xhcbypn5cd556knhiqjq";
     };
     meta = {
       license = [ { fullName = "GNU General Public License v3 (GPLv3)"; } pkgs.lib.licenses.gpl1 ];
     };
   };
   "gprof2dot" = super.buildPythonPackage {
     name = "gprof2dot-2017.9.19";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/9d/36/f977122502979f3dfb50704979c9ed70e6b620787942b089bf1af15f5aba/gprof2dot-2017.9.19.tar.gz";
       sha256 = "17ih23ld2nzgc3xwgbay911l6lh96jp1zshmskm17n1gg2i7mg6f";
     };
     meta = {
       license = [ { fullName = "GNU Lesser General Public License v3 or later (LGPLv3+)"; } { fullName = "LGPL"; } ];
     };
   };
   "greenlet" = super.buildPythonPackage {
     name = "greenlet-0.4.15";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/f8/e8/b30ae23b45f69aa3f024b46064c0ac8e5fcb4f22ace0dca8d6f9c8bbe5e7/greenlet-0.4.15.tar.gz";
       sha256 = "1g4g1wwc472ds89zmqlpyan3fbnzpa8qm48z3z1y6mlk44z485ll";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "gunicorn" = super.buildPythonPackage {
     name = "gunicorn-19.9.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/47/52/68ba8e5e8ba251e54006a49441f7ccabca83b6bef5aedacb4890596c7911/gunicorn-19.9.0.tar.gz";
       sha256 = "1wzlf4xmn6qjirh5w81l6i6kqjnab1n1qqkh7zsj1yb6gh4n49ps";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "hupper" = super.buildPythonPackage {
     name = "hupper-1.10.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/41/24/ea90fef04706e54bd1635c05c50dc9cf87cda543c59303a03e7aa7dda0ce/hupper-1.10.2.tar.gz";
       sha256 = "0am0p6g5cz6xmcaf04xq8q6dzdd9qz0phj6gcmpsckf2mcyza61q";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "idna" = super.buildPythonPackage {
     name = "idna-2.8";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ad/13/eb56951b6f7950cadb579ca166e448ba77f9d24efc03edd7e55fa57d04b7/idna-2.8.tar.gz";
       sha256 = "01rlkigdxg17sf9yar1jl8n18ls59367wqh59hnawlyg53vb6my3";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "BSD-like"; } ];
     };
   };
   "importlib-metadata" = super.buildPythonPackage {
     name = "importlib-metadata-1.6.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."zipp"
       self."pathlib2"
       self."contextlib2"
       self."configparser"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/b4/1b/baab42e3cd64c9d5caac25a9d6c054f8324cdc38975a44d600569f1f7158/importlib_metadata-1.6.0.tar.gz";
       sha256 = "07icyggasn38yv2swdrd8z6i0plazmc9adavsdkbqqj91j53ll9l";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "infrae.cache" = super.buildPythonPackage {
     name = "infrae.cache-1.0.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."beaker"
       self."repoze.lru"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/bb/f0/e7d5e984cf6592fd2807dc7bc44a93f9d18e04e6a61f87fdfb2622422d74/infrae.cache-1.0.1.tar.gz";
       sha256 = "1dvqsjn8vw253wz9d1pz17j79mf4bs53dvp2qxck2qdp1am1njw4";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "invoke" = super.buildPythonPackage {
     name = "invoke-0.13.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/47/bf/d07ef52fa1ac645468858bbac7cb95b246a972a045e821493d17d89c81be/invoke-0.13.0.tar.gz";
       sha256 = "0794vhgxfmkh0vzkkg5cfv1w82g3jc3xr18wim29far9qpx9468s";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "ipaddress" = super.buildPythonPackage {
     name = "ipaddress-1.0.23";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/b9/9a/3e9da40ea28b8210dd6504d3fe9fe7e013b62bf45902b458d1cdc3c34ed9/ipaddress-1.0.23.tar.gz";
       sha256 = "1qp743h30s04m3cg3yk3fycad930jv17q7dsslj4mfw0jlvf1y5p";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   "ipdb" = super.buildPythonPackage {
     name = "ipdb-0.13.2";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
       self."ipython"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/2c/bb/a3e1a441719ebd75c6dac8170d3ddba884b7ee8a5c0f9aefa7297386627a/ipdb-0.13.2.tar.gz";
       sha256 = "0jcd849rx30y3wcgzsqbn06v0yjlzvb9x3076q0yxpycdwm1ryvp";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "ipython" = super.buildPythonPackage {
     name = "ipython-5.1.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
       self."decorator"
       self."pickleshare"
       self."simplegeneric"
       self."traitlets"
       self."prompt-toolkit"
       self."pygments"
       self."pexpect"
       self."backports.shutil-get-terminal-size"
       self."pathlib2"
       self."pexpect"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/89/63/a9292f7cd9d0090a0f995e1167f3f17d5889dcbc9a175261719c513b9848/ipython-5.1.0.tar.gz";
       sha256 = "0qdrf6aj9kvjczd5chj1my8y2iq09am9l8bb2a1334a52d76kx3y";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "ipython-genutils" = super.buildPythonPackage {
     name = "ipython-genutils-0.2.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/e8/69/fbeffffc05236398ebfcfb512b6d2511c622871dca1746361006da310399/ipython_genutils-0.2.0.tar.gz";
       sha256 = "1a4bc9y8hnvq6cp08qs4mckgm6i6ajpndp4g496rvvzcfmp12bpb";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "iso8601" = super.buildPythonPackage {
     name = "iso8601-0.1.12";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/45/13/3db24895497345fb44c4248c08b16da34a9eb02643cea2754b21b5ed08b0/iso8601-0.1.12.tar.gz";
       sha256 = "10nyvvnrhw2w3p09v1ica4lgj6f4g9j3kkfx17qmraiq3w7b5i29";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "isodate" = super.buildPythonPackage {
     name = "isodate-0.6.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/b1/80/fb8c13a4cd38eb5021dc3741a9e588e4d1de88d895c1910c6fc8a08b7a70/isodate-0.6.0.tar.gz";
       sha256 = "1n7jkz68kk5pwni540pr5zdh99bf6ywydk1p5pdrqisrawylldif";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "itsdangerous" = super.buildPythonPackage {
-    name = "itsdangerous-0.24";
+    name = "itsdangerous-1.1.0";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/dc/b4/a60bcdba945c00f6d608d8975131ab3f25b22f2bcfe1dab221165194b2d4/itsdangerous-0.24.tar.gz";
-      sha256 = "06856q6x675ly542ig0plbqcyab6ksfzijlyf1hzhgg3sgwgrcyb";
+      url = "https://files.pythonhosted.org/packages/68/1a/f27de07a8a304ad5fa817bbe383d1238ac4396da447fa11ed937039fa04b/itsdangerous-1.1.0.tar.gz";
+      sha256 = "068zpbksq5q2z4dckh2k1zbcq43ay74ylqn77rni797j0wyh66rj";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "jinja2" = super.buildPythonPackage {
     name = "jinja2-2.9.6";
     doCheck = false;
     propagatedBuildInputs = [
       self."markupsafe"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/90/61/f820ff0076a2599dd39406dcb858ecb239438c02ce706c8e91131ab9c7f1/Jinja2-2.9.6.tar.gz";
       sha256 = "1zzrkywhziqffrzks14kzixz7nd4yh2vc0fb04a68vfd2ai03anx";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "jsonschema" = super.buildPythonPackage {
     name = "jsonschema-2.6.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."functools32"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/58/b9/171dbb07e18c6346090a37f03c7e74410a1a56123f847efed59af260a298/jsonschema-2.6.0.tar.gz";
       sha256 = "00kf3zmpp9ya4sydffpifn0j0mzm342a2vzh82p6r0vh10cg7xbg";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "jupyter-client" = super.buildPythonPackage {
     name = "jupyter-client-5.0.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."traitlets"
       self."jupyter-core"
       self."pyzmq"
       self."python-dateutil"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/e5/6f/65412ed462202b90134b7e761b0b7e7f949e07a549c1755475333727b3d0/jupyter_client-5.0.0.tar.gz";
       sha256 = "0nxw4rqk4wsjhc87gjqd7pv89cb9dnimcfnmcmp85bmrvv1gjri7";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "jupyter-core" = super.buildPythonPackage {
     name = "jupyter-core-4.5.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."traitlets"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/4a/de/ff4ca734656d17ebe0450807b59d728f45277e2e7f4b82bc9aae6cb82961/jupyter_core-4.5.0.tar.gz";
       sha256 = "1xr4pbghwk5hayn5wwnhb7z95380r45p79gf5if5pi1akwg7qvic";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "kombu" = super.buildPythonPackage {
     name = "kombu-4.6.6";
     doCheck = false;
     propagatedBuildInputs = [
       self."amqp"
       self."importlib-metadata"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/20/e6/bc2d9affba6138a1dc143f77fef253e9e08e238fa7c0688d917c09005e96/kombu-4.6.6.tar.gz";
       sha256 = "11mxpcy8mg1l35bgbhba70v29bydr2hrhdbdlb4lg98m3m5vaq0p";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "lxml" = super.buildPythonPackage {
     name = "lxml-4.2.5";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/4b/20/ddf5eb3bd5c57582d2b4652b4bbcf8da301bdfe5d805cb94e805f4d7464d/lxml-4.2.5.tar.gz";
       sha256 = "0zw0y9hs0nflxhl9cs6ipwwh53szi3w2x06wl0k9cylyqac0cwin";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "mako" = super.buildPythonPackage {
     name = "mako-1.1.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."markupsafe"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/b0/3c/8dcd6883d009f7cae0f3157fb53e9afb05a0d3d33b3db1268ec2e6f4a56b/Mako-1.1.0.tar.gz";
       sha256 = "0jqa3qfpykyn4fmkn0kh6043sfls7br8i2bsdbccazcvk9cijsd3";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "markdown" = super.buildPythonPackage {
     name = "markdown-2.6.11";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/b3/73/fc5c850f44af5889192dff783b7b0d8f3fe8d30b65c8e3f78f8f0265fecf/Markdown-2.6.11.tar.gz";
       sha256 = "108g80ryzykh8bj0i7jfp71510wrcixdi771lf2asyghgyf8cmm8";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "markupsafe" = super.buildPythonPackage {
     name = "markupsafe-1.1.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/b9/2e/64db92e53b86efccfaea71321f597fa2e1b2bd3853d8ce658568f7a13094/MarkupSafe-1.1.1.tar.gz";
       sha256 = "0sqipg4fk7xbixqd8kq6rlkxj664d157bdwbh93farcphf92x1r9";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal pkgs.lib.licenses.bsd3 ];
     };
   };
+  "marshmallow" = super.buildPythonPackage {
+    name = "marshmallow-2.18.0";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/ad/0b/5799965d1c6d5f608d684e2c0dce8a828e0309a3bfe8327d9418a89f591c/marshmallow-2.18.0.tar.gz";
+      sha256 = "1g0aafpjn7yaxq06yndy8c7rs9n42adxkqq1ayhlr869pr06d3lm";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.mit ];
+    };
+  };
   "mistune" = super.buildPythonPackage {
     name = "mistune-0.8.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/2d/a4/509f6e7783ddd35482feda27bc7f72e65b5e7dc910eca4ab2164daf9c577/mistune-0.8.4.tar.gz";
       sha256 = "0vkmsh0x480rni51lhyvigfdf06b9247z868pk3bal1wnnfl58sr";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "mock" = super.buildPythonPackage {
     name = "mock-3.0.5";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."funcsigs"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/2e/ab/4fe657d78b270aa6a32f027849513b829b41b0f28d9d8d7f8c3d29ea559a/mock-3.0.5.tar.gz";
       sha256 = "1hrp6j0yrx2xzylfv02qa8kph661m6yq4p0mc8fnimch9j4psrc3";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "OSI Approved :: BSD License"; } ];
     };
   };
   "more-itertools" = super.buildPythonPackage {
     name = "more-itertools-5.0.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/dd/26/30fc0d541d9fdf55faf5ba4b0fd68f81d5bd2447579224820ad525934178/more-itertools-5.0.0.tar.gz";
       sha256 = "1r12cm6mcdwdzz7d47a6g4l437xsvapdlgyhqay3i2nrlv03da9q";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "msgpack-python" = super.buildPythonPackage {
     name = "msgpack-python-0.5.6";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/8a/20/6eca772d1a5830336f84aca1d8198e5a3f4715cd1c7fc36d3cc7f7185091/msgpack-python-0.5.6.tar.gz";
       sha256 = "16wh8qgybmfh4pjp8vfv78mdlkxfmcasg78lzlnm6nslsfkci31p";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "mysql-python" = super.buildPythonPackage {
     name = "mysql-python-1.2.5";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/a5/e9/51b544da85a36a68debe7a7091f068d802fc515a3a202652828c73453cad/MySQL-python-1.2.5.zip";
       sha256 = "0x0c2jg0bb3pp84njaqiic050qkyd7ymwhfvhipnimg58yv40441";
     };
     meta = {
       license = [ pkgs.lib.licenses.gpl1 ];
     };
   };
   "nbconvert" = super.buildPythonPackage {
     name = "nbconvert-5.3.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."mistune"
       self."jinja2"
       self."pygments"
       self."traitlets"
       self."jupyter-core"
       self."nbformat"
       self."entrypoints"
       self."bleach"
       self."pandocfilters"
       self."testpath"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/b9/a4/d0a0938ad6f5eeb4dea4e73d255c617ef94b0b2849d51194c9bbdb838412/nbconvert-5.3.1.tar.gz";
       sha256 = "1f9dkvpx186xjm4xab0qbph588mncp4vqk3fmxrsnqs43mks9c8j";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "nbformat" = super.buildPythonPackage {
     name = "nbformat-4.4.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."ipython-genutils"
       self."traitlets"
       self."jsonschema"
       self."jupyter-core"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/6e/0e/160754f7ae3e984863f585a3743b0ed1702043a81245907c8fae2d537155/nbformat-4.4.0.tar.gz";
       sha256 = "00nlf08h8yc4q73nphfvfhxrcnilaqanb8z0mdy6nxk0vzq4wjgp";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "packaging" = super.buildPythonPackage {
     name = "packaging-20.3";
     doCheck = false;
     propagatedBuildInputs = [
       self."pyparsing"
       self."six"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/65/37/83e3f492eb52d771e2820e88105f605335553fe10422cba9d256faeb1702/packaging-20.3.tar.gz";
       sha256 = "18xpablq278janh03bai9xd4kz9b0yfp6vflazn725ns9x3jna9w";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "BSD or Apache License, Version 2.0"; } pkgs.lib.licenses.asl20 ];
     };
   };
   "pandocfilters" = super.buildPythonPackage {
     name = "pandocfilters-1.4.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/4c/ea/236e2584af67bb6df960832731a6e5325fd4441de001767da328c33368ce/pandocfilters-1.4.2.tar.gz";
       sha256 = "1a8d9b7s48gmq9zj0pmbyv2sivn5i7m6mybgpkk4jm5vd7hp1pdk";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "paste" = super.buildPythonPackage {
     name = "paste-3.4.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/79/4a/45821b71dd40000507549afd1491546afad8279c0a87527c88776a794158/Paste-3.4.0.tar.gz";
       sha256 = "16sichvhyci1gaarkjs35mai8vphh7b244qm14hj1isw38nx4c03";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pastedeploy" = super.buildPythonPackage {
     name = "pastedeploy-2.1.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/c4/e9/972a1c20318b3ae9edcab11a6cef64308fbae5d0d45ab52c6f8b2b8f35b8/PasteDeploy-2.1.0.tar.gz";
       sha256 = "16qsq5y6mryslmbp5pn35x4z8z3ndp5rpgl42h226879nrw9hmg7";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pastescript" = super.buildPythonPackage {
     name = "pastescript-3.2.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."paste"
       self."pastedeploy"
       self."six"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ff/47/45c6f5a3cb8f5abf786fea98dbb8d02400a55768a9b623afb7df12346c61/PasteScript-3.2.0.tar.gz";
       sha256 = "1b3jq7xh383nvrrlblk05m37345bv97xrhx77wshllba3h7mq3wv";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pathlib2" = super.buildPythonPackage {
     name = "pathlib2-2.3.5";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."scandir"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/94/d8/65c86584e7e97ef824a1845c72bbe95d79f5b306364fa778a3c3e401b309/pathlib2-2.3.5.tar.gz";
       sha256 = "0s4qa8c082fdkb17izh4mfgwrjd1n5pya18wvrbwqdvvb5xs9nbc";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "peppercorn" = super.buildPythonPackage {
     name = "peppercorn-0.6";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/e4/77/93085de7108cdf1a0b092ff443872a8f9442c736d7ddebdf2f27627935f4/peppercorn-0.6.tar.gz";
       sha256 = "1ip4bfwcpwkq9hz2dai14k2cyabvwrnvcvrcmzxmqm04g8fnimwn";
     };
     meta = {
       license = [ { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "pexpect" = super.buildPythonPackage {
     name = "pexpect-4.8.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."ptyprocess"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/e5/9b/ff402e0e930e70467a7178abb7c128709a30dfb22d8777c043e501bc1b10/pexpect-4.8.0.tar.gz";
       sha256 = "032cg337h8awydgypz6f4wx848lw8dyrj4zy988x0lyib4ws8rgw";
     };
     meta = {
       license = [ pkgs.lib.licenses.isc { fullName = "ISC License (ISCL)"; } ];
     };
   };
   "pickleshare" = super.buildPythonPackage {
     name = "pickleshare-0.7.5";
     doCheck = false;
     propagatedBuildInputs = [
       self."pathlib2"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/d8/b6/df3c1c9b616e9c0edbc4fbab6ddd09df9535849c64ba51fcb6531c32d4d8/pickleshare-0.7.5.tar.gz";
       sha256 = "1jmghg3c53yp1i8cm6pcrm280ayi8621rwyav9fac7awjr3kss47";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "plaster" = super.buildPythonPackage {
     name = "plaster-1.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/37/e1/56d04382d718d32751017d32f351214384e529b794084eee20bb52405563/plaster-1.0.tar.gz";
       sha256 = "1hy8k0nv2mxq94y5aysk6hjk9ryb4bsd13g83m60hcyzxz3wflc3";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "plaster-pastedeploy" = super.buildPythonPackage {
     name = "plaster-pastedeploy-0.7";
     doCheck = false;
     propagatedBuildInputs = [
       self."pastedeploy"
       self."plaster"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/99/69/2d3bc33091249266a1bd3cf24499e40ab31d54dffb4a7d76fe647950b98c/plaster_pastedeploy-0.7.tar.gz";
       sha256 = "1zg7gcsvc1kzay1ry5p699rg2qavfsxqwl17mqxzr0gzw6j9679r";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pluggy" = super.buildPythonPackage {
     name = "pluggy-0.13.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."importlib-metadata"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/f8/04/7a8542bed4b16a65c2714bf76cf5a0b026157da7f75e87cc88774aa10b14/pluggy-0.13.1.tar.gz";
       sha256 = "1c35qyhvy27q9ih9n899f3h4sdnpgq027dbiilly2qb5cvgarchm";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "premailer" = super.buildPythonPackage {
     name = "premailer-3.6.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."lxml"
       self."cssselect"
       self."cssutils"
       self."requests"
       self."cachetools"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/62/da/2f43cdf9d3d79c80c4856a12389a1f257d65fe9ccc44bc6b4383c8a18e33/premailer-3.6.1.tar.gz";
       sha256 = "08pshx7a110k4ll20x0xhpvyn3kkipkrbgxjjn7ncdxs54ihdhgw";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl { fullName = "Python"; } ];
     };
   };
   "prompt-toolkit" = super.buildPythonPackage {
     name = "prompt-toolkit-1.0.18";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."wcwidth"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/c5/64/c170e5b1913b540bf0c8ab7676b21fdd1d25b65ddeb10025c6ca43cccd4c/prompt_toolkit-1.0.18.tar.gz";
       sha256 = "09h1153wgr5x2ny7ds0w2m81n3bb9j8hjb8sjfnrg506r01clkyx";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "psutil" = super.buildPythonPackage {
     name = "psutil-5.7.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/c4/b8/3512f0e93e0db23a71d82485ba256071ebef99b227351f0f5540f744af41/psutil-5.7.0.tar.gz";
       sha256 = "03jykdi3dgf1cdal9bv4fq9zjvzj9l9bs99gi5ar81sdl5nc2pk8";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "psycopg2" = super.buildPythonPackage {
     name = "psycopg2-2.8.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/84/d7/6a93c99b5ba4d4d22daa3928b983cec66df4536ca50b22ce5dcac65e4e71/psycopg2-2.8.4.tar.gz";
       sha256 = "1djvh98pi4hjd8rxbq8qzc63bg8v78k33yg6pl99wak61b6fb67q";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 { fullName = "GNU Library or Lesser General Public License (LGPL)"; } { fullName = "LGPL with exceptions or ZPL"; } ];
     };
   };
   "ptyprocess" = super.buildPythonPackage {
     name = "ptyprocess-0.6.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/7d/2d/e4b8733cf79b7309d84c9081a4ab558c89d8c89da5961bf4ddb050ca1ce0/ptyprocess-0.6.0.tar.gz";
       sha256 = "1h4lcd3w5nrxnsk436ar7fwkiy5rfn5wj2xwy9l0r4mdqnf2jgwj";
     };
     meta = {
       license = [  ];
     };
   };
   "py" = super.buildPythonPackage {
     name = "py-1.8.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/f1/5a/87ca5909f400a2de1561f1648883af74345fe96349f34f737cdfc94eba8c/py-1.8.0.tar.gz";
       sha256 = "0lsy1gajva083pzc7csj1cvbmminb7b4l6a0prdzyb3fd829nqyw";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "py-bcrypt" = super.buildPythonPackage {
     name = "py-bcrypt-0.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/68/b1/1c3068c5c4d2e35c48b38dcc865301ebfdf45f54507086ac65ced1fd3b3d/py-bcrypt-0.4.tar.gz";
       sha256 = "0y6smdggwi5s72v6p1nn53dg6w05hna3d264cq6kas0lap73p8az";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "py-gfm" = super.buildPythonPackage {
     name = "py-gfm-0.1.4";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
       self."markdown"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/06/ee/004a03a1d92bb386dae44f6dd087db541bc5093374f1637d4d4ae5596cc2/py-gfm-0.1.4.tar.gz";
       sha256 = "0zip06g2isivx8fzgqd4n9qzsa22c25jas1rsb7m2rnjg72m0rzg";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "pyasn1" = super.buildPythonPackage {
     name = "pyasn1-0.4.8";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/a4/db/fffec68299e6d7bad3d504147f9094830b704527a7fc098b721d38cc7fa7/pyasn1-0.4.8.tar.gz";
       sha256 = "1fnhbi3rmk47l9851gbik0flfr64vs5j0hbqx24cafjap6gprxxf";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "pyasn1-modules" = super.buildPythonPackage {
     name = "pyasn1-modules-0.2.6";
     doCheck = false;
     propagatedBuildInputs = [
       self."pyasn1"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/f1/a9/a1ef72a0e43feff643cf0130a08123dea76205e7a0dda37e3efb5f054a31/pyasn1-modules-0.2.6.tar.gz";
       sha256 = "08hph9j1r018drnrny29l7dl2q0cin78csswrhwrh8jmq61pmha3";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal pkgs.lib.licenses.bsd2 ];
     };
   };
   "pycparser" = super.buildPythonPackage {
     name = "pycparser-2.20";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/0f/86/e19659527668d70be91d0369aeaa055b4eb396b0f387a4f92293a20035bd/pycparser-2.20.tar.gz";
       sha256 = "1w0m3xvlrzq4lkbvd1ngfm8mdw64r1yxy6n7djlw6qj5d0km6ird";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "pycrypto" = super.buildPythonPackage {
     name = "pycrypto-2.6.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/60/db/645aa9af249f059cc3a368b118de33889219e0362141e75d4eaf6f80f163/pycrypto-2.6.1.tar.gz";
       sha256 = "0g0ayql5b9mkjam8hym6zyg6bv77lbh66rv1fyvgqb17kfc1xkpj";
     };
     meta = {
       license = [ pkgs.lib.licenses.publicDomain ];
     };
   };
   "pycurl" = super.buildPythonPackage {
     name = "pycurl-7.43.0.3";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ac/b3/0f3979633b7890bab6098d84c84467030b807a1e2b31f5d30103af5a71ca/pycurl-7.43.0.3.tar.gz";
       sha256 = "13nsvqhvnmnvfk75s8iynqsgszyv06cjp4drd3psi7zpbh63623g";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit { fullName = "LGPL/MIT"; } { fullName = "GNU Library or Lesser General Public License (LGPL)"; } ];
     };
   };
   "pygments" = super.buildPythonPackage {
     name = "pygments-2.4.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/7e/ae/26808275fc76bf2832deb10d3a3ed3107bc4de01b85dcccbe525f2cd6d1e/Pygments-2.4.2.tar.gz";
       sha256 = "15v2sqm5g12bqa0c7wikfh9ck2nl97ayizy1hpqhmws5gqalq748";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "pymysql" = super.buildPythonPackage {
     name = "pymysql-0.8.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/44/39/6bcb83cae0095a31b6be4511707fdf2009d3e29903a55a0494d3a9a2fac0/PyMySQL-0.8.1.tar.gz";
       sha256 = "0a96crz55bw4h6myh833skrli7b0ck89m3x673y2z2ryy7zrpq9l";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pyotp" = super.buildPythonPackage {
     name = "pyotp-2.3.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/f7/15/395c4945ea6bc37e8811280bb675615cb4c2b2c1cd70bdc43329da91a386/pyotp-2.3.0.tar.gz";
       sha256 = "18d13ikra1iq0xyfqfm72zhgwxi2qi9ps6z1a6zmqp4qrn57wlzw";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pyparsing" = super.buildPythonPackage {
     name = "pyparsing-2.4.7";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/c1/47/dfc9c342c9842bbe0036c7f763d2d6686bcf5eb1808ba3e170afdb282210/pyparsing-2.4.7.tar.gz";
       sha256 = "1hgc8qrbq1ymxbwfbjghv01fm3fbpjwpjwi0bcailxxzhf3yq0y2";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pyramid" = super.buildPythonPackage {
     name = "pyramid-1.10.4";
     doCheck = false;
     propagatedBuildInputs = [
       self."hupper"
       self."plaster"
       self."plaster-pastedeploy"
       self."setuptools"
       self."translationstring"
       self."venusian"
       self."webob"
       self."zope.deprecation"
       self."zope.interface"
       self."repoze.lru"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/c2/43/1ae701c9c6bb3a434358e678a5e72c96e8aa55cf4cb1d2fa2041b5dd38b7/pyramid-1.10.4.tar.gz";
       sha256 = "0rkxs1ajycg2zh1c94xlmls56mx5m161sn8112skj0amza6cn36q";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "pyramid-debugtoolbar" = super.buildPythonPackage {
     name = "pyramid-debugtoolbar-4.6.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."pyramid"
       self."pyramid-mako"
       self."repoze.lru"
       self."pygments"
       self."ipaddress"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/99/f6/b8603f82c18275be293921bc3a2184205056ca505747bf64ab8a0c08e124/pyramid_debugtoolbar-4.6.1.tar.gz";
       sha256 = "185z7q8n959ga5331iczwra2iljwkidfx4qn6bbd7vm3rm4w6llv";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "pyramid-jinja2" = super.buildPythonPackage {
     name = "pyramid-jinja2-2.7";
     doCheck = false;
     propagatedBuildInputs = [
       self."pyramid"
       self."zope.deprecation"
       self."jinja2"
       self."markupsafe"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/d8/80/d60a7233823de22ce77bd864a8a83736a1fe8b49884b08303a2e68b2c853/pyramid_jinja2-2.7.tar.gz";
       sha256 = "1sz5s0pp5jqhf4w22w9527yz8hgdi4mhr6apd6vw1gm5clghh8aw";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
+  "pyramid-apispec" = super.buildPythonPackage {
+    name = "pyramid-apispec-0.3.2";
+    doCheck = false;
+    propagatedBuildInputs = [
+      self."apispec"
+    ];
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/2a/30/1dea5d81ea635449572ba60ec3148310d75ae4530c3c695f54b0991bb8c7/pyramid_apispec-0.3.2.tar.gz";
+      sha256 = "0ffrcqp9dkykivhfcq0v9lgy6w0qhwl6x78925vfjmayly9r8da0";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.bsdOriginal ];
+    };
+  };
   "pyramid-mailer" = super.buildPythonPackage {
     name = "pyramid-mailer-0.15.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."pyramid"
       self."repoze.sendmail"
       self."transaction"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/a0/f2/6febf5459dff4d7e653314d575469ad2e11b9d2af2c3606360e1c67202f2/pyramid_mailer-0.15.1.tar.gz";
       sha256 = "16vg8jb203jgb7b0hd6wllfqvp542qh2ry1gjai2m6qpv5agy2pc";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "pyramid-mako" = super.buildPythonPackage {
     name = "pyramid-mako-1.1.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."pyramid"
       self."mako"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/63/7b/5e2af68f675071a6bad148c1c393928f0ef5fcd94e95cbf53b89d6471a83/pyramid_mako-1.1.0.tar.gz";
       sha256 = "1qj0m091mnii86j2q1d82yir22nha361rvhclvg3s70z8iiwhrh0";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "pysqlite" = super.buildPythonPackage {
     name = "pysqlite-2.8.3";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/42/02/981b6703e3c83c5b25a829c6e77aad059f9481b0bbacb47e6e8ca12bd731/pysqlite-2.8.3.tar.gz";
       sha256 = "1424gwq9sil2ffmnizk60q36vydkv8rxs6m7xs987kz8cdc37lqp";
     };
     meta = {
       license = [ { fullName = "zlib/libpng License"; } { fullName = "zlib/libpng license"; } ];
     };
   };
   "pytest" = super.buildPythonPackage {
     name = "pytest-4.6.5";
     doCheck = false;
     propagatedBuildInputs = [
       self."py"
       self."six"
       self."packaging"
       self."attrs"
       self."atomicwrites"
       self."pluggy"
       self."importlib-metadata"
       self."wcwidth"
       self."funcsigs"
       self."pathlib2"
       self."more-itertools"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/2a/c6/1d1f32f6a5009900521b12e6560fb6b7245b0d4bc3fb771acd63d10e30e1/pytest-4.6.5.tar.gz";
       sha256 = "0iykwwfp4h181nd7rsihh2120b0rkawlw7rvbl19sgfspncr3hwg";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pytest-cov" = super.buildPythonPackage {
     name = "pytest-cov-2.7.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."pytest"
       self."coverage"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/bb/0f/3db7ff86801883b21d5353b258c994b1b8e2abbc804e2273b8d0fd19004b/pytest-cov-2.7.1.tar.gz";
       sha256 = "0filvmmyqm715azsl09ql8hy2x7h286n6d8z5x42a1wpvvys83p0";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal pkgs.lib.licenses.mit ];
     };
   };
   "pytest-profiling" = super.buildPythonPackage {
     name = "pytest-profiling-1.7.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."pytest"
       self."gprof2dot"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/39/70/22a4b33739f07f1732a63e33bbfbf68e0fa58cfba9d200e76d01921eddbf/pytest-profiling-1.7.0.tar.gz";
       sha256 = "0abz9gi26jpcfdzgsvwad91555lpgdc8kbymicmms8k2fqa8z4wk";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pytest-runner" = super.buildPythonPackage {
     name = "pytest-runner-5.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/d9/6d/4b41a74b31720e25abd4799be72d54811da4b4d0233e38b75864dcc1f7ad/pytest-runner-5.1.tar.gz";
       sha256 = "0ykfcnpp8c22winj63qzc07l5axwlc9ikl8vn05sc32gv3417815";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pytest-sugar" = super.buildPythonPackage {
     name = "pytest-sugar-0.9.2";
     doCheck = false;
     propagatedBuildInputs = [
       self."pytest"
       self."termcolor"
       self."packaging"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/55/59/f02f78d1c80f7e03e23177f60624c8106d4f23d124c921df103f65692464/pytest-sugar-0.9.2.tar.gz";
       sha256 = "1asq7yc4g8bx2sn7yy974mhc9ywvaihasjab4inkirdwn9s7mn7w";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "pytest-timeout" = super.buildPythonPackage {
     name = "pytest-timeout-1.3.3";
     doCheck = false;
     propagatedBuildInputs = [
       self."pytest"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/13/48/7a166eaa29c1dca6cc253e3ba5773ff2e4aa4f567c1ea3905808e95ac5c1/pytest-timeout-1.3.3.tar.gz";
       sha256 = "1cczcjhw4xx5sjkhxlhc5c1bkr7x6fcyx12wrnvwfckshdvblc2a";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit { fullName = "DFSG approved"; } ];
     };
   };
   "python-dateutil" = super.buildPythonPackage {
     name = "python-dateutil-2.8.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/be/ed/5bbc91f03fa4c839c4c7360375da77f9659af5f7086b7a7bdda65771c8e0/python-dateutil-2.8.1.tar.gz";
       sha256 = "0g42w7k5007iv9dam6gnja2ry8ydwirh99mgdll35s12pyfzxsvk";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal pkgs.lib.licenses.asl20 { fullName = "Dual License"; } ];
     };
   };
   "python-editor" = super.buildPythonPackage {
     name = "python-editor-1.0.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/0a/85/78f4a216d28343a67b7397c99825cff336330893f00601443f7c7b2f2234/python-editor-1.0.4.tar.gz";
       sha256 = "0yrjh8w72ivqxi4i7xsg5b1vz15x8fg51xra7c3bgfyxqnyadzai";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 { fullName = "Apache"; } ];
     };
   };
   "python-ldap" = super.buildPythonPackage {
     name = "python-ldap-3.2.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."pyasn1"
       self."pyasn1-modules"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ea/93/596f875e003c770447f4b99267820a0c769dd2dc3ae3ed19afe460fcbad0/python-ldap-3.2.0.tar.gz";
       sha256 = "13nvrhp85yr0jyxixcjj012iw8l9wynxxlykm9j3alss6waln73x";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   "python-memcached" = super.buildPythonPackage {
     name = "python-memcached-1.59";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/90/59/5faf6e3cd8a568dd4f737ddae4f2e54204fd8c51f90bf8df99aca6c22318/python-memcached-1.59.tar.gz";
       sha256 = "0kvyapavbirk2x3n1jx4yb9nyigrj1s3x15nm3qhpvhkpqvqdqm2";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   "python-pam" = super.buildPythonPackage {
     name = "python-pam-1.8.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/01/16/544d01cae9f28e0292dbd092b6b8b0bf222b528f362ee768a5bed2140111/python-pam-1.8.4.tar.gz";
       sha256 = "16whhc0vr7gxsbzvsnq65nq8fs3wwmx755cavm8kkczdkz4djmn8";
     };
     meta = {
       license = [ { fullName = "License :: OSI Approved :: MIT License"; } pkgs.lib.licenses.mit ];
     };
   };
   "python-saml" = super.buildPythonPackage {
     name = "python-saml-2.4.2";
     doCheck = false;
     propagatedBuildInputs = [
       self."dm.xmlsec.binding"
       self."isodate"
       self."defusedxml"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/79/a8/a6611017e0883102fd5e2b73c9d90691b8134e38247c04ee1531d3dc647c/python-saml-2.4.2.tar.gz";
       sha256 = "0dls4hwvf13yg7x5yfjrghbywg8g38vn5vr0rsf70hli3ydbfm43";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pytz" = super.buildPythonPackage {
     name = "pytz-2019.3";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/82/c3/534ddba230bd4fbbd3b7a3d35f3341d014cca213f369a9940925e7e5f691/pytz-2019.3.tar.gz";
       sha256 = "1ghrk1wg45d3nymj7bf4zj03n3bh64xmczhk4pfi577hdkdhcb5h";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pyzmq" = super.buildPythonPackage {
     name = "pyzmq-14.6.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/8a/3b/5463d5a9d712cd8bbdac335daece0d69f6a6792da4e3dd89956c0db4e4e6/pyzmq-14.6.0.tar.gz";
       sha256 = "1frmbjykvhmdg64g7sn20c9fpamrsfxwci1nhhg8q7jgz5pq0ikp";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "LGPL+BSD"; } { fullName = "GNU Library or Lesser General Public License (LGPL)"; } ];
     };
   };
+  "PyYAML" = super.buildPythonPackage {
+    name = "PyYAML-5.3.1";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/64/c2/b80047c7ac2478f9501676c988a5411ed5572f35d1beff9cae07d321512c/PyYAML-5.3.1.tar.gz";
+      sha256 = "0pb4zvkfxfijkpgd1b86xjsqql97ssf1knbd1v53wkg1qm9cgsmq";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.mit ];
+    };
+  };
   "redis" = super.buildPythonPackage {
     name = "redis-3.4.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ef/2e/2c0f59891db7db087a7eeaa79bc7c7f2c039e71a2b5b0a41391e9d462926/redis-3.4.1.tar.gz";
       sha256 = "07yaj0j9fs7xdkg5bg926fa990khyigjbp31si8ai20vj8sv7kqd";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "repoze.lru" = super.buildPythonPackage {
     name = "repoze.lru-0.7";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/12/bc/595a77c4b5e204847fdf19268314ef59c85193a9dc9f83630fc459c0fee5/repoze.lru-0.7.tar.gz";
       sha256 = "0xzz1aw2smy8hdszrq8yhnklx6w1r1mf55061kalw3iq35gafa84";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "repoze.sendmail" = super.buildPythonPackage {
     name = "repoze.sendmail-4.4.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
       self."zope.interface"
       self."transaction"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/12/4e/8ef1fd5c42765d712427b9c391419a77bd48877886d2cbc5e9f23c8cad9b/repoze.sendmail-4.4.1.tar.gz";
       sha256 = "096ln02jr2afk7ab9j2czxqv2ryqq7m86ah572nqplx52iws73ks";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "requests" = super.buildPythonPackage {
     name = "requests-2.22.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."chardet"
       self."idna"
       self."urllib3"
       self."certifi"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/01/62/ddcf76d1d19885e8579acb1b1df26a852b03472c0e46d2b959a714c90608/requests-2.22.0.tar.gz";
       sha256 = "1d5ybh11jr5sm7xp6mz8fyc7vrp4syifds91m7sj60xalal0gq0i";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "rhodecode-enterprise-ce" = super.buildPythonPackage {
-    name = "rhodecode-enterprise-ce-4.20.1";
+    name = "rhodecode-enterprise-ce-4.20.0";
     buildInputs = [
       self."pytest"
       self."py"
       self."pytest-cov"
       self."pytest-sugar"
       self."pytest-runner"
       self."pytest-profiling"
       self."pytest-timeout"
       self."gprof2dot"
       self."mock"
       self."cov-core"
       self."coverage"
       self."webtest"
       self."beautifulsoup4"
       self."configobj"
     ];
     doCheck = true;
     propagatedBuildInputs = [
       self."amqp"
       self."babel"
       self."beaker"
       self."bleach"
       self."celery"
       self."channelstream"
       self."click"
       self."colander"
       self."configobj"
       self."cssselect"
       self."cryptography"
       self."decorator"
       self."deform"
       self."docutils"
       self."dogpile.cache"
       self."dogpile.core"
       self."formencode"
       self."future"
       self."futures"
       self."infrae.cache"
       self."iso8601"
       self."itsdangerous"
       self."kombu"
       self."lxml"
       self."mako"
       self."markdown"
       self."markupsafe"
       self."msgpack-python"
       self."pyotp"
       self."packaging"
       self."pathlib2"
       self."paste"
       self."pastedeploy"
       self."pastescript"
       self."peppercorn"
       self."premailer"
       self."psutil"
       self."py-bcrypt"
       self."pycurl"
       self."pycrypto"
       self."pygments"
       self."pyparsing"
       self."pyramid-debugtoolbar"
       self."pyramid-mako"
       self."pyramid"
       self."pyramid-mailer"
       self."python-dateutil"
       self."python-ldap"
       self."python-memcached"
       self."python-pam"
       self."python-saml"
       self."pytz"
       self."tzlocal"
       self."pyzmq"
       self."py-gfm"
       self."redis"
       self."repoze.lru"
       self."requests"
       self."routes"
       self."simplejson"
       self."six"
       self."sqlalchemy"
       self."sshpubkeys"
       self."subprocess32"
       self."supervisor"
       self."translationstring"
       self."urllib3"
       self."urlobject"
       self."venusian"
       self."weberror"
       self."webhelpers2"
       self."webob"
       self."whoosh"
       self."wsgiref"
       self."zope.cachedescriptors"
       self."zope.deprecation"
       self."zope.event"
       self."zope.interface"
       self."mysql-python"
       self."pymysql"
       self."pysqlite"
       self."psycopg2"
       self."nbconvert"
       self."nbformat"
       self."jupyter-client"
       self."jupyter-core"
       self."alembic"
       self."invoke"
       self."bumpversion"
       self."gevent"
       self."greenlet"
       self."gunicorn"
       self."waitress"
       self."ipdb"
       self."ipython"
       self."rhodecode-tools"
       self."appenlight-client"
       self."pytest"
       self."py"
       self."pytest-cov"
       self."pytest-sugar"
       self."pytest-runner"
       self."pytest-profiling"
       self."pytest-timeout"
       self."gprof2dot"
       self."mock"
       self."cov-core"
       self."coverage"
       self."webtest"
       self."beautifulsoup4"
     ];
     src = ./.;
     meta = {
       license = [ { fullName = "Affero GNU General Public License v3 or later (AGPLv3+)"; } { fullName = "AGPLv3, and Commercial License"; } ];
     };
   };
   "rhodecode-tools" = super.buildPythonPackage {
     name = "rhodecode-tools-1.4.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."click"
       self."future"
       self."six"
       self."mako"
       self."markupsafe"
       self."requests"
       self."urllib3"
       self."whoosh"
       self."elasticsearch"
       self."elasticsearch-dsl"
       self."elasticsearch2"
       self."elasticsearch1-dsl"
     ];
     src = fetchurl {
       url = "https://code.rhodecode.com/rhodecode-tools-ce/artifacts/download/0-ed54e749-2ef5-4bc7-ae7f-7900e3c2aa15.tar.gz?sha256=76f024bad3a1e55fdb3d64f13f5b77ff21a12fee699918de2110fe21effd5a3a";
       sha256 = "0fjszppj3zhh47g1i6b9xqps28gzfxdkzwb47pdmzrd1sfx29w3n";
     };
     meta = {
       license = [ { fullName = "Apache 2.0 and Proprietary"; } ];
     };
   };
   "routes" = super.buildPythonPackage {
     name = "routes-2.4.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."repoze.lru"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/33/38/ea827837e68d9c7dde4cff7ec122a93c319f0effc08ce92a17095576603f/Routes-2.4.1.tar.gz";
       sha256 = "1zamff3m0kc4vyfniyhxpkkcqv1rrgnmh37ykxv34nna1ws47vi6";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "scandir" = super.buildPythonPackage {
     name = "scandir-1.10.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/df/f5/9c052db7bd54d0cbf1bc0bb6554362bba1012d03e5888950a4f5c5dadc4e/scandir-1.10.0.tar.gz";
       sha256 = "1bkqwmf056pkchf05ywbnf659wqlp6lljcdb0y88wr9f0vv32ijd";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "New BSD License"; } ];
     };
   };
   "setproctitle" = super.buildPythonPackage {
     name = "setproctitle-1.1.10";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/5a/0d/dc0d2234aacba6cf1a729964383e3452c52096dc695581248b548786f2b3/setproctitle-1.1.10.tar.gz";
       sha256 = "163kplw9dcrw0lffq1bvli5yws3rngpnvrxrzdw89pbphjjvg0v2";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "setuptools" = super.buildPythonPackage {
     name = "setuptools-44.1.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ed/7b/bbf89ca71e722b7f9464ebffe4b5ee20a9e5c9a555a56e2d3914bb9119a6/setuptools-44.1.0.zip";
       sha256 = "1jja896zvd1ppccnjbhkgagxbwchgq6vfamp6qn1hvywq6q9cjkr";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "simplegeneric" = super.buildPythonPackage {
     name = "simplegeneric-0.8.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/3d/57/4d9c9e3ae9a255cd4e1106bb57e24056d3d0709fc01b2e3e345898e49d5b/simplegeneric-0.8.1.zip";
       sha256 = "0wwi1c6md4vkbcsfsf8dklf3vr4mcdj4mpxkanwgb6jb1432x5yw";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "simplejson" = super.buildPythonPackage {
     name = "simplejson-3.16.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/e3/24/c35fb1c1c315fc0fffe61ea00d3f88e85469004713dab488dee4f35b0aff/simplejson-3.16.0.tar.gz";
       sha256 = "19cws1syk8jzq2pw43878dv6fjkb0ifvjpx0i9aajix6kc9jkwxi";
     };
     meta = {
       license = [ { fullName = "Academic Free License (AFL)"; } pkgs.lib.licenses.mit ];
     };
   };
   "six" = super.buildPythonPackage {
     name = "six-1.11.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/16/d8/bc6316cf98419719bd59c91742194c111b6f2e85abac88e496adefaf7afe/six-1.11.0.tar.gz";
       sha256 = "1scqzwc51c875z23phj48gircqjgnn3af8zy2izjwmnlxrxsgs3h";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "sqlalchemy" = super.buildPythonPackage {
     name = "sqlalchemy-1.3.15";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/8c/30/4134e726dd5ed13728ff814fa91fc01c447ad8700504653fe99d91fdd34b/SQLAlchemy-1.3.15.tar.gz";
       sha256 = "0iglkvymfp35zm5pxy5kzqvcv96kkas0chqdx7xpla86sspa9k64";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "sshpubkeys" = super.buildPythonPackage {
     name = "sshpubkeys-3.1.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."cryptography"
       self."ecdsa"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/00/23/f7508a12007c96861c3da811992f14283d79c819d71a217b3e12d5196649/sshpubkeys-3.1.0.tar.gz";
       sha256 = "105g2li04nm1hb15a2y6hm9m9k7fbrkd5l3gy12w3kgcmsf3k25k";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "subprocess32" = super.buildPythonPackage {
     name = "subprocess32-3.5.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/32/c8/564be4d12629b912ea431f1a50eb8b3b9d00f1a0b1ceff17f266be190007/subprocess32-3.5.4.tar.gz";
       sha256 = "17f7mvwx2271s1wrl0qac3wjqqnrqag866zs3qc8v5wp0k43fagb";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   "supervisor" = super.buildPythonPackage {
     name = "supervisor-4.1.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/de/87/ee1ad8fa533a4b5f2c7623f4a2b585d3c1947af7bed8e65bc7772274320e/supervisor-4.1.0.tar.gz";
       sha256 = "10q36sa1jqljyyyl7cif52akpygl5kmlqq9x91hmx53f8zh6zj1d";
     };
     meta = {
       license = [ { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "tempita" = super.buildPythonPackage {
     name = "tempita-0.5.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/56/c8/8ed6eee83dbddf7b0fc64dd5d4454bc05e6ccaafff47991f73f2894d9ff4/Tempita-0.5.2.tar.gz";
       sha256 = "177wwq45slfyajd8csy477bmdmzipyw0dm7i85k3akb7m85wzkna";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "termcolor" = super.buildPythonPackage {
     name = "termcolor-1.1.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/8a/48/a76be51647d0eb9f10e2a4511bf3ffb8cc1e6b14e9e4fab46173aa79f981/termcolor-1.1.0.tar.gz";
       sha256 = "0fv1vq14rpqwgazxg4981904lfyp84mnammw7y046491cv76jv8x";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "testpath" = super.buildPythonPackage {
     name = "testpath-0.4.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/2c/b3/5d57205e896d8998d77ad12aa42ebce75cd97d8b9a97d00ba078c4c9ffeb/testpath-0.4.4.tar.gz";
       sha256 = "0zpcmq22dz79ipvvsfnw1ykpjcaj6xyzy7ws77s5b5ql3hka7q30";
     };
     meta = {
       license = [  ];
     };
   };
   "traitlets" = super.buildPythonPackage {
     name = "traitlets-4.3.3";
     doCheck = false;
     propagatedBuildInputs = [
       self."ipython-genutils"
       self."six"
       self."decorator"
       self."enum34"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/75/b0/43deb021bc943f18f07cbe3dac1d681626a48997b7ffa1e7fb14ef922b21/traitlets-4.3.3.tar.gz";
       sha256 = "1xsrwgivpkxlbr4dfndfsi098s29yqgswgjc1qqn69yxklvfw8yh";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "transaction" = super.buildPythonPackage {
     name = "transaction-2.4.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."zope.interface"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/9d/7d/0e8af0d059e052b9dcf2bb5a08aad20ae3e238746bdd3f8701a60969b363/transaction-2.4.0.tar.gz";
       sha256 = "17wz1y524ca07vr03yddy8dv0gbscs06dbdywmllxv5rc725jq3j";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "translationstring" = super.buildPythonPackage {
     name = "translationstring-1.3";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/5e/eb/bee578cc150b44c653b63f5ebe258b5d0d812ddac12497e5f80fcad5d0b4/translationstring-1.3.tar.gz";
       sha256 = "0bdpcnd9pv0131dl08h4zbcwmgc45lyvq3pa224xwan5b3x4rr2f";
     };
     meta = {
       license = [ { fullName = "BSD-like (http://repoze.org/license.html)"; } ];
     };
   };
   "tzlocal" = super.buildPythonPackage {
     name = "tzlocal-1.5.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."pytz"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/cb/89/e3687d3ed99bc882793f82634e9824e62499fdfdc4b1ae39e211c5b05017/tzlocal-1.5.1.tar.gz";
       sha256 = "0kiciwiqx0bv0fbc913idxibc4ygg4cb7f8rcpd9ij2shi4bigjf";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "urllib3" = super.buildPythonPackage {
     name = "urllib3-1.25.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/9a/8b/ea6d2beb2da6e331e9857d0a60b79ed4f72dcbc4e2c7f2d2521b0480fda2/urllib3-1.25.2.tar.gz";
       sha256 = "1nq2k4pss1ihsjh02r41sqpjpm5rfqkjfysyq7g7n2i1p7c66c55";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "urlobject" = super.buildPythonPackage {
     name = "urlobject-2.4.3";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/e2/b8/1d0a916f4b34c4618846e6da0e4eeaa8fcb4a2f39e006434fe38acb74b34/URLObject-2.4.3.tar.gz";
       sha256 = "1ahc8ficzfvr2avln71immfh4ls0zyv6cdaa5xmkdj5rd87f5cj7";
     };
     meta = {
       license = [ pkgs.lib.licenses.publicDomain ];
     };
   };
   "venusian" = super.buildPythonPackage {
     name = "venusian-1.2.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/7e/6f/40a9d43ac77cb51cb62be5b5662d170f43f8037bdc4eab56336c4ca92bb7/venusian-1.2.0.tar.gz";
       sha256 = "0ghyx66g8ikx9nx1mnwqvdcqm11i1vlq0hnvwl50s48bp22q5v34";
     };
     meta = {
       license = [ { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "vine" = super.buildPythonPackage {
     name = "vine-1.3.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/1c/e1/79fb8046e607dd6c2ad05c9b8ebac9d0bd31d086a08f02699e96fc5b3046/vine-1.3.0.tar.gz";
       sha256 = "11ydsbhl1vabndc2r979dv61s6j2b0giq6dgvryifvq1m7bycghk";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "waitress" = super.buildPythonPackage {
     name = "waitress-1.3.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/a6/e6/708da7bba65898e5d759ade8391b1077e49d07be0b0223c39f5be04def56/waitress-1.3.1.tar.gz";
       sha256 = "1iysl8ka3l4cdrr0r19fh1cv28q41mwpvgsb81ji7k4shkb0k3i7";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "wcwidth" = super.buildPythonPackage {
     name = "wcwidth-0.1.9";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/25/9d/0acbed6e4a4be4fc99148f275488580968f44ddb5e69b8ceb53fc9df55a0/wcwidth-0.1.9.tar.gz";
       sha256 = "1wf5ycjx8s066rdvr0fgz4xds9a8zhs91c4jzxvvymm1c8l8cwzf";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "webencodings" = super.buildPythonPackage {
     name = "webencodings-0.5.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/0b/02/ae6ceac1baeda530866a85075641cec12989bd8d31af6d5ab4a3e8c92f47/webencodings-0.5.1.tar.gz";
       sha256 = "08qrgrc4hrximb2gqnl69g01s93rhf2842jfxdjljc1dbwj1qsmk";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "weberror" = super.buildPythonPackage {
     name = "weberror-0.13.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."webob"
       self."tempita"
       self."pygments"
       self."paste"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/07/0a/09ca5eb0fab5c0d17b380026babe81c96ecebb13f2b06c3203432dd7be72/WebError-0.13.1.tar.gz";
       sha256 = "0r4qvnf2r92gfnpa1kwygh4j2x6j3axg2i4an6hyxwg2gpaqp7y1";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "webhelpers2" = super.buildPythonPackage {
     name = "webhelpers2-2.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."markupsafe"
       self."six"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ff/30/56342c6ea522439e3662427c8d7b5e5b390dff4ff2dc92d8afcb8ab68b75/WebHelpers2-2.0.tar.gz";
       sha256 = "0aphva1qmxh83n01p53f5fd43m4srzbnfbz5ajvbx9aj2aipwmcs";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "webob" = super.buildPythonPackage {
     name = "webob-1.8.5";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/9d/1a/0c89c070ee2829c934cb6c7082287c822e28236a4fcf90063e6be7c35532/WebOb-1.8.5.tar.gz";
       sha256 = "11khpzaxc88q31v25ic330gsf56fwmbdc9b30br8mvp0fmwspah5";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "webtest" = super.buildPythonPackage {
     name = "webtest-2.0.34";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."webob"
       self."waitress"
       self."beautifulsoup4"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/2c/74/a0e63feee438735d628631e2b70d82280276a930637ac535479e5fad9427/WebTest-2.0.34.tar.gz";
       sha256 = "0x1y2c8z4fmpsny4hbp6ka37si2g10r5r2jwxhvv5mx7g3blq4bi";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "whoosh" = super.buildPythonPackage {
     name = "whoosh-2.7.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/25/2b/6beed2107b148edc1321da0d489afc4617b9ed317ef7b72d4993cad9b684/Whoosh-2.7.4.tar.gz";
       sha256 = "10qsqdjpbc85fykc1vgcs8xwbgn4l2l52c8d83xf1q59pwyn79bw";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal pkgs.lib.licenses.bsd2 ];
     };
   };
   "ws4py" = super.buildPythonPackage {
     name = "ws4py-0.5.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/53/20/4019a739b2eefe9282d3822ef6a225250af964b117356971bd55e274193c/ws4py-0.5.1.tar.gz";
       sha256 = "10slbbf2jm4hpr92jx7kh7mhf48sjl01v2w4d8z3f1p0ybbp7l19";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "wsgiref" = super.buildPythonPackage {
     name = "wsgiref-0.1.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/41/9e/309259ce8dff8c596e8c26df86dbc4e848b9249fd36797fd60be456f03fc/wsgiref-0.1.2.zip";
       sha256 = "0y8fyjmpq7vwwm4x732w97qbkw78rjwal5409k04cw4m03411rn7";
     };
     meta = {
       license = [ { fullName = "PSF or ZPL"; } ];
     };
   };
   "zipp" = super.buildPythonPackage {
     name = "zipp-1.2.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."contextlib2"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/78/08/d52f0ea643bc1068d6dc98b412f4966a9b63255d20911a23ac3220c033c4/zipp-1.2.0.tar.gz";
       sha256 = "1c91lnv1bxjimh8as27hz7bghsjkkbxn1d37xq7in9c82iai0167";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "zope.cachedescriptors" = super.buildPythonPackage {
     name = "zope.cachedescriptors-4.3.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/2f/89/ebe1890cc6d3291ebc935558fa764d5fffe571018dbbee200e9db78762cb/zope.cachedescriptors-4.3.1.tar.gz";
       sha256 = "0jhr3m5p74c6r7k8iv0005b8bfsialih9d7zl5vx38rf5xq1lk8z";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "zope.deprecation" = super.buildPythonPackage {
     name = "zope.deprecation-4.4.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/34/da/46e92d32d545dd067b9436279d84c339e8b16de2ca393d7b892bc1e1e9fd/zope.deprecation-4.4.0.tar.gz";
       sha256 = "1pz2cv7gv9y1r3m0bdv7ks1alagmrn5msm5spwdzkb2by0w36i8d";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "zope.event" = super.buildPythonPackage {
     name = "zope.event-4.4";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/4c/b2/51c0369adcf5be2334280eed230192ab3b03f81f8efda9ddea6f65cc7b32/zope.event-4.4.tar.gz";
       sha256 = "1ksbc726av9xacml6jhcfyn828hlhb9xlddpx6fcvnlvmpmpvhk9";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "zope.interface" = super.buildPythonPackage {
     name = "zope.interface-4.6.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/4e/d0/c9d16bd5b38de44a20c6dc5d5ed80a49626fafcb3db9f9efdc2a19026db6/zope.interface-4.6.0.tar.gz";
       sha256 = "1rgh2x3rcl9r0v0499kf78xy86rnmanajf4ywmqb943wpk50sg8v";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
 
 ### Test requirements
 
   
 }

requirements.txt

@@ -1,123 +1,123 b''
 ## dependencies
 
 amqp==2.5.2
 babel==1.3
 beaker==1.9.1
 bleach==3.1.3
 celery==4.3.0
-channelstream==0.5.2
+channelstream==0.6.14
 click==7.0
 colander==1.7.0
 # our custom configobj
 https://code.rhodecode.com/upstream/configobj/artifacts/download/0-012de99a-b1e1-4f64-a5c0-07a98a41b324.tar.gz?md5=6a513f51fe04b2c18cf84c1395a7c626#egg=configobj==5.0.6
 cssselect==1.0.3
 cryptography==2.6.1
 decorator==4.1.2
 deform==2.0.8
 docutils==0.16.0
 dogpile.cache==0.9.0
 dogpile.core==0.4.1
 formencode==1.2.4
 future==0.14.3
 futures==3.0.2
 infrae.cache==1.0.1
 iso8601==0.1.12
-itsdangerous==0.24
+itsdangerous==1.1.0
 kombu==4.6.6
 lxml==4.2.5
 mako==1.1.0
 markdown==2.6.11
 markupsafe==1.1.1
 msgpack-python==0.5.6
 pyotp==2.3.0
 packaging==20.3
 pathlib2==2.3.5
 paste==3.4.0
 pastedeploy==2.1.0
 pastescript==3.2.0
 peppercorn==0.6
 premailer==3.6.1
 psutil==5.7.0
 py-bcrypt==0.4
 pycurl==7.43.0.3
 pycrypto==2.6.1
 pygments==2.4.2
 pyparsing==2.4.7
 pyramid-debugtoolbar==4.6.1
 pyramid-mako==1.1.0
 pyramid==1.10.4
 pyramid_mailer==0.15.1
 python-dateutil==2.8.1
 python-ldap==3.2.0
 python-memcached==1.59
 python-pam==1.8.4
 python-saml==2.4.2
 pytz==2019.3
 tzlocal==1.5.1
 pyzmq==14.6.0
 py-gfm==0.1.4
 redis==3.4.1
 repoze.lru==0.7
 requests==2.22.0
 routes==2.4.1
 simplejson==3.16.0
 six==1.11.0
 sqlalchemy==1.3.15
 sshpubkeys==3.1.0
 subprocess32==3.5.4
 supervisor==4.1.0
 translationstring==1.3
 urllib3==1.25.2
 urlobject==2.4.3
 venusian==1.2.0
 weberror==0.13.1
 webhelpers2==2.0
 webob==1.8.5
 whoosh==2.7.4
 wsgiref==0.1.2
 zope.cachedescriptors==4.3.1
 zope.deprecation==4.4.0
 zope.event==4.4.0
 zope.interface==4.6.0
 
 # DB drivers
 mysql-python==1.2.5
 pymysql==0.8.1
 pysqlite==2.8.3
 psycopg2==2.8.4
 
 # IPYTHON RENDERING
 # entrypoints backport, pypi version doesn't support egg installs
 https://code.rhodecode.com/upstream/entrypoints/artifacts/download/0-8e9ee9e4-c4db-409c-b07e-81568fd1832d.tar.gz?md5=3a027b8ff1d257b91fe257de6c43357d#egg=entrypoints==0.2.2.rhodecode-upstream1
 nbconvert==5.3.1
 nbformat==4.4.0
 jupyter-client==5.0.0
 jupyter-core==4.5.0
 
 ## cli tools
 alembic==1.4.2
 invoke==0.13.0
 bumpversion==0.5.3
 
 ## http servers
 gevent==1.5.0
 greenlet==0.4.15
 gunicorn==19.9.0
 waitress==1.3.1
 
 ## debug
 ipdb==0.13.2
 ipython==5.1.0
 
 ## rhodecode-tools, special case, use file://PATH.tar.gz#egg=rhodecode-tools==X.Y.Z, to test local version
 https://code.rhodecode.com/rhodecode-tools-ce/artifacts/download/0-ed54e749-2ef5-4bc7-ae7f-7900e3c2aa15.tar.gz?sha256=76f024bad3a1e55fdb3d64f13f5b77ff21a12fee699918de2110fe21effd5a3a#egg=rhodecode-tools==1.4.0
 
 
 ## appenlight
 appenlight-client==0.6.26
 
 ## test related requirements
 -r requirements_test.txt
 
 ## uncomment to add the debug libraries
 #-r requirements_debug.txt

requirements_pinned.txt

@@ -1,27 +1,28 b''
 # contains not directly required libraries we want to pin the version.
 
 atomicwrites==1.3.0
 attrs==19.3.0
 asn1crypto==0.24.0
 billiard==3.6.1.0
 cffi==1.12.3
 chameleon==2.24
 configparser==4.0.2
 contextlib2==0.6.0.post1
 ecdsa==0.13.2
 gnureadline==6.3.8
 hupper==1.10.2
 ipaddress==1.0.23
 importlib-metadata==1.6.0
 jinja2==2.9.6
 jsonschema==2.6.0
 pluggy==0.13.1
 pyasn1-modules==0.2.6
 pyramid-jinja2==2.7
+pyramid-apispec==0.3.2
 scandir==1.10.0
 setproctitle==1.1.10
 tempita==0.5.2
 testpath==0.4.4
 transaction==2.4.0
 vine==1.3.0
-wcwidth==0.1.9
+wcwidth==0.1.9
  No newline at end of file

rhodecode/VERSION

@@ -1,1 +1,1 b''
-4.20.1
  No newline at end of file
+4.21.0
  No newline at end of file

rhodecode/__init__.py

@@ -1,60 +1,60 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 from collections import OrderedDict
 
 import sys
 import platform
 
 VERSION = tuple(open(os.path.join(
     os.path.dirname(__file__), 'VERSION')).read().split('.'))
 
 BACKENDS = OrderedDict()
 
 BACKENDS['hg'] = 'Mercurial repository'
 BACKENDS['git'] = 'Git repository'
 BACKENDS['svn'] = 'Subversion repository'
 
 
 CELERY_ENABLED = False
 CELERY_EAGER = False
 
 # link to config for pyramid
 CONFIG = {}
 
 # Populated with the settings dictionary from application init in
 # rhodecode.conf.environment.load_pyramid_environment
 PYRAMID_SETTINGS = {}
 
 # Linked module for extensions
 EXTENSIONS = {}
 
 __version__ = ('.'.join((str(each) for each in VERSION[:3])))
-__dbversion__ = 108  # defines current db version for migrations
+__dbversion__ = 109  # defines current db version for migrations
 __platform__ = platform.system()
 __license__ = 'AGPLv3, and Commercial License'
 __author__ = 'RhodeCode GmbH'
 __url__ = 'https://code.rhodecode.com'
 
 is_windows = __platform__ in ['Windows']
 is_unix = not is_windows
 is_test = False
 disable_error_handler = False

rhodecode/api/utils.py

@@ -1,453 +1,452 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 JSON RPC utils
 """
 
 import collections
 import logging
 
 from rhodecode.api.exc import JSONRPCError
 from rhodecode.lib.auth import (
     HasPermissionAnyApi, HasRepoPermissionAnyApi, HasRepoGroupPermissionAnyApi)
 from rhodecode.lib.utils import safe_unicode
 from rhodecode.lib.vcs.exceptions import RepositoryError
 from rhodecode.lib.view_utils import get_commit_from_ref_name
 from rhodecode.lib.utils2 import str2bool
 
 log = logging.getLogger(__name__)
 
 
 class OAttr(object):
     """
     Special Option that defines other attribute, and can default to them
 
     Example::
 
         def test(apiuser, userid=Optional(OAttr('apiuser')):
             user = Optional.extract(userid, evaluate_locals=local())
             #if we pass in userid, we get it, else it will default to apiuser
             #attribute
     """
 
     def __init__(self, attr_name):
         self.attr_name = attr_name
 
     def __repr__(self):
         return '<OptionalAttr:%s>' % self.attr_name
 
     def __call__(self):
         return self
 
 
 class Optional(object):
     """
     Defines an optional parameter::
 
         param = param.getval() if isinstance(param, Optional) else param
         param = param() if isinstance(param, Optional) else param
 
     is equivalent of::
 
         param = Optional.extract(param)
 
     """
 
     def __init__(self, type_):
         self.type_ = type_
 
     def __repr__(self):
         return '<Optional:%s>' % self.type_.__repr__()
 
     def __call__(self):
         return self.getval()
 
     def getval(self, evaluate_locals=None):
         """
         returns value from this Optional instance
         """
         if isinstance(self.type_, OAttr):
             param_name = self.type_.attr_name
             if evaluate_locals:
                 return evaluate_locals[param_name]
             # use params name
             return param_name
         return self.type_
 
     @classmethod
     def extract(cls, val, evaluate_locals=None, binary=None):
         """
         Extracts value from Optional() instance
 
         :param val:
         :return: original value if it's not Optional instance else
             value of instance
         """
         if isinstance(val, cls):
             val = val.getval(evaluate_locals)
 
         if binary:
             val = str2bool(val)
 
         return val
 
 
 def parse_args(cli_args, key_prefix=''):
     from rhodecode.lib.utils2 import (escape_split)
     kwargs = collections.defaultdict(dict)
     for el in escape_split(cli_args, ','):
         kv = escape_split(el, '=', 1)
         if len(kv) == 2:
             k, v = kv
             kwargs[key_prefix + k] = v
     return kwargs
 
 
 def get_origin(obj):
     """
     Get origin of permission from object.
 
     :param obj:
     """
     origin = 'permission'
 
     if getattr(obj, 'owner_row', '') and getattr(obj, 'admin_row', ''):
         # admin and owner case, maybe we should use dual string ?
         origin = 'owner'
     elif getattr(obj, 'owner_row', ''):
         origin = 'owner'
     elif getattr(obj, 'admin_row', ''):
         origin = 'super-admin'
     return origin
 
 
 def store_update(updates, attr, name):
     """
     Stores param in updates dict if it's not instance of Optional
     allows easy updates of passed in params
     """
     if not isinstance(attr, Optional):
         updates[name] = attr
 
 
 def has_superadmin_permission(apiuser):
     """
     Return True if apiuser is admin or return False
 
     :param apiuser:
     """
     if HasPermissionAnyApi('hg.admin')(user=apiuser):
         return True
     return False
 
 
 def validate_repo_permissions(apiuser, repoid, repo, perms):
     """
     Raise JsonRPCError if apiuser is not authorized or return True
 
     :param apiuser:
     :param repoid:
     :param repo:
     :param perms:
     """
     if not HasRepoPermissionAnyApi(*perms)(
             user=apiuser, repo_name=repo.repo_name):
-        raise JSONRPCError(
-            'repository `%s` does not exist' % repoid)
+        raise JSONRPCError('repository `%s` does not exist' % repoid)
 
     return True
 
 
 def validate_repo_group_permissions(apiuser, repogroupid, repo_group, perms):
     """
     Raise JsonRPCError if apiuser is not authorized or return True
 
     :param apiuser:
     :param repogroupid: just the id of repository group
     :param repo_group: instance of repo_group
     :param perms:
     """
     if not HasRepoGroupPermissionAnyApi(*perms)(
             user=apiuser, group_name=repo_group.group_name):
         raise JSONRPCError(
             'repository group `%s` does not exist' % repogroupid)
 
     return True
 
 
 def validate_set_owner_permissions(apiuser, owner):
     if isinstance(owner, Optional):
         owner = get_user_or_error(apiuser.user_id)
     else:
         if has_superadmin_permission(apiuser):
             owner = get_user_or_error(owner)
         else:
             # forbid setting owner for non-admins
             raise JSONRPCError(
                 'Only RhodeCode super-admin can specify `owner` param')
     return owner
 
 
 def get_user_or_error(userid):
     """
     Get user by id or name or return JsonRPCError if not found
 
     :param userid:
     """
     from rhodecode.model.user import UserModel
     user_model = UserModel()
 
     if isinstance(userid, (int, long)):
         try:
             user = user_model.get_user(userid)
         except ValueError:
             user = None
     else:
         user = user_model.get_by_username(userid)
 
     if user is None:
         raise JSONRPCError(
             'user `%s` does not exist' % (userid,))
     return user
 
 
 def get_repo_or_error(repoid):
     """
     Get repo by id or name or return JsonRPCError if not found
 
     :param repoid:
     """
     from rhodecode.model.repo import RepoModel
     repo_model = RepoModel()
 
     if isinstance(repoid, (int, long)):
         try:
             repo = repo_model.get_repo(repoid)
         except ValueError:
             repo = None
     else:
         repo = repo_model.get_by_repo_name(repoid)
 
     if repo is None:
         raise JSONRPCError(
             'repository `%s` does not exist' % (repoid,))
     return repo
 
 
 def get_repo_group_or_error(repogroupid):
     """
     Get repo group by id or name or return JsonRPCError if not found
 
     :param repogroupid:
     """
     from rhodecode.model.repo_group import RepoGroupModel
     repo_group_model = RepoGroupModel()
 
     if isinstance(repogroupid, (int, long)):
         try:
             repo_group = repo_group_model._get_repo_group(repogroupid)
         except ValueError:
             repo_group = None
     else:
         repo_group = repo_group_model.get_by_group_name(repogroupid)
 
     if repo_group is None:
         raise JSONRPCError(
             'repository group `%s` does not exist' % (repogroupid,))
     return repo_group
 
 
 def get_user_group_or_error(usergroupid):
     """
     Get user group by id or name or return JsonRPCError if not found
 
     :param usergroupid:
     """
     from rhodecode.model.user_group import UserGroupModel
     user_group_model = UserGroupModel()
 
     if isinstance(usergroupid, (int, long)):
         try:
             user_group = user_group_model.get_group(usergroupid)
         except ValueError:
             user_group = None
     else:
         user_group = user_group_model.get_by_name(usergroupid)
 
     if user_group is None:
         raise JSONRPCError(
             'user group `%s` does not exist' % (usergroupid,))
     return user_group
 
 
 def get_perm_or_error(permid, prefix=None):
     """
     Get permission by id or name or return JsonRPCError if not found
 
     :param permid:
     """
     from rhodecode.model.permission import PermissionModel
 
     perm = PermissionModel.cls.get_by_key(permid)
     if perm is None:
         msg = 'permission `{}` does not exist.'.format(permid)
         if prefix:
             msg += ' Permission should start with prefix: `{}`'.format(prefix)
         raise JSONRPCError(msg)
 
     if prefix:
         if not perm.permission_name.startswith(prefix):
             raise JSONRPCError('permission `%s` is invalid, '
                                'should start with %s' % (permid, prefix))
     return perm
 
 
 def get_gist_or_error(gistid):
     """
     Get gist by id or gist_access_id or return JsonRPCError if not found
 
     :param gistid:
     """
     from rhodecode.model.gist import GistModel
 
     gist = GistModel.cls.get_by_access_id(gistid)
     if gist is None:
         raise JSONRPCError('gist `%s` does not exist' % (gistid,))
     return gist
 
 
 def get_pull_request_or_error(pullrequestid):
     """
     Get pull request by id or return JsonRPCError if not found
 
     :param pullrequestid:
     """
     from rhodecode.model.pull_request import PullRequestModel
 
     try:
         pull_request = PullRequestModel().get(int(pullrequestid))
     except ValueError:
         raise JSONRPCError('pullrequestid must be an integer')
     if not pull_request:
         raise JSONRPCError('pull request `%s` does not exist' % (
             pullrequestid,))
     return pull_request
 
 
 def build_commit_data(commit, detail_level):
     parsed_diff = []
     if detail_level == 'extended':
         for f_path in commit.added_paths:
             parsed_diff.append(_get_commit_dict(filename=f_path, op='A'))
         for f_path in commit.changed_paths:
             parsed_diff.append(_get_commit_dict(filename=f_path, op='M'))
         for f_path in commit.removed_paths:
             parsed_diff.append(_get_commit_dict(filename=f_path, op='D'))
 
     elif detail_level == 'full':
         from rhodecode.lib.diffs import DiffProcessor
         diff_processor = DiffProcessor(commit.diff())
         for dp in diff_processor.prepare():
             del dp['stats']['ops']
             _stats = dp['stats']
             parsed_diff.append(_get_commit_dict(
                 filename=dp['filename'], op=dp['operation'],
                 new_revision=dp['new_revision'],
                 old_revision=dp['old_revision'],
                 raw_diff=dp['raw_diff'], stats=_stats))
 
     return parsed_diff
 
 
 def get_commit_or_error(ref, repo):
     try:
         ref_type, _, ref_hash = ref.split(':')
     except ValueError:
         raise JSONRPCError(
             'Ref `{ref}` given in a wrong format. Please check the API'
             ' documentation for more details'.format(ref=ref))
     try:
         # TODO: dan: refactor this to use repo.scm_instance().get_commit()
         # once get_commit supports ref_types
         return get_commit_from_ref_name(repo, ref_hash)
     except RepositoryError:
         raise JSONRPCError('Ref `{ref}` does not exist'.format(ref=ref))
 
 
 def _get_ref_hash(repo, type_, name):
     vcs_repo = repo.scm_instance()
     if type_ in ['branch'] and vcs_repo.alias in ('hg', 'git'):
         return vcs_repo.branches[name]
     elif type_ in ['bookmark', 'book'] and vcs_repo.alias == 'hg':
         return vcs_repo.bookmarks[name]
     else:
         raise ValueError()
 
 
 def resolve_ref_or_error(ref, repo, allowed_ref_types=None):
     allowed_ref_types = allowed_ref_types or ['bookmark', 'book', 'tag', 'branch']
 
     def _parse_ref(type_, name, hash_=None):
         return type_, name, hash_
 
     try:
         ref_type, ref_name, ref_hash = _parse_ref(*ref.split(':'))
     except TypeError:
         raise JSONRPCError(
             'Ref `{ref}` given in a wrong format. Please check the API'
             ' documentation for more details'.format(ref=ref))
 
     if ref_type not in allowed_ref_types:
         raise JSONRPCError(
             'Ref `{ref}` type is not allowed. '
             'Only:{allowed_refs} are possible.'.format(
                 ref=ref, allowed_refs=allowed_ref_types))
 
     try:
         ref_hash = ref_hash or _get_ref_hash(repo, ref_type, ref_name)
     except (KeyError, ValueError):
         raise JSONRPCError(
             'The specified value:{type}:`{name}` does not exist, or is not allowed.'.format(
                 type=ref_type, name=ref_name))
 
     return ':'.join([ref_type, ref_name, ref_hash])
 
 
 def _get_commit_dict(
         filename, op, new_revision=None, old_revision=None,
         raw_diff=None, stats=None):
     if stats is None:
         stats = {
             "added": None,
             "binary": None,
             "deleted": None
         }
     return {
         "filename": safe_unicode(filename),
         "op": op,
 
         # extra details
         "new_revision": new_revision,
         "old_revision": old_revision,
 
         "raw_diff": raw_diff,
         "stats": stats
     }

rhodecode/api/views/repo_api.py

@@ -1,2506 +1,2507 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import time
 
 import rhodecode
 from rhodecode.api import (
     jsonrpc_method, JSONRPCError, JSONRPCForbidden, JSONRPCValidationError)
 from rhodecode.api.utils import (
     has_superadmin_permission, Optional, OAttr, get_repo_or_error,
     get_user_group_or_error, get_user_or_error, validate_repo_permissions,
     get_perm_or_error, parse_args, get_origin, build_commit_data,
     validate_set_owner_permissions)
 from rhodecode.lib import audit_logger, rc_cache
 from rhodecode.lib import repo_maintenance
 from rhodecode.lib.auth import (
     HasPermissionAnyApi, HasUserGroupPermissionAnyApi,
     HasRepoPermissionAnyApi)
 from rhodecode.lib.celerylib.utils import get_task_id
 from rhodecode.lib.utils2 import (
     str2bool, time_to_datetime, safe_str, safe_int, safe_unicode)
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.exceptions import (
     StatusChangeOnClosedPullRequestError, CommentVersionMismatch)
 from rhodecode.lib.vcs import RepositoryError
 from rhodecode.lib.vcs.exceptions import NodeDoesNotExistError
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.db import (
     Session, ChangesetStatus, RepositoryField, Repository, RepoGroup,
     ChangesetComment)
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.pull_request import PullRequestModel
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.scm import ScmModel, RepoList
 from rhodecode.model.settings import SettingsModel, VcsSettingsModel
 from rhodecode.model import validation_schema
 from rhodecode.model.validation_schema.schemas import repo_schema
 
 log = logging.getLogger(__name__)
 
 
 @jsonrpc_method()
 def get_repo(request, apiuser, repoid, cache=Optional(True)):
     """
     Gets an existing repository by its name or repository_id.
 
     The members section so the output returns users groups or users
     associated with that repository.
 
     This command can only be run using an |authtoken| with admin rights,
     or users with at least read rights to the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository id.
     :type repoid: str or int
     :param cache: use the cached value for last changeset
     :type: cache: Optional(bool)
 
     Example output:
 
     .. code-block:: bash
 
         {
           "error": null,
           "id": <repo_id>,
           "result": {
             "clone_uri": null,
             "created_on": "timestamp",
             "description": "repo description",
             "enable_downloads": false,
             "enable_locking": false,
             "enable_statistics": false,
             "followers": [
               {
                 "active": true,
                 "admin": false,
                 "api_key": "****************************************",
                 "api_keys": [
                   "****************************************"
                 ],
                 "email": "[e-mail]",
                 "emails": [
                   "[e-mail]"
                 ],
                 "extern_name": "rhodecode",
                 "extern_type": "rhodecode",
                 "firstname": "username",
                 "ip_addresses": [],
                 "language": null,
                 "last_login": "2015-09-16T17:16:35.854",
                 "lastname": "surname",
                 "user_id": <user_id>,
                 "username": "name"
               }
             ],
             "fork_of": "parent-repo",
             "landing_rev": [
               "rev",
               "tip"
             ],
             "last_changeset": {
               "author": "User <[e-mail]>",
               "branch": "default",
               "date": "timestamp",
               "message": "last commit message",
               "parents": [
                 {
                   "raw_id": "commit-id"
                 }
               ],
               "raw_id": "commit-id",
               "revision": <revision number>,
               "short_id": "short id"
             },
             "lock_reason": null,
             "locked_by": null,
             "locked_date": null,
             "owner": "owner-name",
             "permissions": [
               {
                 "name": "super-admin-name",
                 "origin": "super-admin",
                 "permission": "repository.admin",
                 "type": "user"
               },
               {
                 "name": "owner-name",
                 "origin": "owner",
                 "permission": "repository.admin",
                 "type": "user"
               },
               {
                 "name": "user-group-name",
                 "origin": "permission",
                 "permission": "repository.write",
                 "type": "user_group"
               }
             ],
             "private": true,
             "repo_id": 676,
             "repo_name": "user-group/repo-name",
             "repo_type": "hg"
           }
         }
     """
 
     repo = get_repo_or_error(repoid)
     cache = Optional.extract(cache)
 
     include_secrets = False
     if has_superadmin_permission(apiuser):
         include_secrets = True
     else:
         # check if we have at least read permission for this repo !
         _perms = (
             'repository.admin', 'repository.write', 'repository.read',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     permissions = []
     for _user in repo.permissions():
         user_data = {
             'name': _user.username,
             'permission': _user.permission,
             'origin': get_origin(_user),
             'type': "user",
         }
         permissions.append(user_data)
 
     for _user_group in repo.permission_user_groups():
         user_group_data = {
             'name': _user_group.users_group_name,
             'permission': _user_group.permission,
             'origin': get_origin(_user_group),
             'type': "user_group",
         }
         permissions.append(user_group_data)
 
     following_users = [
         user.user.get_api_data(include_secrets=include_secrets)
         for user in repo.followers]
 
     if not cache:
         repo.update_commit_cache()
     data = repo.get_api_data(include_secrets=include_secrets)
     data['permissions'] = permissions
     data['followers'] = following_users
     return data
 
 
 @jsonrpc_method()
 def get_repos(request, apiuser, root=Optional(None), traverse=Optional(True)):
     """
     Lists all existing repositories.
 
     This command can only be run using an |authtoken| with admin rights,
     or users with at least read rights to |repos|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param root: specify root repository group to fetch repositories.
         filters the returned repositories to be members of given root group.
     :type root: Optional(None)
     :param traverse: traverse given root into subrepositories. With this flag
         set to False, it will only return top-level repositories from `root`.
         if root is empty it will return just top-level repositories.
     :type traverse: Optional(True)
 
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: [
                   {
                     "repo_id" :          "<repo_id>",
                     "repo_name" :        "<reponame>"
                     "repo_type" :        "<repo_type>",
                     "clone_uri" :        "<clone_uri>",
                     "private": :         "<bool>",
                     "created_on" :       "<datetimecreated>",
                     "description" :      "<description>",
                     "landing_rev":       "<landing_rev>",
                     "owner":             "<repo_owner>",
                     "fork_of":           "<name_of_fork_parent>",
                     "enable_downloads":  "<bool>",
                     "enable_locking":    "<bool>",
                     "enable_statistics": "<bool>",
                   },
                   ...
                 ]
         error:  null
     """
 
     include_secrets = has_superadmin_permission(apiuser)
     _perms = ('repository.read', 'repository.write', 'repository.admin',)
     extras = {'user': apiuser}
 
     root = Optional.extract(root)
     traverse = Optional.extract(traverse, binary=True)
 
     if root:
         # verify parent existance, if it's empty return an error
         parent = RepoGroup.get_by_group_name(root)
         if not parent:
             raise JSONRPCError(
                 'Root repository group `{}` does not exist'.format(root))
 
         if traverse:
             repos = RepoModel().get_repos_for_root(root=root, traverse=traverse)
         else:
             repos = RepoModel().get_repos_for_root(root=parent)
     else:
         if traverse:
             repos = RepoModel().get_all()
         else:
             # return just top-level
             repos = RepoModel().get_repos_for_root(root=None)
 
     repo_list = RepoList(repos, perm_set=_perms, extra_kwargs=extras)
     return [repo.get_api_data(include_secrets=include_secrets)
             for repo in repo_list]
 
 
 @jsonrpc_method()
 def get_repo_changeset(request, apiuser, repoid, revision,
                        details=Optional('basic')):
     """
     Returns information about a changeset.
 
     Additionally parameters define the amount of details returned by
     this function.
 
     This command can only be run using an |authtoken| with admin rights,
     or users with at least read rights to the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository id
     :type repoid: str or int
     :param revision: revision for which listing should be done
     :type revision: str
     :param details: details can be 'basic|extended|full' full gives diff
         info details like the diff itself, and number of changed files etc.
     :type details: Optional(str)
 
     """
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
-        _perms = (
-            'repository.admin', 'repository.write', 'repository.read',)
+        _perms = ('repository.admin', 'repository.write', 'repository.read',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     changes_details = Optional.extract(details)
     _changes_details_types = ['basic', 'extended', 'full']
     if changes_details not in _changes_details_types:
         raise JSONRPCError(
             'ret_type must be one of %s' % (
                 ','.join(_changes_details_types)))
 
     pre_load = ['author', 'branch', 'date', 'message', 'parents',
                 'status', '_commit', '_file_paths']
 
     try:
         cs = repo.get_commit(commit_id=revision, pre_load=pre_load)
     except TypeError as e:
         raise JSONRPCError(safe_str(e))
     _cs_json = cs.__json__()
     _cs_json['diff'] = build_commit_data(cs, changes_details)
     if changes_details == 'full':
         _cs_json['refs'] = cs._get_refs()
     return _cs_json
 
 
 @jsonrpc_method()
 def get_repo_changesets(request, apiuser, repoid, start_rev, limit,
                         details=Optional('basic')):
     """
     Returns a set of commits limited by the number starting
     from the `start_rev` option.
 
     Additional parameters define the amount of details returned by this
     function.
 
     This command can only be run using an |authtoken| with admin rights,
     or users with at least read rights to |repos|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository ID.
     :type repoid: str or int
     :param start_rev: The starting revision from where to get changesets.
     :type start_rev: str
     :param limit: Limit the number of commits to this amount
     :type limit: str or int
     :param details: Set the level of detail returned. Valid option are:
         ``basic``, ``extended`` and ``full``.
     :type details: Optional(str)
 
     .. note::
 
        Setting the parameter `details` to the value ``full`` is extensive
        and returns details like the diff itself, and the number
        of changed files.
 
     """
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
-        _perms = (
-            'repository.admin', 'repository.write', 'repository.read',)
+        _perms = ('repository.admin', 'repository.write', 'repository.read',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     changes_details = Optional.extract(details)
     _changes_details_types = ['basic', 'extended', 'full']
     if changes_details not in _changes_details_types:
         raise JSONRPCError(
             'ret_type must be one of %s' % (
                 ','.join(_changes_details_types)))
 
     limit = int(limit)
     pre_load = ['author', 'branch', 'date', 'message', 'parents',
                 'status', '_commit', '_file_paths']
 
     vcs_repo = repo.scm_instance()
     # SVN needs a special case to distinguish its index and commit id
     if vcs_repo and vcs_repo.alias == 'svn' and (start_rev == '0'):
         start_rev = vcs_repo.commit_ids[0]
 
     try:
         commits = vcs_repo.get_commits(
             start_id=start_rev, pre_load=pre_load, translate_tags=False)
     except TypeError as e:
         raise JSONRPCError(safe_str(e))
     except Exception:
         log.exception('Fetching of commits failed')
         raise JSONRPCError('Error occurred during commit fetching')
 
     ret = []
     for cnt, commit in enumerate(commits):
         if cnt >= limit != -1:
             break
         _cs_json = commit.__json__()
         _cs_json['diff'] = build_commit_data(commit, changes_details)
         if changes_details == 'full':
             _cs_json['refs'] = {
                 'branches': [commit.branch],
                 'bookmarks': getattr(commit, 'bookmarks', []),
                 'tags': commit.tags
             }
         ret.append(_cs_json)
     return ret
 
 
 @jsonrpc_method()
 def get_repo_nodes(request, apiuser, repoid, revision, root_path,
                    ret_type=Optional('all'), details=Optional('basic'),
                    max_file_bytes=Optional(None)):
     """
     Returns a list of nodes and children in a flat list for a given
     path at given revision.
 
     It's possible to specify ret_type to show only `files` or `dirs`.
 
     This command can only be run using an |authtoken| with admin rights,
     or users with at least read rights to |repos|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository ID.
     :type repoid: str or int
     :param revision: The revision for which listing should be done.
     :type revision: str
     :param root_path: The path from which to start displaying.
     :type root_path: str
     :param ret_type: Set the return type. Valid options are
         ``all`` (default), ``files`` and ``dirs``.
     :type ret_type: Optional(str)
     :param details: Returns extended information about nodes, such as
         md5, binary, and or content.
         The valid options are ``basic`` and ``full``.
     :type details: Optional(str)
     :param max_file_bytes: Only return file content under this file size bytes
     :type details: Optional(int)
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: [
                     {
                       "binary": false,
                       "content": "File line",
                       "extension": "md",
                       "lines": 2,
                       "md5": "059fa5d29b19c0657e384749480f6422",
                       "mimetype": "text/x-minidsrc",
                       "name": "file.md",
                       "size": 580,
                       "type": "file"
                     },
                   ...
                 ]
         error:  null
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin', 'repository.write', 'repository.read',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     ret_type = Optional.extract(ret_type)
     details = Optional.extract(details)
     _extended_types = ['basic', 'full']
     if details not in _extended_types:
         raise JSONRPCError('ret_type must be one of %s' % (','.join(_extended_types)))
     extended_info = False
     content = False
     if details == 'basic':
         extended_info = True
 
     if details == 'full':
         extended_info = content = True
 
     _map = {}
     try:
         # check if repo is not empty by any chance, skip quicker if it is.
         _scm = repo.scm_instance()
         if _scm.is_empty():
             return []
 
         _d, _f = ScmModel().get_nodes(
             repo, revision, root_path, flat=False,
             extended_info=extended_info, content=content,
             max_file_bytes=max_file_bytes)
         _map = {
             'all': _d + _f,
             'files': _f,
             'dirs': _d,
         }
         return _map[ret_type]
     except KeyError:
         raise JSONRPCError(
             'ret_type must be one of %s' % (','.join(sorted(_map.keys()))))
     except Exception:
         log.exception("Exception occurred while trying to get repo nodes")
         raise JSONRPCError(
             'failed to get repo: `%s` nodes' % repo.repo_name
         )
 
 
 @jsonrpc_method()
 def get_repo_file(request, apiuser, repoid, commit_id, file_path,
                   max_file_bytes=Optional(None), details=Optional('basic'),
                   cache=Optional(True)):
     """
     Returns a single file from repository at given revision.
 
     This command can only be run using an |authtoken| with admin rights,
     or users with at least read rights to |repos|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository ID.
     :type repoid: str or int
     :param commit_id: The revision for which listing should be done.
     :type commit_id: str
     :param file_path: The path from which to start displaying.
     :type file_path: str
     :param details: Returns different set of information about nodes.
         The valid options are ``minimal`` ``basic`` and ``full``.
     :type details: Optional(str)
     :param max_file_bytes: Only return file content under this file size bytes
     :type max_file_bytes: Optional(int)
     :param cache: Use internal caches for fetching files. If disabled fetching
         files is slower but more memory efficient
     :type cache: Optional(bool)
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: {
             "binary": false,
             "extension": "py",
             "lines": 35,
             "content": "....",
             "md5": "76318336366b0f17ee249e11b0c99c41",
             "mimetype": "text/x-python",
             "name": "python.py",
             "size": 817,
             "type": "file",
         }
         error:  null
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin', 'repository.write', 'repository.read',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     cache = Optional.extract(cache, binary=True)
     details = Optional.extract(details)
     _extended_types = ['minimal', 'minimal+search', 'basic', 'full']
     if details not in _extended_types:
         raise JSONRPCError(
             'ret_type must be one of %s, got %s' % (','.join(_extended_types)), details)
     extended_info = False
     content = False
 
     if details == 'minimal':
         extended_info = False
 
     elif details == 'basic':
         extended_info = True
 
     elif details == 'full':
         extended_info = content = True
 
     file_path = safe_unicode(file_path)
     try:
         # check if repo is not empty by any chance, skip quicker if it is.
         _scm = repo.scm_instance()
         if _scm.is_empty():
             return None
 
         node = ScmModel().get_node(
             repo, commit_id, file_path, extended_info=extended_info,
             content=content, max_file_bytes=max_file_bytes, cache=cache)
     except NodeDoesNotExistError:
         raise JSONRPCError(u'There is no file in repo: `{}` at path `{}` for commit: `{}`'.format(
             repo.repo_name, file_path, commit_id))
     except Exception:
         log.exception(u"Exception occurred while trying to get repo %s file",
                       repo.repo_name)
         raise JSONRPCError(u'failed to get repo: `{}` file at path {}'.format(
             repo.repo_name, file_path))
 
     return node
 
 
 @jsonrpc_method()
 def get_repo_fts_tree(request, apiuser, repoid, commit_id, root_path):
     """
     Returns a list of tree nodes for path at given revision. This api is built
     strictly for usage in full text search building, and shouldn't be consumed
 
     This command can only be run using an |authtoken| with admin rights,
     or users with at least read rights to |repos|.
 
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin', 'repository.write', 'repository.read',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     repo_id = repo.repo_id
     cache_seconds = safe_int(rhodecode.CONFIG.get('rc_cache.cache_repo.expiration_time'))
     cache_on = cache_seconds > 0
 
     cache_namespace_uid = 'cache_repo.{}'.format(repo_id)
     region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
 
     def compute_fts_tree(cache_ver, repo_id, commit_id, root_path):
         return ScmModel().get_fts_data(repo_id, commit_id, root_path)
 
     try:
         # check if repo is not empty by any chance, skip quicker if it is.
         _scm = repo.scm_instance()
         if _scm.is_empty():
             return []
     except RepositoryError:
         log.exception("Exception occurred while trying to get repo nodes")
         raise JSONRPCError('failed to get repo: `%s` nodes' % repo.repo_name)
 
     try:
         # we need to resolve commit_id to a FULL sha for cache to work correctly.
         # sending 'master' is a pointer that needs to be translated to current commit.
         commit_id = _scm.get_commit(commit_id=commit_id).raw_id
         log.debug(
             'Computing FTS REPO TREE for repo_id %s commit_id `%s` '
             'with caching: %s[TTL: %ss]' % (
                 repo_id, commit_id, cache_on, cache_seconds or 0))
 
         tree_files = compute_fts_tree(rc_cache.FILE_TREE_CACHE_VER, repo_id, commit_id, root_path)
         return tree_files
 
     except Exception:
         log.exception("Exception occurred while trying to get repo nodes")
         raise JSONRPCError('failed to get repo: `%s` nodes' % repo.repo_name)
 
 
 @jsonrpc_method()
 def get_repo_refs(request, apiuser, repoid):
     """
     Returns a dictionary of current references. It returns
     bookmarks, branches, closed_branches, and tags for given repository
 
     It's possible to specify ret_type to show only `files` or `dirs`.
 
     This command can only be run using an |authtoken| with admin rights,
     or users with at least read rights to |repos|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository ID.
     :type repoid: str or int
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         "result": {
             "bookmarks": {
               "dev": "5611d30200f4040ba2ab4f3d64e5b06408a02188",
               "master": "367f590445081d8ec8c2ea0456e73ae1f1c3d6cf"
             },
             "branches": {
               "default": "5611d30200f4040ba2ab4f3d64e5b06408a02188",
               "stable": "367f590445081d8ec8c2ea0456e73ae1f1c3d6cf"
             },
             "branches_closed": {},
             "tags": {
               "tip": "5611d30200f4040ba2ab4f3d64e5b06408a02188",
               "v4.4.0": "1232313f9e6adac5ce5399c2a891dc1e72b79022",
               "v4.4.1": "cbb9f1d329ae5768379cdec55a62ebdd546c4e27",
               "v4.4.2": "24ffe44a27fcd1c5b6936144e176b9f6dd2f3a17",
             }
         }
         error:  null
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin', 'repository.write', 'repository.read',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     try:
         # check if repo is not empty by any chance, skip quicker if it is.
         vcs_instance = repo.scm_instance()
         refs = vcs_instance.refs()
         return refs
     except Exception:
         log.exception("Exception occurred while trying to get repo refs")
         raise JSONRPCError(
             'failed to get repo: `%s` references' % repo.repo_name
         )
 
 
 @jsonrpc_method()
 def create_repo(
         request, apiuser, repo_name, repo_type,
         owner=Optional(OAttr('apiuser')),
         description=Optional(''),
         private=Optional(False),
         clone_uri=Optional(None),
         push_uri=Optional(None),
         landing_rev=Optional(None),
         enable_statistics=Optional(False),
         enable_locking=Optional(False),
         enable_downloads=Optional(False),
         copy_permissions=Optional(False)):
     """
     Creates a repository.
 
     * If the repository name contains "/", repository will be created inside
       a repository group or nested repository groups
 
       For example "foo/bar/repo1" will create |repo| called "repo1" inside
       group "foo/bar". You have to have permissions to access and write to
       the last repository group ("bar" in this example)
 
     This command can only be run using an |authtoken| with at least
     permissions to create repositories, or write permissions to
     parent repository groups.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repo_name: Set the repository name.
     :type repo_name: str
     :param repo_type: Set the repository type; 'hg','git', or 'svn'.
     :type repo_type: str
     :param owner: user_id or username
     :type owner: Optional(str)
     :param description: Set the repository description.
     :type description: Optional(str)
     :param private: set repository as private
     :type private: bool
     :param clone_uri: set clone_uri
     :type clone_uri: str
     :param push_uri: set push_uri
     :type push_uri: str
     :param landing_rev: <rev_type>:<rev>, e.g branch:default, book:dev, rev:abcd
     :type landing_rev: str
     :param enable_locking:
     :type enable_locking: bool
     :param enable_downloads:
     :type enable_downloads: bool
     :param enable_statistics:
     :type enable_statistics: bool
     :param copy_permissions: Copy permission from group in which the
         repository is being created.
     :type copy_permissions: bool
 
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: {
                   "msg": "Created new repository `<reponame>`",
                   "success": true,
                   "task": "<celery task id or None if done sync>"
                 }
         error:  null
 
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
          'failed to create repository `<repo_name>`'
       }
 
     """
 
     owner = validate_set_owner_permissions(apiuser, owner)
 
     description = Optional.extract(description)
     copy_permissions = Optional.extract(copy_permissions)
     clone_uri = Optional.extract(clone_uri)
     push_uri = Optional.extract(push_uri)
 
     defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
     if isinstance(private, Optional):
         private = defs.get('repo_private') or Optional.extract(private)
     if isinstance(repo_type, Optional):
         repo_type = defs.get('repo_type')
     if isinstance(enable_statistics, Optional):
         enable_statistics = defs.get('repo_enable_statistics')
     if isinstance(enable_locking, Optional):
         enable_locking = defs.get('repo_enable_locking')
     if isinstance(enable_downloads, Optional):
         enable_downloads = defs.get('repo_enable_downloads')
 
     landing_ref, _label = ScmModel.backend_landing_ref(repo_type)
     ref_choices, _labels = ScmModel().get_repo_landing_revs(request.translate)
     ref_choices = list(set(ref_choices + [landing_ref]))
 
     landing_commit_ref = Optional.extract(landing_rev) or landing_ref
 
     schema = repo_schema.RepoSchema().bind(
         repo_type_options=rhodecode.BACKENDS.keys(),
         repo_ref_options=ref_choices,
         repo_type=repo_type,
         # user caller
         user=apiuser)
 
     try:
         schema_data = schema.deserialize(dict(
             repo_name=repo_name,
             repo_type=repo_type,
             repo_owner=owner.username,
             repo_description=description,
             repo_landing_commit_ref=landing_commit_ref,
             repo_clone_uri=clone_uri,
             repo_push_uri=push_uri,
             repo_private=private,
             repo_copy_permissions=copy_permissions,
             repo_enable_statistics=enable_statistics,
             repo_enable_downloads=enable_downloads,
             repo_enable_locking=enable_locking))
     except validation_schema.Invalid as err:
         raise JSONRPCValidationError(colander_exc=err)
 
     try:
         data = {
             'owner': owner,
             'repo_name': schema_data['repo_group']['repo_name_without_group'],
             'repo_name_full': schema_data['repo_name'],
             'repo_group': schema_data['repo_group']['repo_group_id'],
             'repo_type': schema_data['repo_type'],
             'repo_description': schema_data['repo_description'],
             'repo_private': schema_data['repo_private'],
             'clone_uri': schema_data['repo_clone_uri'],
             'push_uri': schema_data['repo_push_uri'],
             'repo_landing_rev': schema_data['repo_landing_commit_ref'],
             'enable_statistics': schema_data['repo_enable_statistics'],
             'enable_locking': schema_data['repo_enable_locking'],
             'enable_downloads': schema_data['repo_enable_downloads'],
             'repo_copy_permissions': schema_data['repo_copy_permissions'],
         }
 
         task = RepoModel().create(form_data=data, cur_user=owner.user_id)
         task_id = get_task_id(task)
         # no commit, it's done in RepoModel, or async via celery
         return {
             'msg': "Created new repository `%s`" % (schema_data['repo_name'],),
             'success': True,  # cannot return the repo data here since fork
             # can be done async
             'task': task_id
         }
     except Exception:
         log.exception(
             u"Exception while trying to create the repository %s",
             schema_data['repo_name'])
         raise JSONRPCError(
             'failed to create repository `%s`' % (schema_data['repo_name'],))
 
 
 @jsonrpc_method()
 def add_field_to_repo(request, apiuser, repoid, key, label=Optional(''),
                       description=Optional('')):
     """
     Adds an extra field to a repository.
 
     This command can only be run using an |authtoken| with at least
     write permissions to the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository id.
     :type repoid: str or int
     :param key: Create a unique field key for this repository.
     :type key: str
     :param label:
     :type label: Optional(str)
     :param description:
     :type description: Optional(str)
     """
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     label = Optional.extract(label) or key
     description = Optional.extract(description)
 
     field = RepositoryField.get_by_key_name(key, repo)
     if field:
         raise JSONRPCError('Field with key '
                            '`%s` exists for repo `%s`' % (key, repoid))
 
     try:
         RepoModel().add_repo_field(repo, key, field_label=label,
                                    field_desc=description)
         Session().commit()
         return {
             'msg': "Added new repository field `%s`" % (key,),
             'success': True,
         }
     except Exception:
         log.exception("Exception occurred while trying to add field to repo")
         raise JSONRPCError(
             'failed to create new field for repository `%s`' % (repoid,))
 
 
 @jsonrpc_method()
 def remove_field_from_repo(request, apiuser, repoid, key):
     """
     Removes an extra field from a repository.
 
     This command can only be run using an |authtoken| with at least
     write permissions to the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository ID.
     :type repoid: str or int
     :param key: Set the unique field key for this repository.
     :type key: str
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     field = RepositoryField.get_by_key_name(key, repo)
     if not field:
         raise JSONRPCError('Field with key `%s` does not '
                            'exists for repo `%s`' % (key, repoid))
 
     try:
         RepoModel().delete_repo_field(repo, field_key=key)
         Session().commit()
         return {
             'msg': "Deleted repository field `%s`" % (key,),
             'success': True,
         }
     except Exception:
         log.exception(
             "Exception occurred while trying to delete field from repo")
         raise JSONRPCError(
             'failed to delete field for repository `%s`' % (repoid,))
 
 
 @jsonrpc_method()
 def update_repo(
         request, apiuser, repoid, repo_name=Optional(None),
         owner=Optional(OAttr('apiuser')), description=Optional(''),
         private=Optional(False),
         clone_uri=Optional(None), push_uri=Optional(None),
         landing_rev=Optional(None), fork_of=Optional(None),
         enable_statistics=Optional(False),
         enable_locking=Optional(False),
         enable_downloads=Optional(False), fields=Optional('')):
     """
     Updates a repository with the given information.
 
     This command can only be run using an |authtoken| with at least
     admin permissions to the |repo|.
 
     * If the repository name contains "/", repository will be updated
       accordingly with a repository group or nested repository groups
 
       For example repoid=repo-test name="foo/bar/repo-test" will update |repo|
       called "repo-test" and place it inside group "foo/bar".
       You have to have permissions to access and write to the last repository
       group ("bar" in this example)
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: repository name or repository ID.
     :type repoid: str or int
     :param repo_name: Update the |repo| name, including the
         repository group it's in.
     :type repo_name: str
     :param owner: Set the |repo| owner.
     :type owner: str
     :param fork_of: Set the |repo| as fork of another |repo|.
     :type fork_of: str
     :param description: Update the |repo| description.
     :type description: str
     :param private: Set the |repo| as private. (True | False)
     :type private: bool
     :param clone_uri: Update the |repo| clone URI.
     :type clone_uri: str
     :param landing_rev: Set the |repo| landing revision. e.g branch:default, book:dev, rev:abcd
     :type landing_rev: str
     :param enable_statistics: Enable statistics on the |repo|, (True | False).
     :type enable_statistics: bool
     :param enable_locking: Enable |repo| locking.
     :type enable_locking: bool
     :param enable_downloads: Enable downloads from the |repo|, (True | False).
     :type enable_downloads: bool
     :param fields: Add extra fields to the |repo|. Use the following
         example format: ``field_key=field_val,field_key2=fieldval2``.
         Escape ', ' with \,
     :type fields: str
     """
 
     repo = get_repo_or_error(repoid)
 
     include_secrets = False
     if not has_superadmin_permission(apiuser):
-        validate_repo_permissions(apiuser, repoid, repo, ('repository.admin',))
+        _perms = ('repository.admin',)
+        validate_repo_permissions(apiuser, repoid, repo, _perms)
     else:
         include_secrets = True
 
     updates = dict(
         repo_name=repo_name
         if not isinstance(repo_name, Optional) else repo.repo_name,
 
         fork_id=fork_of
         if not isinstance(fork_of, Optional) else repo.fork.repo_name if repo.fork else None,
 
         user=owner
         if not isinstance(owner, Optional) else repo.user.username,
 
         repo_description=description
         if not isinstance(description, Optional) else repo.description,
 
         repo_private=private
         if not isinstance(private, Optional) else repo.private,
 
         clone_uri=clone_uri
         if not isinstance(clone_uri, Optional) else repo.clone_uri,
 
         push_uri=push_uri
         if not isinstance(push_uri, Optional) else repo.push_uri,
 
         repo_landing_rev=landing_rev
         if not isinstance(landing_rev, Optional) else repo._landing_revision,
 
         repo_enable_statistics=enable_statistics
         if not isinstance(enable_statistics, Optional) else repo.enable_statistics,
 
         repo_enable_locking=enable_locking
         if not isinstance(enable_locking, Optional) else repo.enable_locking,
 
         repo_enable_downloads=enable_downloads
         if not isinstance(enable_downloads, Optional) else repo.enable_downloads)
 
     landing_ref, _label = ScmModel.backend_landing_ref(repo.repo_type)
     ref_choices, _labels = ScmModel().get_repo_landing_revs(
         request.translate, repo=repo)
     ref_choices = list(set(ref_choices + [landing_ref]))
 
     old_values = repo.get_api_data()
     repo_type = repo.repo_type
     schema = repo_schema.RepoSchema().bind(
         repo_type_options=rhodecode.BACKENDS.keys(),
         repo_ref_options=ref_choices,
         repo_type=repo_type,
         # user caller
         user=apiuser,
         old_values=old_values)
     try:
         schema_data = schema.deserialize(dict(
             # we save old value, users cannot change type
             repo_type=repo_type,
 
             repo_name=updates['repo_name'],
             repo_owner=updates['user'],
             repo_description=updates['repo_description'],
             repo_clone_uri=updates['clone_uri'],
             repo_push_uri=updates['push_uri'],
             repo_fork_of=updates['fork_id'],
             repo_private=updates['repo_private'],
             repo_landing_commit_ref=updates['repo_landing_rev'],
             repo_enable_statistics=updates['repo_enable_statistics'],
             repo_enable_downloads=updates['repo_enable_downloads'],
             repo_enable_locking=updates['repo_enable_locking']))
     except validation_schema.Invalid as err:
         raise JSONRPCValidationError(colander_exc=err)
 
     # save validated data back into the updates dict
     validated_updates = dict(
         repo_name=schema_data['repo_group']['repo_name_without_group'],
         repo_group=schema_data['repo_group']['repo_group_id'],
 
         user=schema_data['repo_owner'],
         repo_description=schema_data['repo_description'],
         repo_private=schema_data['repo_private'],
         clone_uri=schema_data['repo_clone_uri'],
         push_uri=schema_data['repo_push_uri'],
         repo_landing_rev=schema_data['repo_landing_commit_ref'],
         repo_enable_statistics=schema_data['repo_enable_statistics'],
         repo_enable_locking=schema_data['repo_enable_locking'],
         repo_enable_downloads=schema_data['repo_enable_downloads'],
     )
 
     if schema_data['repo_fork_of']:
         fork_repo = get_repo_or_error(schema_data['repo_fork_of'])
         validated_updates['fork_id'] = fork_repo.repo_id
 
     # extra fields
     fields = parse_args(Optional.extract(fields), key_prefix='ex_')
     if fields:
         validated_updates.update(fields)
 
     try:
         RepoModel().update(repo, **validated_updates)
         audit_logger.store_api(
             'repo.edit', action_data={'old_data': old_values},
             user=apiuser, repo=repo)
         Session().commit()
         return {
             'msg': 'updated repo ID:%s %s' % (repo.repo_id, repo.repo_name),
             'repository': repo.get_api_data(include_secrets=include_secrets)
         }
     except Exception:
         log.exception(
             u"Exception while trying to update the repository %s",
             repoid)
         raise JSONRPCError('failed to update repo `%s`' % repoid)
 
 
 @jsonrpc_method()
 def fork_repo(request, apiuser, repoid, fork_name,
               owner=Optional(OAttr('apiuser')),
               description=Optional(''),
               private=Optional(False),
               clone_uri=Optional(None),
               landing_rev=Optional(None),
               copy_permissions=Optional(False)):
     """
     Creates a fork of the specified |repo|.
 
     * If the fork_name contains "/", fork will be created inside
       a repository group or nested repository groups
 
       For example "foo/bar/fork-repo" will create fork called "fork-repo"
       inside group "foo/bar". You have to have permissions to access and
       write to the last repository group ("bar" in this example)
 
     This command can only be run using an |authtoken| with minimum
     read permissions of the forked repo, create fork permissions for an user.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set repository name or repository ID.
     :type repoid: str or int
     :param fork_name: Set the fork name, including it's repository group membership.
     :type fork_name: str
     :param owner: Set the fork owner.
     :type owner: str
     :param description: Set the fork description.
     :type description: str
     :param copy_permissions: Copy permissions from parent |repo|. The
         default is False.
     :type copy_permissions: bool
     :param private: Make the fork private. The default is False.
     :type private: bool
     :param landing_rev: Set the landing revision. E.g branch:default, book:dev, rev:abcd
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_for_response>
         api_key : "<api_key>"
         args:     {
                     "repoid" :          "<reponame or repo_id>",
                     "fork_name":        "<forkname>",
                     "owner":            "<username or user_id = Optional(=apiuser)>",
                     "description":      "<description>",
                     "copy_permissions": "<bool>",
                     "private":          "<bool>",
                     "landing_rev":      "<landing_rev>"
                   }
 
     Example error output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: {
                   "msg": "Created fork of `<reponame>` as `<forkname>`",
                   "success": true,
                   "task": "<celery task id or None if done sync>"
                 }
         error:  null
 
     """
 
     repo = get_repo_or_error(repoid)
     repo_name = repo.repo_name
 
     if not has_superadmin_permission(apiuser):
         # check if we have at least read permission for
         # this repo that we fork !
-        _perms = (
-            'repository.admin', 'repository.write', 'repository.read')
+        _perms = ('repository.admin', 'repository.write', 'repository.read')
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
         # check if the regular user has at least fork permissions as well
         if not HasPermissionAnyApi('hg.fork.repository')(user=apiuser):
             raise JSONRPCForbidden()
 
     # check if user can set owner parameter
     owner = validate_set_owner_permissions(apiuser, owner)
 
     description = Optional.extract(description)
     copy_permissions = Optional.extract(copy_permissions)
     clone_uri = Optional.extract(clone_uri)
 
     landing_ref, _label = ScmModel.backend_landing_ref(repo.repo_type)
     ref_choices, _labels = ScmModel().get_repo_landing_revs(request.translate)
     ref_choices = list(set(ref_choices + [landing_ref]))
     landing_commit_ref = Optional.extract(landing_rev) or landing_ref
 
     private = Optional.extract(private)
 
     schema = repo_schema.RepoSchema().bind(
         repo_type_options=rhodecode.BACKENDS.keys(),
         repo_ref_options=ref_choices,
         repo_type=repo.repo_type,
         # user caller
         user=apiuser)
 
     try:
         schema_data = schema.deserialize(dict(
             repo_name=fork_name,
             repo_type=repo.repo_type,
             repo_owner=owner.username,
             repo_description=description,
             repo_landing_commit_ref=landing_commit_ref,
             repo_clone_uri=clone_uri,
             repo_private=private,
             repo_copy_permissions=copy_permissions))
     except validation_schema.Invalid as err:
         raise JSONRPCValidationError(colander_exc=err)
 
     try:
         data = {
             'fork_parent_id': repo.repo_id,
 
             'repo_name': schema_data['repo_group']['repo_name_without_group'],
             'repo_name_full': schema_data['repo_name'],
             'repo_group': schema_data['repo_group']['repo_group_id'],
             'repo_type': schema_data['repo_type'],
             'description': schema_data['repo_description'],
             'private': schema_data['repo_private'],
             'copy_permissions': schema_data['repo_copy_permissions'],
             'landing_rev': schema_data['repo_landing_commit_ref'],
         }
 
         task = RepoModel().create_fork(data, cur_user=owner.user_id)
         # no commit, it's done in RepoModel, or async via celery
         task_id = get_task_id(task)
 
         return {
             'msg': 'Created fork of `%s` as `%s`' % (
                 repo.repo_name, schema_data['repo_name']),
             'success': True,  # cannot return the repo data here since fork
             # can be done async
             'task': task_id
         }
     except Exception:
         log.exception(
             u"Exception while trying to create fork %s",
             schema_data['repo_name'])
         raise JSONRPCError(
             'failed to fork repository `%s` as `%s`' % (
                 repo_name, schema_data['repo_name']))
 
 
 @jsonrpc_method()
 def delete_repo(request, apiuser, repoid, forks=Optional('')):
     """
     Deletes a repository.
 
     * When the `forks` parameter is set it's possible to detach or delete
       forks of deleted repository.
 
     This command can only be run using an |authtoken| with admin
     permissions on the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository ID.
     :type repoid: str or int
     :param forks: Set to `detach` or `delete` forks from the |repo|.
     :type forks: Optional(str)
 
     Example error output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: {
                   "msg": "Deleted repository `<reponame>`",
                   "success": true
                 }
         error:  null
     """
 
     repo = get_repo_or_error(repoid)
     repo_name = repo.repo_name
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     try:
         handle_forks = Optional.extract(forks)
         _forks_msg = ''
         _forks = [f for f in repo.forks]
         if handle_forks == 'detach':
             _forks_msg = ' ' + 'Detached %s forks' % len(_forks)
         elif handle_forks == 'delete':
             _forks_msg = ' ' + 'Deleted %s forks' % len(_forks)
         elif _forks:
             raise JSONRPCError(
                 'Cannot delete `%s` it still contains attached forks' %
                 (repo.repo_name,)
             )
         old_data = repo.get_api_data()
         RepoModel().delete(repo, forks=forks)
 
         repo = audit_logger.RepoWrap(repo_id=None,
                                      repo_name=repo.repo_name)
 
         audit_logger.store_api(
             'repo.delete', action_data={'old_data': old_data},
             user=apiuser, repo=repo)
 
         ScmModel().mark_for_invalidation(repo_name, delete=True)
         Session().commit()
         return {
             'msg': 'Deleted repository `%s`%s' % (repo_name, _forks_msg),
             'success': True
         }
     except Exception:
         log.exception("Exception occurred while trying to delete repo")
         raise JSONRPCError(
             'failed to delete repository `%s`' % (repo_name,)
         )
 
 
 #TODO: marcink, change name ?
 @jsonrpc_method()
 def invalidate_cache(request, apiuser, repoid, delete_keys=Optional(False)):
     """
     Invalidates the cache for the specified repository.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Sets the repository name or repository ID.
     :type repoid: str or int
     :param delete_keys: This deletes the invalidated keys instead of
         just flagging them.
     :type delete_keys: Optional(``True`` | ``False``)
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         'msg': Cache for repository `<repository name>` was invalidated,
         'repository': <repository name>
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error : {
          'Error occurred during cache invalidation action'
       }
 
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin', 'repository.write',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     delete = Optional.extract(delete_keys)
     try:
         ScmModel().mark_for_invalidation(repo.repo_name, delete=delete)
         return {
             'msg': 'Cache for repository `%s` was invalidated' % (repoid,),
             'repository': repo.repo_name
         }
     except Exception:
         log.exception(
             "Exception occurred while trying to invalidate repo cache")
         raise JSONRPCError(
             'Error occurred during cache invalidation action'
         )
 
 
 #TODO: marcink, change name ?
 @jsonrpc_method()
 def lock(request, apiuser, repoid, locked=Optional(None),
          userid=Optional(OAttr('apiuser'))):
     """
     Sets the lock state of the specified |repo| by the given user.
     From more information, see :ref:`repo-locking`.
 
     * If the ``userid`` option is not set, the repository is locked to the
       user who called the method.
     * If the ``locked`` parameter is not set, the current lock state of the
       repository is displayed.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Sets the repository name or repository ID.
     :type repoid: str or int
     :param locked: Sets the lock state.
     :type locked: Optional(``True`` | ``False``)
     :param userid: Set the repository lock to this user.
     :type userid: Optional(str or int)
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         'repo': '<reponame>',
         'locked': <bool: lock state>,
         'locked_since': <int: lock timestamp>,
         'locked_by': <username of person who made the lock>,
         'lock_reason': <str: reason for locking>,
         'lock_state_changed': <bool: True if lock state has been changed in this request>,
         'msg': 'Repo `<reponame>` locked by `<username>` on <timestamp>.'
         or
         'msg': 'Repo `<repository name>` not locked.'
         or
         'msg': 'User `<user name>` set lock state for repo `<repository name>` to `<new lock state>`'
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         'Error occurred locking repository `<reponame>`'
       }
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         # check if we have at least write permission for this repo !
         _perms = ('repository.admin', 'repository.write',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
         # make sure normal user does not pass someone else userid,
         # he is not allowed to do that
         if not isinstance(userid, Optional) and userid != apiuser.user_id:
             raise JSONRPCError('userid is not the same as your user')
 
     if isinstance(userid, Optional):
         userid = apiuser.user_id
 
     user = get_user_or_error(userid)
 
     if isinstance(locked, Optional):
         lockobj = repo.locked
 
         if lockobj[0] is None:
             _d = {
                 'repo': repo.repo_name,
                 'locked': False,
                 'locked_since': None,
                 'locked_by': None,
                 'lock_reason': None,
                 'lock_state_changed': False,
                 'msg': 'Repo `%s` not locked.' % repo.repo_name
             }
             return _d
         else:
             _user_id, _time, _reason = lockobj
             lock_user = get_user_or_error(userid)
             _d = {
                 'repo': repo.repo_name,
                 'locked': True,
                 'locked_since': _time,
                 'locked_by': lock_user.username,
                 'lock_reason': _reason,
                 'lock_state_changed': False,
                 'msg': ('Repo `%s` locked by `%s` on `%s`.'
                         % (repo.repo_name, lock_user.username,
                            json.dumps(time_to_datetime(_time))))
             }
             return _d
 
     # force locked state through a flag
     else:
         locked = str2bool(locked)
         lock_reason = Repository.LOCK_API
         try:
             if locked:
                 lock_time = time.time()
                 Repository.lock(repo, user.user_id, lock_time, lock_reason)
             else:
                 lock_time = None
                 Repository.unlock(repo)
             _d = {
                 'repo': repo.repo_name,
                 'locked': locked,
                 'locked_since': lock_time,
                 'locked_by': user.username,
                 'lock_reason': lock_reason,
                 'lock_state_changed': True,
                 'msg': ('User `%s` set lock state for repo `%s` to `%s`'
                         % (user.username, repo.repo_name, locked))
             }
             return _d
         except Exception:
             log.exception(
                 "Exception occurred while trying to lock repository")
             raise JSONRPCError(
                 'Error occurred locking repository `%s`' % repo.repo_name
             )
 
 
 @jsonrpc_method()
 def comment_commit(
         request, apiuser, repoid, commit_id, message, status=Optional(None),
         comment_type=Optional(ChangesetComment.COMMENT_TYPE_NOTE),
         resolves_comment_id=Optional(None), extra_recipients=Optional([]),
         userid=Optional(OAttr('apiuser')), send_email=Optional(True)):
     """
     Set a commit comment, and optionally change the status of the commit.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository ID.
     :type repoid: str or int
     :param commit_id: Specify the commit_id for which to set a comment.
     :type commit_id: str
     :param message: The comment text.
     :type message: str
     :param status: (**Optional**) status of commit, one of: 'not_reviewed',
         'approved', 'rejected', 'under_review'
     :type status: str
     :param comment_type: Comment type, one of: 'note', 'todo'
     :type comment_type: Optional(str), default: 'note'
     :param resolves_comment_id: id of comment which this one will resolve
     :type resolves_comment_id: Optional(int)
     :param extra_recipients: list of user ids or usernames to add
         notifications for this comment. Acts like a CC for notification
     :type extra_recipients: Optional(list)
     :param userid: Set the user name of the comment creator.
     :type userid: Optional(str or int)
     :param send_email: Define if this comment should also send email notification
     :type send_email: Optional(bool)
 
     Example error output:
 
     .. code-block:: bash
 
         {
             "id" : <id_given_in_input>,
             "result" : {
                 "msg": "Commented on commit `<commit_id>` for repository `<repoid>`",
                 "status_change": null or <status>,
                 "success": true
             },
             "error" :  null
         }
 
     """
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.read', 'repository.write', 'repository.admin')
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     try:
         commit = repo.scm_instance().get_commit(commit_id=commit_id)
         commit_id = commit.raw_id
     except Exception as e:
         log.exception('Failed to fetch commit')
         raise JSONRPCError(safe_str(e))
 
     if isinstance(userid, Optional):
         userid = apiuser.user_id
 
     user = get_user_or_error(userid)
     status = Optional.extract(status)
     comment_type = Optional.extract(comment_type)
     resolves_comment_id = Optional.extract(resolves_comment_id)
     extra_recipients = Optional.extract(extra_recipients)
     send_email = Optional.extract(send_email, binary=True)
 
     allowed_statuses = [x[0] for x in ChangesetStatus.STATUSES]
     if status and status not in allowed_statuses:
         raise JSONRPCError('Bad status, must be on '
                            'of %s got %s' % (allowed_statuses, status,))
 
     if resolves_comment_id:
         comment = ChangesetComment.get(resolves_comment_id)
         if not comment:
             raise JSONRPCError(
                 'Invalid resolves_comment_id `%s` for this commit.'
                 % resolves_comment_id)
         if comment.comment_type != ChangesetComment.COMMENT_TYPE_TODO:
             raise JSONRPCError(
                 'Comment `%s` is wrong type for setting status to resolved.'
                 % resolves_comment_id)
 
     try:
         rc_config = SettingsModel().get_all_settings()
         renderer = rc_config.get('rhodecode_markup_renderer', 'rst')
         status_change_label = ChangesetStatus.get_status_lbl(status)
         comment = CommentsModel().create(
             message, repo, user, commit_id=commit_id,
             status_change=status_change_label,
             status_change_type=status,
             renderer=renderer,
             comment_type=comment_type,
             resolves_comment_id=resolves_comment_id,
             auth_user=apiuser,
             extra_recipients=extra_recipients,
             send_email=send_email
         )
         if status:
             # also do a status change
             try:
                 ChangesetStatusModel().set_status(
                     repo, status, user, comment, revision=commit_id,
                     dont_allow_on_closed_pull_request=True
                 )
             except StatusChangeOnClosedPullRequestError:
                 log.exception(
                     "Exception occurred while trying to change repo commit status")
                 msg = ('Changing status on a commit associated with '
                        'a closed pull request is not allowed')
                 raise JSONRPCError(msg)
 
         CommentsModel().trigger_commit_comment_hook(
             repo, apiuser, 'create',
             data={'comment': comment, 'commit': commit})
 
         Session().commit()
         return {
             'msg': (
                 'Commented on commit `%s` for repository `%s`' % (
                     comment.revision, repo.repo_name)),
             'status_change': status,
             'success': True,
         }
     except JSONRPCError:
         # catch any inside errors, and re-raise them to prevent from
         # below global catch to silence them
         raise
     except Exception:
         log.exception("Exception occurred while trying to comment on commit")
         raise JSONRPCError(
             'failed to set comment on repository `%s`' % (repo.repo_name,)
         )
 
 
 @jsonrpc_method()
 def get_repo_comments(request, apiuser, repoid,
                       commit_id=Optional(None), comment_type=Optional(None),
                       userid=Optional(None)):
     """
     Get all comments for a repository
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository ID.
     :type repoid: str or int
     :param commit_id: Optionally filter the comments by the commit_id
     :type commit_id: Optional(str), default: None
     :param comment_type: Optionally filter the comments by the comment_type
         one of: 'note', 'todo'
     :type comment_type: Optional(str), default: None
     :param userid: Optionally filter the comments by the author of comment
     :type userid: Optional(str or int), Default: None
 
     Example error output:
 
     .. code-block:: bash
 
         {
             "id" : <id_given_in_input>,
             "result" : [
                 {
                   "comment_author": <USER_DETAILS>,
                   "comment_created_on": "2017-02-01T14:38:16.309",
                   "comment_f_path": "file.txt",
                   "comment_id": 282,
                   "comment_lineno": "n1",
                   "comment_resolved_by": null,
                   "comment_status": [],
                   "comment_text": "This file needs a header",
                   "comment_type": "todo",
                   "comment_last_version: 0
                 }
             ],
             "error" :  null
         }
 
     """
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.read', 'repository.write', 'repository.admin')
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     commit_id = Optional.extract(commit_id)
 
     userid = Optional.extract(userid)
     if userid:
         user = get_user_or_error(userid)
     else:
         user = None
 
     comment_type = Optional.extract(comment_type)
     if comment_type and comment_type not in ChangesetComment.COMMENT_TYPES:
         raise JSONRPCError(
                 'comment_type must be one of `{}` got {}'.format(
                     ChangesetComment.COMMENT_TYPES, comment_type)
             )
 
     comments = CommentsModel().get_repository_comments(
         repo=repo, comment_type=comment_type, user=user, commit_id=commit_id)
     return comments
 
 
 @jsonrpc_method()
 def get_comment(request, apiuser, comment_id):
     """
     Get single comment from repository or pull_request
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param comment_id: comment id found in the URL of comment
     :type comment_id: str or int
 
     Example error output:
 
     .. code-block:: bash
 
         {
             "id" : <id_given_in_input>,
             "result" : {
                   "comment_author": <USER_DETAILS>,
                   "comment_created_on": "2017-02-01T14:38:16.309",
                   "comment_f_path": "file.txt",
                   "comment_id": 282,
                   "comment_lineno": "n1",
                   "comment_resolved_by": null,
                   "comment_status": [],
                   "comment_text": "This file needs a header",
                   "comment_type": "todo",
                   "comment_last_version: 0
             },
             "error" :  null
         }
 
     """
 
     comment = ChangesetComment.get(comment_id)
     if not comment:
         raise JSONRPCError('comment `%s` does not exist' % (comment_id,))
 
     perms = ('repository.read', 'repository.write', 'repository.admin')
     has_comment_perm = HasRepoPermissionAnyApi(*perms)\
         (user=apiuser, repo_name=comment.repo.repo_name)
 
     if not has_comment_perm:
         raise JSONRPCError('comment `%s` does not exist' % (comment_id,))
 
     return comment
 
 
 @jsonrpc_method()
 def edit_comment(request, apiuser, message, comment_id, version,
                  userid=Optional(OAttr('apiuser'))):
     """
     Edit comment on the pull request or commit,
     specified by the `comment_id` and version. Initially version should be 0
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param comment_id: Specify the comment_id for editing
     :type comment_id: int
     :param version: version of the comment that will be created, starts from 0
     :type version: int
     :param message: The text content of the comment.
     :type message: str
     :param userid: Comment on the pull request as this user
     :type userid: Optional(str or int)
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result : {
             "comment": "<comment data>",
             "version": "<Integer>",
         },
         error :  null
     """
 
     auth_user = apiuser
     comment = ChangesetComment.get(comment_id)
     if not comment:
         raise JSONRPCError('comment `%s` does not exist' % (comment_id,))
 
     is_super_admin = has_superadmin_permission(apiuser)
     is_repo_admin = HasRepoPermissionAnyApi('repository.admin')\
         (user=apiuser, repo_name=comment.repo.repo_name)
 
     if not isinstance(userid, Optional):
         if is_super_admin or is_repo_admin:
             apiuser = get_user_or_error(userid)
             auth_user = apiuser.AuthUser()
         else:
             raise JSONRPCError('userid is not the same as your user')
 
     comment_author = comment.author.user_id == auth_user.user_id
     if not (comment.immutable is False and (is_super_admin or is_repo_admin) or comment_author):
         raise JSONRPCError("you don't have access to edit this comment")
 
     try:
         comment_history = CommentsModel().edit(
             comment_id=comment_id,
             text=message,
             auth_user=auth_user,
             version=version,
         )
         Session().commit()
     except CommentVersionMismatch:
         raise JSONRPCError(
             'comment ({}) version ({}) mismatch'.format(comment_id, version)
         )
     if not comment_history and not message:
         raise JSONRPCError(
             "comment ({}) can't be changed with empty string".format(comment_id)
         )
 
     if comment.pull_request:
         pull_request = comment.pull_request
         PullRequestModel().trigger_pull_request_hook(
             pull_request, apiuser, 'comment_edit',
             data={'comment': comment})
     else:
         db_repo = comment.repo
         commit_id = comment.revision
         commit = db_repo.get_commit(commit_id)
         CommentsModel().trigger_commit_comment_hook(
             db_repo, apiuser, 'edit',
             data={'comment': comment, 'commit': commit})
 
     data = {
         'comment': comment,
         'version': comment_history.version if comment_history else None,
     }
     return data
 
 
 # TODO(marcink): write this with all required logic for deleting a comments in PR or commits
 # @jsonrpc_method()
 # def delete_comment(request, apiuser, comment_id):
 #     auth_user = apiuser
 #
 #     comment = ChangesetComment.get(comment_id)
 #     if not comment:
 #         raise JSONRPCError('comment `%s` does not exist' % (comment_id,))
 #
 #     is_super_admin = has_superadmin_permission(apiuser)
 #     is_repo_admin = HasRepoPermissionAnyApi('repository.admin')\
 #         (user=apiuser, repo_name=comment.repo.repo_name)
 #
 #     comment_author = comment.author.user_id == auth_user.user_id
 #     if not (comment.immutable is False and (is_super_admin or is_repo_admin) or comment_author):
 #         raise JSONRPCError("you don't have access to edit this comment")
 
 @jsonrpc_method()
 def grant_user_permission(request, apiuser, repoid, userid, perm):
     """
     Grant permissions for the specified user on the given repository,
     or update existing permissions if found.
 
     This command can only be run using an |authtoken| with admin
     permissions on the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository ID.
     :type repoid: str or int
     :param userid: Set the user name.
     :type userid: str
     :param perm: Set the user permissions, using the following format
         ``(repository.(none|read|write|admin))``
     :type perm: str
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: {
                   "msg" : "Granted perm: `<perm>` for user: `<username>` in repo: `<reponame>`",
                   "success": true
                 }
         error:  null
     """
 
     repo = get_repo_or_error(repoid)
     user = get_user_or_error(userid)
     perm = get_perm_or_error(perm)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     perm_additions = [[user.user_id, perm.permission_name, "user"]]
     try:
         changes = RepoModel().update_permissions(
             repo=repo, perm_additions=perm_additions, cur_user=apiuser)
 
         action_data = {
             'added': changes['added'],
             'updated': changes['updated'],
             'deleted': changes['deleted'],
         }
         audit_logger.store_api(
             'repo.edit.permissions', action_data=action_data, user=apiuser, repo=repo)
         Session().commit()
         PermissionModel().flush_user_permission_caches(changes)
 
         return {
             'msg': 'Granted perm: `%s` for user: `%s` in repo: `%s`' % (
                 perm.permission_name, user.username, repo.repo_name
             ),
             'success': True
         }
     except Exception:
         log.exception("Exception occurred while trying edit permissions for repo")
         raise JSONRPCError(
             'failed to edit permission for user: `%s` in repo: `%s`' % (
                 userid, repoid
             )
         )
 
 
 @jsonrpc_method()
 def revoke_user_permission(request, apiuser, repoid, userid):
     """
     Revoke permission for a user on the specified repository.
 
     This command can only be run using an |authtoken| with admin
     permissions on the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository ID.
     :type repoid: str or int
     :param userid: Set the user name of revoked user.
     :type userid: str or int
 
     Example error output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: {
                   "msg" : "Revoked perm for user: `<username>` in repo: `<reponame>`",
                   "success": true
                 }
         error:  null
     """
 
     repo = get_repo_or_error(repoid)
     user = get_user_or_error(userid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     perm_deletions = [[user.user_id, None, "user"]]
     try:
         changes = RepoModel().update_permissions(
             repo=repo, perm_deletions=perm_deletions, cur_user=user)
 
         action_data = {
             'added': changes['added'],
             'updated': changes['updated'],
             'deleted': changes['deleted'],
         }
         audit_logger.store_api(
             'repo.edit.permissions', action_data=action_data, user=apiuser, repo=repo)
         Session().commit()
         PermissionModel().flush_user_permission_caches(changes)
 
         return {
             'msg': 'Revoked perm for user: `%s` in repo: `%s`' % (
                 user.username, repo.repo_name
             ),
             'success': True
         }
     except Exception:
         log.exception("Exception occurred while trying revoke permissions to repo")
         raise JSONRPCError(
             'failed to edit permission for user: `%s` in repo: `%s`' % (
                 userid, repoid
             )
         )
 
 
 @jsonrpc_method()
 def grant_user_group_permission(request, apiuser, repoid, usergroupid, perm):
     """
     Grant permission for a user group on the specified repository,
     or update existing permissions.
 
     This command can only be run using an |authtoken| with admin
     permissions on the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository ID.
     :type repoid: str or int
     :param usergroupid: Specify the ID of the user group.
     :type usergroupid: str or int
     :param perm: Set the user group permissions using the following
         format: (repository.(none|read|write|admin))
     :type perm: str
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         "msg" : "Granted perm: `<perm>` for group: `<usersgroupname>` in repo: `<reponame>`",
         "success": true
 
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         "failed to edit permission for user group: `<usergroup>` in repo `<repo>`'
       }
 
     """
 
     repo = get_repo_or_error(repoid)
     perm = get_perm_or_error(perm)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     user_group = get_user_group_or_error(usergroupid)
     if not has_superadmin_permission(apiuser):
         # check if we have at least read permission for this user group !
         _perms = ('usergroup.read', 'usergroup.write', 'usergroup.admin',)
         if not HasUserGroupPermissionAnyApi(*_perms)(
                 user=apiuser, user_group_name=user_group.users_group_name):
             raise JSONRPCError(
                 'user group `%s` does not exist' % (usergroupid,))
 
     perm_additions = [[user_group.users_group_id, perm.permission_name, "user_group"]]
     try:
         changes = RepoModel().update_permissions(
             repo=repo, perm_additions=perm_additions, cur_user=apiuser)
         action_data = {
             'added': changes['added'],
             'updated': changes['updated'],
             'deleted': changes['deleted'],
         }
         audit_logger.store_api(
             'repo.edit.permissions', action_data=action_data, user=apiuser, repo=repo)
         Session().commit()
         PermissionModel().flush_user_permission_caches(changes)
 
         return {
             'msg': 'Granted perm: `%s` for user group: `%s` in '
                    'repo: `%s`' % (
                        perm.permission_name, user_group.users_group_name,
                        repo.repo_name
                    ),
             'success': True
         }
     except Exception:
         log.exception(
             "Exception occurred while trying change permission on repo")
         raise JSONRPCError(
             'failed to edit permission for user group: `%s` in '
             'repo: `%s`' % (
                 usergroupid, repo.repo_name
             )
         )
 
 
 @jsonrpc_method()
 def revoke_user_group_permission(request, apiuser, repoid, usergroupid):
     """
     Revoke the permissions of a user group on a given repository.
 
     This command can only be run using an |authtoken| with admin
     permissions on the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository ID.
     :type repoid: str or int
     :param usergroupid: Specify the user group ID.
     :type usergroupid: str or int
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: {
                   "msg" : "Revoked perm for group: `<usersgroupname>` in repo: `<reponame>`",
                   "success": true
                 }
         error:  null
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     user_group = get_user_group_or_error(usergroupid)
     if not has_superadmin_permission(apiuser):
         # check if we have at least read permission for this user group !
         _perms = ('usergroup.read', 'usergroup.write', 'usergroup.admin',)
         if not HasUserGroupPermissionAnyApi(*_perms)(
                 user=apiuser, user_group_name=user_group.users_group_name):
             raise JSONRPCError(
                 'user group `%s` does not exist' % (usergroupid,))
 
     perm_deletions = [[user_group.users_group_id, None, "user_group"]]
     try:
         changes = RepoModel().update_permissions(
             repo=repo, perm_deletions=perm_deletions, cur_user=apiuser)
         action_data = {
             'added': changes['added'],
             'updated': changes['updated'],
             'deleted': changes['deleted'],
         }
         audit_logger.store_api(
             'repo.edit.permissions', action_data=action_data, user=apiuser, repo=repo)
         Session().commit()
         PermissionModel().flush_user_permission_caches(changes)
 
         return {
             'msg': 'Revoked perm for user group: `%s` in repo: `%s`' % (
                 user_group.users_group_name, repo.repo_name
             ),
             'success': True
         }
     except Exception:
         log.exception("Exception occurred while trying revoke "
                       "user group permission on repo")
         raise JSONRPCError(
             'failed to edit permission for user group: `%s` in '
             'repo: `%s`' % (
                 user_group.users_group_name, repo.repo_name
             )
         )
 
 
 @jsonrpc_method()
 def pull(request, apiuser, repoid, remote_uri=Optional(None)):
     """
     Triggers a pull on the given repository from a remote location. You
     can use this to keep remote repositories up-to-date.
 
     This command can only be run using an |authtoken| with admin
     rights to the specified repository. For more information,
     see :ref:`config-token-ref`.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository ID.
     :type repoid: str or int
     :param remote_uri: Optional remote URI to pass in for pull
     :type remote_uri: str
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         "msg": "Pulled from url `<remote_url>` on repo `<repository name>`"
         "repository": "<repository name>"
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         "Unable to push changes from `<remote_url>`"
       }
 
     """
 
     repo = get_repo_or_error(repoid)
     remote_uri = Optional.extract(remote_uri)
     remote_uri_display = remote_uri or repo.clone_uri_hidden
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     try:
         ScmModel().pull_changes(
             repo.repo_name, apiuser.username, remote_uri=remote_uri)
         return {
             'msg': 'Pulled from url `%s` on repo `%s`' % (
                 remote_uri_display, repo.repo_name),
             'repository': repo.repo_name
         }
     except Exception:
         log.exception("Exception occurred while trying to "
                       "pull changes from remote location")
         raise JSONRPCError(
             'Unable to pull changes from `%s`' % remote_uri_display
         )
 
 
 @jsonrpc_method()
 def strip(request, apiuser, repoid, revision, branch):
     """
     Strips the given revision from the specified repository.
 
     * This will remove the revision and all of its decendants.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository ID.
     :type repoid: str or int
     :param revision: The revision you wish to strip.
     :type revision: str
     :param branch: The branch from which to strip the revision.
     :type branch: str
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         "msg": "'Stripped commit <commit_hash> from repo `<repository name>`'"
         "repository": "<repository name>"
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         "Unable to strip commit <commit_hash> from repo `<repository name>`"
       }
 
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     try:
         ScmModel().strip(repo, revision, branch)
         audit_logger.store_api(
             'repo.commit.strip', action_data={'commit_id': revision},
             repo=repo,
             user=apiuser, commit=True)
 
         return {
             'msg': 'Stripped commit %s from repo `%s`' % (
                 revision, repo.repo_name),
             'repository': repo.repo_name
         }
     except Exception:
         log.exception("Exception while trying to strip")
         raise JSONRPCError(
             'Unable to strip commit %s from repo `%s`' % (
                 revision, repo.repo_name)
         )
 
 
 @jsonrpc_method()
 def get_repo_settings(request, apiuser, repoid, key=Optional(None)):
     """
     Returns all settings for a repository. If key is given it only returns the
     setting identified by the key or null.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository id.
     :type repoid: str or int
     :param key: Key of the setting to return.
     :type: key: Optional(str)
 
     Example output:
 
     .. code-block:: bash
 
         {
             "error": null,
             "id": 237,
             "result": {
                 "extensions_largefiles": true,
                 "extensions_evolve": true,
                 "hooks_changegroup_push_logger": true,
                 "hooks_changegroup_repo_size": false,
                 "hooks_outgoing_pull_logger": true,
                 "phases_publish": "True",
                 "rhodecode_hg_use_rebase_for_merging": true,
                 "rhodecode_pr_merge_enabled": true,
                 "rhodecode_use_outdated_comments": true
             }
         }
     """
 
-    # Restrict access to this api method to admins only.
+    # Restrict access to this api method to super-admins, and repo admins only.
+    repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
-        raise JSONRPCForbidden()
+        _perms = ('repository.admin',)
+        validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     try:
-        repo = get_repo_or_error(repoid)
         settings_model = VcsSettingsModel(repo=repo)
         settings = settings_model.get_global_settings()
         settings.update(settings_model.get_repo_settings())
 
         # If only a single setting is requested fetch it from all settings.
         key = Optional.extract(key)
         if key is not None:
             settings = settings.get(key, None)
     except Exception:
         msg = 'Failed to fetch settings for repository `{}`'.format(repoid)
         log.exception(msg)
         raise JSONRPCError(msg)
 
     return settings
 
 
 @jsonrpc_method()
 def set_repo_settings(request, apiuser, repoid, settings):
     """
     Update repository settings. Returns true on success.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository id.
     :type repoid: str or int
     :param settings: The new settings for the repository.
     :type: settings: dict
 
     Example output:
 
     .. code-block:: bash
 
         {
             "error": null,
             "id": 237,
             "result": true
         }
     """
-    # Restrict access to this api method to admins only.
+    # Restrict access to this api method to super-admins, and repo admins only.
+    repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
-        raise JSONRPCForbidden()
+        _perms = ('repository.admin',)
+        validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     if type(settings) is not dict:
         raise JSONRPCError('Settings have to be a JSON Object.')
 
     try:
         settings_model = VcsSettingsModel(repo=repoid)
 
         # Merge global, repo and incoming settings.
         new_settings = settings_model.get_global_settings()
         new_settings.update(settings_model.get_repo_settings())
         new_settings.update(settings)
 
         # Update the settings.
         inherit_global_settings = new_settings.get(
             'inherit_global_settings', False)
         settings_model.create_or_update_repo_settings(
             new_settings, inherit_global_settings=inherit_global_settings)
         Session().commit()
     except Exception:
         msg = 'Failed to update settings for repository `{}`'.format(repoid)
         log.exception(msg)
         raise JSONRPCError(msg)
 
     # Indicate success.
     return True
 
 
 @jsonrpc_method()
 def maintenance(request, apiuser, repoid):
     """
     Triggers a maintenance on the given repository.
 
     This command can only be run using an |authtoken| with admin
     rights to the specified repository. For more information,
     see :ref:`config-token-ref`.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository ID.
     :type repoid: str or int
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         "msg": "executed maintenance command",
         "executed_actions": [
            <action_message>, <action_message2>...
         ],
         "repository": "<repository name>"
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         "Unable to execute maintenance on `<reponame>`"
       }
 
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     try:
         maintenance = repo_maintenance.RepoMaintenance()
         executed_actions = maintenance.execute(repo)
 
         return {
             'msg': 'executed maintenance command',
             'executed_actions': executed_actions,
             'repository': repo.repo_name
         }
     except Exception:
         log.exception("Exception occurred while trying to run maintenance")
         raise JSONRPCError(
             'Unable to execute maintenance on `%s`' % repo.repo_name)

rhodecode/apps/channelstream/views.py

@@ -1,177 +1,187 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import uuid
 
 from pyramid.view import view_config
 from pyramid.httpexceptions import HTTPBadRequest, HTTPForbidden, HTTPBadGateway
 
 from rhodecode.apps._base import BaseAppView
 from rhodecode.lib.channelstream import (
     channelstream_request, get_channelstream_server_url,
     ChannelstreamConnectionException,
     ChannelstreamPermissionException,
     check_channel_permissions,
     get_connection_validators,
     get_user_data,
     parse_channels_info,
     update_history_from_logs,
-    STATE_PUBLIC_KEYS)
+    USER_STATE_PUBLIC_KEYS)
 
 from rhodecode.lib.auth import NotAnonymous
 
 log = logging.getLogger(__name__)
 
 
 class ChannelstreamView(BaseAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         self.channelstream_config = \
             self.request.registry.rhodecode_plugins['channelstream']
         if not self.channelstream_config.get('enabled'):
             log.error('Channelstream plugin is disabled')
             raise HTTPBadRequest()
 
         return c
 
     @NotAnonymous()
     @view_config(route_name='channelstream_connect', renderer='json_ext')
     def connect(self):
         """ handle authorization of users trying to connect """
 
         self.load_default_context()
         try:
             json_body = self.request.json_body
         except Exception:
             log.exception('Failed to decode json from request')
             raise HTTPBadRequest()
 
         try:
             channels = check_channel_permissions(
                 json_body.get('channels'),
                 get_connection_validators(self.request.registry))
         except ChannelstreamPermissionException:
             log.error('Incorrect permissions for requested channels')
             raise HTTPForbidden()
 
         user = self._rhodecode_user
         if user.user_id:
             user_data = get_user_data(user.user_id)
         else:
             user_data = {
                 'id': None,
                 'username': None,
                 'first_name': None,
                 'last_name': None,
                 'icon_link': None,
                 'display_name': None,
                 'display_link': None,
             }
-        user_data['permissions'] = self._rhodecode_user.permissions_safe
+
+        #user_data['permissions'] = self._rhodecode_user.permissions_safe
+
         payload = {
             'username': user.username,
             'user_state': user_data,
             'conn_id': str(uuid.uuid4()),
             'channels': channels,
             'channel_configs': {},
-            'state_public_keys': STATE_PUBLIC_KEYS,
+            'state_public_keys': USER_STATE_PUBLIC_KEYS,
             'info': {
                 'exclude_channels': ['broadcast']
             }
         }
         filtered_channels = [channel for channel in channels
                              if channel != 'broadcast']
         for channel in filtered_channels:
             payload['channel_configs'][channel] = {
                 'notify_presence': True,
                 'history_size': 100,
                 'store_history': True,
                 'broadcast_presence_with_user_lists': True
             }
         # connect user to server
         channelstream_url = get_channelstream_server_url(
             self.channelstream_config, '/connect')
         try:
             connect_result = channelstream_request(
                 self.channelstream_config, payload, '/connect')
         except ChannelstreamConnectionException:
             log.exception(
                 'Channelstream service at {} is down'.format(channelstream_url))
             return HTTPBadGateway()
 
+        channel_info = connect_result.get('channels_info')
+        if not channel_info:
+            raise HTTPBadRequest()
+
         connect_result['channels'] = channels
         connect_result['channels_info'] = parse_channels_info(
-            connect_result['channels_info'],
-            include_channel_info=filtered_channels)
+            channel_info, include_channel_info=filtered_channels)
         update_history_from_logs(self.channelstream_config,
                                  filtered_channels, connect_result)
         return connect_result
 
     @NotAnonymous()
     @view_config(route_name='channelstream_subscribe', renderer='json_ext')
     def subscribe(self):
         """ can be used to subscribe specific connection to other channels """
         self.load_default_context()
         try:
             json_body = self.request.json_body
         except Exception:
             log.exception('Failed to decode json from request')
             raise HTTPBadRequest()
         try:
             channels = check_channel_permissions(
                 json_body.get('channels'),
                 get_connection_validators(self.request.registry))
         except ChannelstreamPermissionException:
             log.error('Incorrect permissions for requested channels')
             raise HTTPForbidden()
         payload = {'conn_id': json_body.get('conn_id', ''),
                    'channels': channels,
                    'channel_configs': {},
                    'info': {
                        'exclude_channels': ['broadcast']}
                    }
         filtered_channels = [chan for chan in channels if chan != 'broadcast']
         for channel in filtered_channels:
             payload['channel_configs'][channel] = {
                 'notify_presence': True,
                 'history_size': 100,
                 'store_history': True,
                 'broadcast_presence_with_user_lists': True
             }
 
         channelstream_url = get_channelstream_server_url(
             self.channelstream_config, '/subscribe')
         try:
             connect_result = channelstream_request(
                 self.channelstream_config, payload, '/subscribe')
         except ChannelstreamConnectionException:
             log.exception(
                 'Channelstream service at {} is down'.format(channelstream_url))
             return HTTPBadGateway()
+
+        channel_info = connect_result.get('channels_info')
+        if not channel_info:
+            raise HTTPBadRequest()
+
         # include_channel_info will limit history only to new channel
         # to not overwrite histories on other channels in client
         connect_result['channels_info'] = parse_channels_info(
-            connect_result['channels_info'],
+            channel_info,
             include_channel_info=filtered_channels)
         update_history_from_logs(
             self.channelstream_config, filtered_channels, connect_result)
         return connect_result

rhodecode/apps/file_store/__init__.py

@@ -1,52 +1,52 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import os
 from rhodecode.apps.file_store import config_keys
 from rhodecode.config.middleware import _bool_setting, _string_setting
 
 
 def _sanitize_settings_and_apply_defaults(settings):
     """
     Set defaults, convert to python types and validate settings.
     """
     _bool_setting(settings, config_keys.enabled, 'true')
 
     _string_setting(settings, config_keys.backend, 'local')
 
     default_store = os.path.join(os.path.dirname(settings['__file__']), 'upload_store')
     _string_setting(settings, config_keys.store_path, default_store)
 
 
 def includeme(config):
     settings = config.registry.settings
     _sanitize_settings_and_apply_defaults(settings)
 
     config.add_route(
         name='upload_file',
         pattern='/_file_store/upload')
     config.add_route(
         name='download_file',
-        pattern='/_file_store/download/{fid}')
+        pattern='/_file_store/download/{fid:.*}')
     config.add_route(
         name='download_file_by_token',
-        pattern='/_file_store/token-download/{_auth_token}/{fid}')
+        pattern='/_file_store/token-download/{_auth_token}/{fid:.*}')
 
     # Scan module for configuration decorators.
     config.scan('.views', ignore='.tests')

rhodecode/apps/file_store/backends/local_store.py

@@ -1,240 +1,261 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import time
+import errno
 import shutil
 import hashlib
 
 from rhodecode.lib.ext_json import json
 from rhodecode.apps.file_store import utils
 from rhodecode.apps.file_store.extensions import resolve_extensions
 from rhodecode.apps.file_store.exceptions import (
     FileNotAllowedException, FileOverSizeException)
 
 METADATA_VER = 'v1'
 
 
+def safe_make_dirs(dir_path):
+    if not os.path.exists(dir_path):
+        try:
+            os.makedirs(dir_path)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+            return
+
+
 class LocalFileStorage(object):
 
     @classmethod
+    def apply_counter(cls, counter, filename):
+        name_counted = '%d-%s' % (counter, filename)
+        return name_counted
+
+    @classmethod
     def resolve_name(cls, name, directory):
         """
         Resolves a unique name and the correct path. If a filename
         for that path already exists then a numeric prefix with values > 0 will be
         added, for example test.jpg -> 1-test.jpg etc. initially file would have 0 prefix.
 
         :param name: base name of file
         :param directory: absolute directory path
         """
 
         counter = 0
         while True:
-            name = '%d-%s' % (counter, name)
+            name_counted = cls.apply_counter(counter, name)
 
             # sub_store prefix to optimize disk usage, e.g some_path/ab/final_file
-            sub_store = cls._sub_store_from_filename(name)
+            sub_store = cls._sub_store_from_filename(name_counted)
             sub_store_path = os.path.join(directory, sub_store)
-            if not os.path.exists(sub_store_path):
-                os.makedirs(sub_store_path)
+            safe_make_dirs(sub_store_path)
 
-            path = os.path.join(sub_store_path, name)
+            path = os.path.join(sub_store_path, name_counted)
             if not os.path.exists(path):
-                return name, path
+                return name_counted, path
             counter += 1
 
     @classmethod
     def _sub_store_from_filename(cls, filename):
         return filename[:2]
 
     @classmethod
     def calculate_path_hash(cls, file_path):
         """
         Efficient calculation of file_path sha256 sum
 
         :param file_path:
         :return: sha256sum
         """
         digest = hashlib.sha256()
         with open(file_path, 'rb') as f:
             for chunk in iter(lambda: f.read(1024 * 100), b""):
                 digest.update(chunk)
 
         return digest.hexdigest()
 
     def __init__(self, base_path, extension_groups=None):
 
         """
         Local file storage
 
         :param base_path: the absolute base path where uploads are stored
         :param extension_groups: extensions string
         """
 
         extension_groups = extension_groups or ['any']
         self.base_path = base_path
         self.extensions = resolve_extensions([], groups=extension_groups)
 
     def __repr__(self):
         return '{}@{}'.format(self.__class__, self.base_path)
 
     def store_path(self, filename):
         """
         Returns absolute file path of the filename, joined to the
         base_path.
 
         :param filename: base name of file
         """
-        sub_store = self._sub_store_from_filename(filename)
-        return os.path.join(self.base_path, sub_store, filename)
+        prefix_dir = ''
+        if '/' in filename:
+            prefix_dir, filename = filename.split('/')
+            sub_store = self._sub_store_from_filename(filename)
+        else:
+            sub_store = self._sub_store_from_filename(filename)
+        return os.path.join(self.base_path, prefix_dir, sub_store, filename)
 
     def delete(self, filename):
         """
         Deletes the filename. Filename is resolved with the
         absolute path based on base_path. If file does not exist,
         returns **False**, otherwise **True**
 
         :param filename: base name of file
         """
         if self.exists(filename):
             os.remove(self.store_path(filename))
             return True
         return False
 
     def exists(self, filename):
         """
         Checks if file exists. Resolves filename's absolute
         path based on base_path.
 
-        :param filename: base name of file
+        :param filename: file_uid name of file, e.g 0-f62b2b2d-9708-4079-a071-ec3f958448d4.svg
         """
         return os.path.exists(self.store_path(filename))
 
     def filename_allowed(self, filename, extensions=None):
         """Checks if a filename has an allowed extension
 
         :param filename: base name of file
         :param extensions: iterable of extensions (or self.extensions)
         """
         _, ext = os.path.splitext(filename)
         return self.extension_allowed(ext, extensions)
 
     def extension_allowed(self, ext, extensions=None):
         """
         Checks if an extension is permitted. Both e.g. ".jpg" and
         "jpg" can be passed in. Extension lookup is case-insensitive.
 
         :param ext: extension to check
         :param extensions: iterable of extensions to validate against (or self.extensions)
         """
         def normalize_ext(_ext):
             if _ext.startswith('.'):
                 _ext = _ext[1:]
             return _ext.lower()
 
         extensions = extensions or self.extensions
         if not extensions:
             return True
 
         ext = normalize_ext(ext)
 
         return ext in [normalize_ext(x) for x in extensions]
 
     def save_file(self, file_obj, filename, directory=None, extensions=None,
-                  extra_metadata=None, max_filesize=None, **kwargs):
+                  extra_metadata=None, max_filesize=None, randomized_name=True, **kwargs):
         """
         Saves a file object to the uploads location.
         Returns the resolved filename, i.e. the directory +
         the (randomized/incremented) base name.
 
         :param file_obj: **cgi.FieldStorage** object (or similar)
         :param filename: original filename
         :param directory: relative path of sub-directory
         :param extensions: iterable of allowed extensions, if not default
         :param max_filesize: maximum size of file that should be allowed
+        :param randomized_name: generate random generated UID or fixed based on the filename
         :param extra_metadata: extra JSON metadata to store next to the file with .meta suffix
 
         """
 
         extensions = extensions or self.extensions
 
         if not self.filename_allowed(filename, extensions):
             raise FileNotAllowedException()
 
         if directory:
             dest_directory = os.path.join(self.base_path, directory)
         else:
             dest_directory = self.base_path
 
-        if not os.path.exists(dest_directory):
-            os.makedirs(dest_directory)
+        safe_make_dirs(dest_directory)
 
-        filename = utils.uid_filename(filename)
+        uid_filename = utils.uid_filename(filename, randomized=randomized_name)
 
         # resolve also produces special sub-dir for file optimized store
-        filename, path = self.resolve_name(filename, dest_directory)
+        filename, path = self.resolve_name(uid_filename, dest_directory)
         stored_file_dir = os.path.dirname(path)
 
         file_obj.seek(0)
 
         with open(path, "wb") as dest:
             shutil.copyfileobj(file_obj, dest)
 
         metadata = {}
         if extra_metadata:
             metadata = extra_metadata
 
         size = os.stat(path).st_size
 
         if max_filesize and size > max_filesize:
             # free up the copied file, and raise exc
             os.remove(path)
             raise FileOverSizeException()
 
         file_hash = self.calculate_path_hash(path)
 
-        metadata.update(
-            {"filename": filename,
+        metadata.update({
+            "filename": filename,
              "size": size,
              "time": time.time(),
              "sha256": file_hash,
-             "meta_ver": METADATA_VER})
+             "meta_ver": METADATA_VER
+        })
 
         filename_meta = filename + '.meta'
         with open(os.path.join(stored_file_dir, filename_meta), "wb") as dest_meta:
             dest_meta.write(json.dumps(metadata))
 
         if directory:
             filename = os.path.join(directory, filename)
 
         return filename, metadata
 
     def get_metadata(self, filename):
         """
         Reads JSON stored metadata for a file
 
         :param filename:
         :return:
         """
         filename = self.store_path(filename)
         filename_meta = filename + '.meta'
 
         with open(filename_meta, "rb") as source_meta:
             return json.loads(source_meta.read())

rhodecode/apps/file_store/utils.py

@@ -1,54 +1,58 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 import uuid
-
+import StringIO
 import pathlib2
 
 
 def get_file_storage(settings):
     from rhodecode.apps.file_store.backends.local_store import LocalFileStorage
     from rhodecode.apps.file_store import config_keys
     store_path = settings.get(config_keys.store_path)
     return LocalFileStorage(base_path=store_path)
 
 
 def splitext(filename):
     ext = ''.join(pathlib2.Path(filename).suffixes)
     return filename, ext
 
 
 def uid_filename(filename, randomized=True):
     """
     Generates a randomized or stable (uuid) filename,
     preserving the original extension.
 
     :param filename: the original filename
     :param randomized: define if filename should be stable (sha1 based) or randomized
     """
 
     _, ext = splitext(filename)
     if randomized:
         uid = uuid.uuid4()
     else:
         hash_key = '{}.{}'.format(filename, 'store')
         uid = uuid.uuid5(uuid.NAMESPACE_URL, hash_key)
     return str(uid) + ext.lower()
+
+
+def bytes_to_file_obj(bytes_data):
+    return StringIO.StringIO(bytes_data)

rhodecode/apps/file_store/views.py

@@ -1,195 +1,195 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import logging
 
 from pyramid.view import view_config
 from pyramid.response import FileResponse
 from pyramid.httpexceptions import HTTPFound, HTTPNotFound
 
 from rhodecode.apps._base import BaseAppView
 from rhodecode.apps.file_store import utils
 from rhodecode.apps.file_store.exceptions import (
     FileNotAllowedException, FileOverSizeException)
 
 from rhodecode.lib import helpers as h
 from rhodecode.lib import audit_logger
 from rhodecode.lib.auth import (
     CSRFRequired, NotAnonymous, HasRepoPermissionAny, HasRepoGroupPermissionAny,
     LoginRequired)
 from rhodecode.lib.vcs.conf.mtypes import get_mimetypes_db
 from rhodecode.model.db import Session, FileStore, UserApiKeys
 
 log = logging.getLogger(__name__)
 
 
 class FileStoreView(BaseAppView):
     upload_key = 'store_file'
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         self.storage = utils.get_file_storage(self.request.registry.settings)
         return c
 
     def _guess_type(self, file_name):
         """
         Our own type guesser for mimetypes using the rich DB
         """
         if not hasattr(self, 'db'):
             self.db = get_mimetypes_db()
         _content_type, _encoding = self.db.guess_type(file_name, strict=False)
         return _content_type, _encoding
 
     def _serve_file(self, file_uid):
 
         if not self.storage.exists(file_uid):
             store_path = self.storage.store_path(file_uid)
             log.debug('File with FID:%s not found in the store under `%s`',
                       file_uid, store_path)
             raise HTTPNotFound()
 
-        db_obj = FileStore().query().filter(FileStore.file_uid == file_uid).scalar()
+        db_obj = FileStore.get_by_store_uid(file_uid, safe=True)
         if not db_obj:
             raise HTTPNotFound()
 
         # private upload for user
         if db_obj.check_acl and db_obj.scope_user_id:
             log.debug('Artifact: checking scope access for bound artifact user: `%s`',
                       db_obj.scope_user_id)
             user = db_obj.user
             if self._rhodecode_db_user.user_id != user.user_id:
                 log.warning('Access to file store object forbidden')
                 raise HTTPNotFound()
 
         # scoped to repository permissions
         if db_obj.check_acl and db_obj.scope_repo_id:
             log.debug('Artifact: checking scope access for bound artifact repo: `%s`',
                       db_obj.scope_repo_id)
             repo = db_obj.repo
             perm_set = ['repository.read', 'repository.write', 'repository.admin']
             has_perm = HasRepoPermissionAny(*perm_set)(repo.repo_name, 'FileStore check')
             if not has_perm:
                 log.warning('Access to file store object `%s` forbidden', file_uid)
                 raise HTTPNotFound()
 
         # scoped to repository group permissions
         if db_obj.check_acl and db_obj.scope_repo_group_id:
             log.debug('Artifact: checking scope access for bound artifact repo group: `%s`',
                       db_obj.scope_repo_group_id)
             repo_group = db_obj.repo_group
             perm_set = ['group.read', 'group.write', 'group.admin']
             has_perm = HasRepoGroupPermissionAny(*perm_set)(repo_group.group_name, 'FileStore check')
             if not has_perm:
                 log.warning('Access to file store object `%s` forbidden', file_uid)
                 raise HTTPNotFound()
 
         FileStore.bump_access_counter(file_uid)
 
         file_path = self.storage.store_path(file_uid)
         content_type = 'application/octet-stream'
         content_encoding = None
 
         _content_type, _encoding = self._guess_type(file_path)
         if _content_type:
             content_type = _content_type
 
         # For file store we don't submit any session data, this logic tells the
         # Session lib to skip it
         setattr(self.request, '_file_response', True)
         return FileResponse(file_path, request=self.request,
                             content_type=content_type, content_encoding=content_encoding)
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     @view_config(route_name='upload_file', request_method='POST', renderer='json_ext')
     def upload_file(self):
         self.load_default_context()
         file_obj = self.request.POST.get(self.upload_key)
 
         if file_obj is None:
             return {'store_fid': None,
                     'access_path': None,
                     'error': '{} data field is missing'.format(self.upload_key)}
 
         if not hasattr(file_obj, 'filename'):
             return {'store_fid': None,
                     'access_path': None,
                     'error': 'filename cannot be read from the data field'}
 
         filename = file_obj.filename
 
         metadata = {
             'user_uploaded': {'username': self._rhodecode_user.username,
                               'user_id': self._rhodecode_user.user_id,
                               'ip': self._rhodecode_user.ip_addr}}
         try:
             store_uid, metadata = self.storage.save_file(
                 file_obj.file, filename, extra_metadata=metadata)
         except FileNotAllowedException:
             return {'store_fid': None,
                     'access_path': None,
                     'error': 'File {} is not allowed.'.format(filename)}
 
         except FileOverSizeException:
             return {'store_fid': None,
                     'access_path': None,
                     'error': 'File {} is exceeding allowed limit.'.format(filename)}
 
         try:
             entry = FileStore.create(
                 file_uid=store_uid, filename=metadata["filename"],
                 file_hash=metadata["sha256"], file_size=metadata["size"],
                 file_description=u'upload attachment',
                 check_acl=False, user_id=self._rhodecode_user.user_id
             )
             Session().add(entry)
             Session().commit()
             log.debug('Stored upload in DB as %s', entry)
         except Exception:
             log.exception('Failed to store file %s', filename)
             return {'store_fid': None,
                     'access_path': None,
                     'error': 'File {} failed to store in DB.'.format(filename)}
 
         return {'store_fid': store_uid,
                 'access_path': h.route_path('download_file', fid=store_uid)}
 
     # ACL is checked by scopes, if no scope the file is accessible to all
     @view_config(route_name='download_file')
     def download_file(self):
         self.load_default_context()
         file_uid = self.request.matchdict['fid']
         log.debug('Requesting FID:%s from store %s', file_uid, self.storage)
         return self._serve_file(file_uid)
 
     # in addition to @LoginRequired ACL is checked by scopes
     @LoginRequired(auth_token_access=[UserApiKeys.ROLE_ARTIFACT_DOWNLOAD])
     @NotAnonymous()
     @view_config(route_name='download_file_by_token')
     def download_file_by_token(self):
         """
         Special view that allows to access the download file by special URL that
         is stored inside the URL.
 
         http://example.com/_file_store/token-download/TOKEN/FILE_UID
         """
         self.load_default_context()
         file_uid = self.request.matchdict['fid']
         return self._serve_file(file_uid)

rhodecode/apps/repository/__init__.py

@@ -1,533 +1,543 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 from rhodecode.apps._base import add_route_with_slash
 
 
 def includeme(config):
 
     # repo creating checks, special cases that aren't repo routes
     config.add_route(
         name='repo_creating',
         pattern='/{repo_name:.*?[^/]}/repo_creating')
 
     config.add_route(
         name='repo_creating_check',
         pattern='/{repo_name:.*?[^/]}/repo_creating_check')
 
     # Summary
     # NOTE(marcink): one additional route is defined in very bottom, catch
     # all pattern
     config.add_route(
         name='repo_summary_explicit',
         pattern='/{repo_name:.*?[^/]}/summary', repo_route=True)
     config.add_route(
         name='repo_summary_commits',
         pattern='/{repo_name:.*?[^/]}/summary-commits', repo_route=True)
 
     # Commits
     config.add_route(
         name='repo_commit',
         pattern='/{repo_name:.*?[^/]}/changeset/{commit_id}', repo_route=True)
 
     config.add_route(
         name='repo_commit_children',
         pattern='/{repo_name:.*?[^/]}/changeset_children/{commit_id}', repo_route=True)
 
     config.add_route(
         name='repo_commit_parents',
         pattern='/{repo_name:.*?[^/]}/changeset_parents/{commit_id}', repo_route=True)
 
     config.add_route(
         name='repo_commit_raw',
         pattern='/{repo_name:.*?[^/]}/changeset-diff/{commit_id}', repo_route=True)
 
     config.add_route(
         name='repo_commit_patch',
         pattern='/{repo_name:.*?[^/]}/changeset-patch/{commit_id}', repo_route=True)
 
     config.add_route(
         name='repo_commit_download',
         pattern='/{repo_name:.*?[^/]}/changeset-download/{commit_id}', repo_route=True)
 
     config.add_route(
         name='repo_commit_data',
         pattern='/{repo_name:.*?[^/]}/changeset-data/{commit_id}', repo_route=True)
 
     config.add_route(
         name='repo_commit_comment_create',
         pattern='/{repo_name:.*?[^/]}/changeset/{commit_id}/comment/create', repo_route=True)
 
     config.add_route(
         name='repo_commit_comment_preview',
         pattern='/{repo_name:.*?[^/]}/changeset/{commit_id}/comment/preview', repo_route=True)
 
     config.add_route(
         name='repo_commit_comment_history_view',
         pattern='/{repo_name:.*?[^/]}/changeset/{commit_id}/comment/{comment_history_id}/history_view', repo_route=True)
 
     config.add_route(
         name='repo_commit_comment_attachment_upload',
         pattern='/{repo_name:.*?[^/]}/changeset/{commit_id}/comment/attachment_upload', repo_route=True)
 
     config.add_route(
         name='repo_commit_comment_delete',
         pattern='/{repo_name:.*?[^/]}/changeset/{commit_id}/comment/{comment_id}/delete', repo_route=True)
 
     config.add_route(
         name='repo_commit_comment_edit',
         pattern='/{repo_name:.*?[^/]}/changeset/{commit_id}/comment/{comment_id}/edit', repo_route=True)
 
     # still working url for backward compat.
     config.add_route(
         name='repo_commit_raw_deprecated',
         pattern='/{repo_name:.*?[^/]}/raw-changeset/{commit_id}', repo_route=True)
 
     # Files
     config.add_route(
         name='repo_archivefile',
         pattern='/{repo_name:.*?[^/]}/archive/{fname:.*}', repo_route=True)
 
     config.add_route(
         name='repo_files_diff',
         pattern='/{repo_name:.*?[^/]}/diff/{f_path:.*}', repo_route=True)
     config.add_route(  # legacy route to make old links work
         name='repo_files_diff_2way_redirect',
         pattern='/{repo_name:.*?[^/]}/diff-2way/{f_path:.*}', repo_route=True)
 
     config.add_route(
         name='repo_files',
         pattern='/{repo_name:.*?[^/]}/files/{commit_id}/{f_path:.*}', repo_route=True)
     config.add_route(
         name='repo_files:default_path',
         pattern='/{repo_name:.*?[^/]}/files/{commit_id}/', repo_route=True)
     config.add_route(
         name='repo_files:default_commit',
         pattern='/{repo_name:.*?[^/]}/files', repo_route=True)
 
     config.add_route(
         name='repo_files:rendered',
         pattern='/{repo_name:.*?[^/]}/render/{commit_id}/{f_path:.*}', repo_route=True)
 
     config.add_route(
         name='repo_files:annotated',
         pattern='/{repo_name:.*?[^/]}/annotate/{commit_id}/{f_path:.*}', repo_route=True)
     config.add_route(
         name='repo_files:annotated_previous',
         pattern='/{repo_name:.*?[^/]}/annotate-previous/{commit_id}/{f_path:.*}', repo_route=True)
 
     config.add_route(
         name='repo_nodetree_full',
         pattern='/{repo_name:.*?[^/]}/nodetree_full/{commit_id}/{f_path:.*}', repo_route=True)
     config.add_route(
         name='repo_nodetree_full:default_path',
         pattern='/{repo_name:.*?[^/]}/nodetree_full/{commit_id}/', repo_route=True)
 
     config.add_route(
         name='repo_files_nodelist',
         pattern='/{repo_name:.*?[^/]}/nodelist/{commit_id}/{f_path:.*}', repo_route=True)
 
     config.add_route(
         name='repo_file_raw',
         pattern='/{repo_name:.*?[^/]}/raw/{commit_id}/{f_path:.*}', repo_route=True)
 
     config.add_route(
         name='repo_file_download',
         pattern='/{repo_name:.*?[^/]}/download/{commit_id}/{f_path:.*}', repo_route=True)
     config.add_route(  # backward compat to keep old links working
         name='repo_file_download:legacy',
         pattern='/{repo_name:.*?[^/]}/rawfile/{commit_id}/{f_path:.*}',
         repo_route=True)
 
     config.add_route(
         name='repo_file_history',
         pattern='/{repo_name:.*?[^/]}/history/{commit_id}/{f_path:.*}', repo_route=True)
 
     config.add_route(
         name='repo_file_authors',
         pattern='/{repo_name:.*?[^/]}/authors/{commit_id}/{f_path:.*}', repo_route=True)
 
     config.add_route(
         name='repo_files_check_head',
         pattern='/{repo_name:.*?[^/]}/check_head/{commit_id}/{f_path:.*}',
         repo_route=True)
     config.add_route(
         name='repo_files_remove_file',
         pattern='/{repo_name:.*?[^/]}/remove_file/{commit_id}/{f_path:.*}',
         repo_route=True)
     config.add_route(
         name='repo_files_delete_file',
         pattern='/{repo_name:.*?[^/]}/delete_file/{commit_id}/{f_path:.*}',
         repo_route=True)
     config.add_route(
         name='repo_files_edit_file',
         pattern='/{repo_name:.*?[^/]}/edit_file/{commit_id}/{f_path:.*}',
         repo_route=True)
     config.add_route(
         name='repo_files_update_file',
         pattern='/{repo_name:.*?[^/]}/update_file/{commit_id}/{f_path:.*}',
         repo_route=True)
     config.add_route(
         name='repo_files_add_file',
         pattern='/{repo_name:.*?[^/]}/add_file/{commit_id}/{f_path:.*}',
         repo_route=True)
     config.add_route(
         name='repo_files_upload_file',
         pattern='/{repo_name:.*?[^/]}/upload_file/{commit_id}/{f_path:.*}',
         repo_route=True)
     config.add_route(
         name='repo_files_create_file',
         pattern='/{repo_name:.*?[^/]}/create_file/{commit_id}/{f_path:.*}',
         repo_route=True)
 
     # Refs data
     config.add_route(
         name='repo_refs_data',
         pattern='/{repo_name:.*?[^/]}/refs-data', repo_route=True)
 
     config.add_route(
         name='repo_refs_changelog_data',
         pattern='/{repo_name:.*?[^/]}/refs-data-changelog', repo_route=True)
 
     config.add_route(
         name='repo_stats',
         pattern='/{repo_name:.*?[^/]}/repo_stats/{commit_id}', repo_route=True)
 
     # Commits
     config.add_route(
         name='repo_commits',
         pattern='/{repo_name:.*?[^/]}/commits', repo_route=True)
     config.add_route(
         name='repo_commits_file',
         pattern='/{repo_name:.*?[^/]}/commits/{commit_id}/{f_path:.*}', repo_route=True)
     config.add_route(
         name='repo_commits_elements',
         pattern='/{repo_name:.*?[^/]}/commits_elements', repo_route=True)
     config.add_route(
         name='repo_commits_elements_file',
         pattern='/{repo_name:.*?[^/]}/commits_elements/{commit_id}/{f_path:.*}', repo_route=True)
 
     # Changelog (old deprecated name for commits page)
     config.add_route(
         name='repo_changelog',
         pattern='/{repo_name:.*?[^/]}/changelog', repo_route=True)
     config.add_route(
         name='repo_changelog_file',
         pattern='/{repo_name:.*?[^/]}/changelog/{commit_id}/{f_path:.*}', repo_route=True)
 
     # Compare
     config.add_route(
         name='repo_compare_select',
         pattern='/{repo_name:.*?[^/]}/compare', repo_route=True)
 
     config.add_route(
         name='repo_compare',
         pattern='/{repo_name:.*?[^/]}/compare/{source_ref_type}@{source_ref:.*?}...{target_ref_type}@{target_ref:.*?}', repo_route=True)
 
     # Tags
     config.add_route(
         name='tags_home',
         pattern='/{repo_name:.*?[^/]}/tags', repo_route=True)
 
     # Branches
     config.add_route(
         name='branches_home',
         pattern='/{repo_name:.*?[^/]}/branches', repo_route=True)
 
     # Bookmarks
     config.add_route(
         name='bookmarks_home',
         pattern='/{repo_name:.*?[^/]}/bookmarks', repo_route=True)
 
     # Forks
     config.add_route(
         name='repo_fork_new',
         pattern='/{repo_name:.*?[^/]}/fork', repo_route=True,
         repo_forbid_when_archived=True,
         repo_accepted_types=['hg', 'git'])
 
     config.add_route(
         name='repo_fork_create',
         pattern='/{repo_name:.*?[^/]}/fork/create', repo_route=True,
         repo_forbid_when_archived=True,
         repo_accepted_types=['hg', 'git'])
 
     config.add_route(
         name='repo_forks_show_all',
         pattern='/{repo_name:.*?[^/]}/forks', repo_route=True,
         repo_accepted_types=['hg', 'git'])
     config.add_route(
         name='repo_forks_data',
         pattern='/{repo_name:.*?[^/]}/forks/data', repo_route=True,
         repo_accepted_types=['hg', 'git'])
 
     # Pull Requests
     config.add_route(
         name='pullrequest_show',
         pattern='/{repo_name:.*?[^/]}/pull-request/{pull_request_id:\d+}',
         repo_route=True)
 
     config.add_route(
         name='pullrequest_show_all',
         pattern='/{repo_name:.*?[^/]}/pull-request',
         repo_route=True, repo_accepted_types=['hg', 'git'])
 
     config.add_route(
         name='pullrequest_show_all_data',
         pattern='/{repo_name:.*?[^/]}/pull-request-data',
         repo_route=True, repo_accepted_types=['hg', 'git'])
 
     config.add_route(
         name='pullrequest_repo_refs',
         pattern='/{repo_name:.*?[^/]}/pull-request/refs/{target_repo_name:.*?[^/]}',
         repo_route=True)
 
     config.add_route(
         name='pullrequest_repo_targets',
         pattern='/{repo_name:.*?[^/]}/pull-request/repo-targets',
         repo_route=True)
 
     config.add_route(
         name='pullrequest_new',
         pattern='/{repo_name:.*?[^/]}/pull-request/new',
         repo_route=True, repo_accepted_types=['hg', 'git'],
         repo_forbid_when_archived=True)
 
     config.add_route(
         name='pullrequest_create',
         pattern='/{repo_name:.*?[^/]}/pull-request/create',
         repo_route=True, repo_accepted_types=['hg', 'git'],
         repo_forbid_when_archived=True)
 
     config.add_route(
         name='pullrequest_update',
         pattern='/{repo_name:.*?[^/]}/pull-request/{pull_request_id:\d+}/update',
         repo_route=True, repo_forbid_when_archived=True)
 
     config.add_route(
         name='pullrequest_merge',
         pattern='/{repo_name:.*?[^/]}/pull-request/{pull_request_id:\d+}/merge',
         repo_route=True, repo_forbid_when_archived=True)
 
     config.add_route(
         name='pullrequest_delete',
         pattern='/{repo_name:.*?[^/]}/pull-request/{pull_request_id:\d+}/delete',
         repo_route=True, repo_forbid_when_archived=True)
 
     config.add_route(
         name='pullrequest_comment_create',
         pattern='/{repo_name:.*?[^/]}/pull-request/{pull_request_id:\d+}/comment',
         repo_route=True)
 
     config.add_route(
         name='pullrequest_comment_edit',
         pattern='/{repo_name:.*?[^/]}/pull-request/{pull_request_id:\d+}/comment/{comment_id}/edit',
         repo_route=True, repo_accepted_types=['hg', 'git'])
 
     config.add_route(
         name='pullrequest_comment_delete',
         pattern='/{repo_name:.*?[^/]}/pull-request/{pull_request_id:\d+}/comment/{comment_id}/delete',
         repo_route=True, repo_accepted_types=['hg', 'git'])
 
+    config.add_route(
+        name='pullrequest_comments',
+        pattern='/{repo_name:.*?[^/]}/pull-request/{pull_request_id:\d+}/comments',
+        repo_route=True)
+
+    config.add_route(
+        name='pullrequest_todos',
+        pattern='/{repo_name:.*?[^/]}/pull-request/{pull_request_id:\d+}/todos',
+        repo_route=True)
+
     # Artifacts, (EE feature)
     config.add_route(
         name='repo_artifacts_list',
         pattern='/{repo_name:.*?[^/]}/artifacts', repo_route=True)
 
     # Settings
     config.add_route(
         name='edit_repo',
         pattern='/{repo_name:.*?[^/]}/settings', repo_route=True)
     # update is POST on edit_repo
 
     # Settings advanced
     config.add_route(
         name='edit_repo_advanced',
         pattern='/{repo_name:.*?[^/]}/settings/advanced', repo_route=True)
     config.add_route(
         name='edit_repo_advanced_archive',
         pattern='/{repo_name:.*?[^/]}/settings/advanced/archive', repo_route=True)
     config.add_route(
         name='edit_repo_advanced_delete',
         pattern='/{repo_name:.*?[^/]}/settings/advanced/delete', repo_route=True)
     config.add_route(
         name='edit_repo_advanced_locking',
         pattern='/{repo_name:.*?[^/]}/settings/advanced/locking', repo_route=True)
     config.add_route(
         name='edit_repo_advanced_journal',
         pattern='/{repo_name:.*?[^/]}/settings/advanced/journal', repo_route=True)
     config.add_route(
         name='edit_repo_advanced_fork',
         pattern='/{repo_name:.*?[^/]}/settings/advanced/fork', repo_route=True)
 
     config.add_route(
         name='edit_repo_advanced_hooks',
         pattern='/{repo_name:.*?[^/]}/settings/advanced/hooks', repo_route=True)
 
     # Caches
     config.add_route(
         name='edit_repo_caches',
         pattern='/{repo_name:.*?[^/]}/settings/caches', repo_route=True)
 
     # Permissions
     config.add_route(
         name='edit_repo_perms',
         pattern='/{repo_name:.*?[^/]}/settings/permissions', repo_route=True)
 
     config.add_route(
         name='edit_repo_perms_set_private',
         pattern='/{repo_name:.*?[^/]}/settings/permissions/set_private', repo_route=True)
 
     # Permissions Branch (EE feature)
     config.add_route(
         name='edit_repo_perms_branch',
         pattern='/{repo_name:.*?[^/]}/settings/branch_permissions', repo_route=True)
     config.add_route(
         name='edit_repo_perms_branch_delete',
         pattern='/{repo_name:.*?[^/]}/settings/branch_permissions/{rule_id}/delete',
         repo_route=True)
 
     # Maintenance
     config.add_route(
         name='edit_repo_maintenance',
         pattern='/{repo_name:.*?[^/]}/settings/maintenance', repo_route=True)
 
     config.add_route(
         name='edit_repo_maintenance_execute',
         pattern='/{repo_name:.*?[^/]}/settings/maintenance/execute', repo_route=True)
 
     # Fields
     config.add_route(
         name='edit_repo_fields',
         pattern='/{repo_name:.*?[^/]}/settings/fields', repo_route=True)
     config.add_route(
         name='edit_repo_fields_create',
         pattern='/{repo_name:.*?[^/]}/settings/fields/create', repo_route=True)
     config.add_route(
         name='edit_repo_fields_delete',
         pattern='/{repo_name:.*?[^/]}/settings/fields/{field_id}/delete', repo_route=True)
 
     # Locking
     config.add_route(
         name='repo_edit_toggle_locking',
         pattern='/{repo_name:.*?[^/]}/settings/toggle_locking', repo_route=True)
 
     # Remote
     config.add_route(
         name='edit_repo_remote',
         pattern='/{repo_name:.*?[^/]}/settings/remote', repo_route=True)
     config.add_route(
         name='edit_repo_remote_pull',
         pattern='/{repo_name:.*?[^/]}/settings/remote/pull', repo_route=True)
     config.add_route(
         name='edit_repo_remote_push',
         pattern='/{repo_name:.*?[^/]}/settings/remote/push', repo_route=True)
 
     # Statistics
     config.add_route(
         name='edit_repo_statistics',
         pattern='/{repo_name:.*?[^/]}/settings/statistics', repo_route=True)
     config.add_route(
         name='edit_repo_statistics_reset',
         pattern='/{repo_name:.*?[^/]}/settings/statistics/update', repo_route=True)
 
     # Issue trackers
     config.add_route(
         name='edit_repo_issuetracker',
         pattern='/{repo_name:.*?[^/]}/settings/issue_trackers', repo_route=True)
     config.add_route(
         name='edit_repo_issuetracker_test',
         pattern='/{repo_name:.*?[^/]}/settings/issue_trackers/test', repo_route=True)
     config.add_route(
         name='edit_repo_issuetracker_delete',
         pattern='/{repo_name:.*?[^/]}/settings/issue_trackers/delete', repo_route=True)
     config.add_route(
         name='edit_repo_issuetracker_update',
         pattern='/{repo_name:.*?[^/]}/settings/issue_trackers/update', repo_route=True)
 
     # VCS Settings
     config.add_route(
         name='edit_repo_vcs',
         pattern='/{repo_name:.*?[^/]}/settings/vcs', repo_route=True)
     config.add_route(
         name='edit_repo_vcs_update',
         pattern='/{repo_name:.*?[^/]}/settings/vcs/update', repo_route=True)
 
     # svn pattern
     config.add_route(
         name='edit_repo_vcs_svn_pattern_delete',
         pattern='/{repo_name:.*?[^/]}/settings/vcs/svn_pattern/delete', repo_route=True)
 
     # Repo Review Rules (EE feature)
     config.add_route(
         name='repo_reviewers',
         pattern='/{repo_name:.*?[^/]}/settings/review/rules', repo_route=True)
 
     config.add_route(
         name='repo_default_reviewers_data',
         pattern='/{repo_name:.*?[^/]}/settings/review/default-reviewers', repo_route=True)
 
     # Repo Automation (EE feature)
     config.add_route(
         name='repo_automation',
         pattern='/{repo_name:.*?[^/]}/settings/automation', repo_route=True)
 
     # Strip
     config.add_route(
         name='edit_repo_strip',
         pattern='/{repo_name:.*?[^/]}/settings/strip', repo_route=True)
 
     config.add_route(
         name='strip_check',
         pattern='/{repo_name:.*?[^/]}/settings/strip_check', repo_route=True)
 
     config.add_route(
         name='strip_execute',
         pattern='/{repo_name:.*?[^/]}/settings/strip_execute', repo_route=True)
 
     # Audit logs
     config.add_route(
         name='edit_repo_audit_logs',
         pattern='/{repo_name:.*?[^/]}/settings/audit_logs', repo_route=True)
 
     # ATOM/RSS Feed, shouldn't contain slashes for outlook compatibility
     config.add_route(
         name='rss_feed_home',
         pattern='/{repo_name:.*?[^/]}/feed-rss', repo_route=True)
 
     config.add_route(
         name='atom_feed_home',
         pattern='/{repo_name:.*?[^/]}/feed-atom', repo_route=True)
 
     config.add_route(
         name='rss_feed_home_old',
         pattern='/{repo_name:.*?[^/]}/feed/rss', repo_route=True)
 
     config.add_route(
         name='atom_feed_home_old',
         pattern='/{repo_name:.*?[^/]}/feed/atom', repo_route=True)
 
     # NOTE(marcink): needs to be at the end for catch-all
     add_route_with_slash(
         config,
         name='repo_summary',
         pattern='/{repo_name:.*?[^/]}', repo_route=True)
 
     # Scan module for configuration decorators.
     config.scan('.views', ignore='.tests')

rhodecode/apps/repository/tests/test_repo_commit_comments.py

@@ -1,507 +1,494 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import pytest
 
 from rhodecode.tests import TestController
 
 from rhodecode.model.db import ChangesetComment, Notification
 from rhodecode.model.meta import Session
 from rhodecode.lib import helpers as h
 
 
 def route_path(name, params=None, **kwargs):
     import urllib
 
     base_url = {
         'repo_commit': '/{repo_name}/changeset/{commit_id}',
         'repo_commit_comment_create': '/{repo_name}/changeset/{commit_id}/comment/create',
         'repo_commit_comment_preview': '/{repo_name}/changeset/{commit_id}/comment/preview',
         'repo_commit_comment_delete': '/{repo_name}/changeset/{commit_id}/comment/{comment_id}/delete',
         'repo_commit_comment_edit': '/{repo_name}/changeset/{commit_id}/comment/{comment_id}/edit',
     }[name].format(**kwargs)
 
     if params:
         base_url = '{}?{}'.format(base_url, urllib.urlencode(params))
     return base_url
 
 
 @pytest.mark.backends("git", "hg", "svn")
 class TestRepoCommitCommentsView(TestController):
 
     @pytest.fixture(autouse=True)
     def prepare(self, request, baseapp):
         for x in ChangesetComment.query().all():
             Session().delete(x)
         Session().commit()
 
         for x in Notification.query().all():
             Session().delete(x)
         Session().commit()
 
         request.addfinalizer(self.cleanup)
 
     def cleanup(self):
         for x in ChangesetComment.query().all():
             Session().delete(x)
         Session().commit()
 
         for x in Notification.query().all():
             Session().delete(x)
         Session().commit()
 
     @pytest.mark.parametrize('comment_type', ChangesetComment.COMMENT_TYPES)
     def test_create(self, comment_type, backend):
         self.log_user()
         commit = backend.repo.get_commit('300')
         commit_id = commit.raw_id
         text = u'CommentOnCommit'
 
         params = {'text': text, 'csrf_token': self.csrf_token,
                   'comment_type': comment_type}
         self.app.post(
             route_path('repo_commit_comment_create',
                        repo_name=backend.repo_name, commit_id=commit_id),
             params=params)
 
         response = self.app.get(
             route_path('repo_commit',
                        repo_name=backend.repo_name, commit_id=commit_id))
 
         # test DB
         assert ChangesetComment.query().count() == 1
         assert_comment_links(response, ChangesetComment.query().count(), 0)
 
         assert Notification.query().count() == 1
         assert ChangesetComment.query().count() == 1
 
         notification = Notification.query().all()[0]
 
         comment_id = ChangesetComment.query().first().comment_id
         assert notification.type_ == Notification.TYPE_CHANGESET_COMMENT
 
         author = notification.created_by_user.username_and_name
         sbj = '@{0} left a {1} on commit `{2}` in the `{3}` repository'.format(
             author, comment_type, h.show_id(commit), backend.repo_name)
         assert sbj == notification.subject
 
         lnk = (u'/{0}/changeset/{1}#comment-{2}'.format(
             backend.repo_name, commit_id, comment_id))
         assert lnk in notification.body
 
     @pytest.mark.parametrize('comment_type', ChangesetComment.COMMENT_TYPES)
     def test_create_inline(self, comment_type, backend):
         self.log_user()
         commit = backend.repo.get_commit('300')
         commit_id = commit.raw_id
         text = u'CommentOnCommit'
         f_path = 'vcs/web/simplevcs/views/repository.py'
         line = 'n1'
 
         params = {'text': text, 'f_path': f_path, 'line': line,
                   'comment_type': comment_type,
                   'csrf_token': self.csrf_token}
 
         self.app.post(
             route_path('repo_commit_comment_create',
                        repo_name=backend.repo_name, commit_id=commit_id),
             params=params)
 
         response = self.app.get(
             route_path('repo_commit',
                        repo_name=backend.repo_name, commit_id=commit_id))
 
         # test DB
         assert ChangesetComment.query().count() == 1
         assert_comment_links(response, 0, ChangesetComment.query().count())
 
         if backend.alias == 'svn':
             response.mustcontain(
                 '''data-f-path="vcs/commands/summary.py" '''
                 '''data-anchor-id="c-300-ad05457a43f8"'''
             )
         if backend.alias == 'git':
             response.mustcontain(
                 '''data-f-path="vcs/backends/hg.py" '''
                 '''data-anchor-id="c-883e775e89ea-9c390eb52cd6"'''
             )
 
         if backend.alias == 'hg':
             response.mustcontain(
                 '''data-f-path="vcs/backends/hg.py" '''
                 '''data-anchor-id="c-e58d85a3973b-9c390eb52cd6"'''
             )
 
         assert Notification.query().count() == 1
         assert ChangesetComment.query().count() == 1
 
         notification = Notification.query().all()[0]
         comment = ChangesetComment.query().first()
         assert notification.type_ == Notification.TYPE_CHANGESET_COMMENT
 
         assert comment.revision == commit_id
 
         author = notification.created_by_user.username_and_name
         sbj = '@{0} left a {1} on file `{2}` in commit `{3}` in the `{4}` repository'.format(
             author, comment_type, f_path, h.show_id(commit), backend.repo_name)
 
         assert sbj == notification.subject
 
         lnk = (u'/{0}/changeset/{1}#comment-{2}'.format(
             backend.repo_name, commit_id, comment.comment_id))
         assert lnk in notification.body
         assert 'on line n1' in notification.body
 
     def test_create_with_mention(self, backend):
         self.log_user()
 
         commit_id = backend.repo.get_commit('300').raw_id
         text = u'@test_regular check CommentOnCommit'
 
         params = {'text': text, 'csrf_token': self.csrf_token}
         self.app.post(
             route_path('repo_commit_comment_create',
                        repo_name=backend.repo_name, commit_id=commit_id),
             params=params)
 
         response = self.app.get(
             route_path('repo_commit',
                        repo_name=backend.repo_name, commit_id=commit_id))
         # test DB
         assert ChangesetComment.query().count() == 1
         assert_comment_links(response, ChangesetComment.query().count(), 0)
 
         notification = Notification.query().one()
 
         assert len(notification.recipients) == 2
         users = [x.username for x in notification.recipients]
 
         # test_regular gets notification by @mention
         assert sorted(users) == [u'test_admin', u'test_regular']
 
     def test_create_with_status_change(self, backend):
         self.log_user()
         commit = backend.repo.get_commit('300')
         commit_id = commit.raw_id
         text = u'CommentOnCommit'
         f_path = 'vcs/web/simplevcs/views/repository.py'
         line = 'n1'
 
         params = {'text': text, 'changeset_status': 'approved',
                   'csrf_token': self.csrf_token}
 
         self.app.post(
             route_path(
                 'repo_commit_comment_create',
                 repo_name=backend.repo_name, commit_id=commit_id),
             params=params)
 
         response = self.app.get(
             route_path('repo_commit',
                        repo_name=backend.repo_name, commit_id=commit_id))
 
         # test DB
         assert ChangesetComment.query().count() == 1
         assert_comment_links(response, ChangesetComment.query().count(), 0)
 
         assert Notification.query().count() == 1
         assert ChangesetComment.query().count() == 1
 
         notification = Notification.query().all()[0]
 
         comment_id = ChangesetComment.query().first().comment_id
         assert notification.type_ == Notification.TYPE_CHANGESET_COMMENT
 
         author = notification.created_by_user.username_and_name
         sbj = '[status: Approved] @{0} left a note on commit `{1}` in the `{2}` repository'.format(
             author, h.show_id(commit), backend.repo_name)
         assert sbj == notification.subject
 
         lnk = (u'/{0}/changeset/{1}#comment-{2}'.format(
             backend.repo_name, commit_id, comment_id))
         assert lnk in notification.body
 
     def test_delete(self, backend):
         self.log_user()
         commit_id = backend.repo.get_commit('300').raw_id
         text = u'CommentOnCommit'
 
         params = {'text': text, 'csrf_token': self.csrf_token}
         self.app.post(
             route_path(
                 'repo_commit_comment_create',
                 repo_name=backend.repo_name, commit_id=commit_id),
             params=params)
 
         comments = ChangesetComment.query().all()
         assert len(comments) == 1
         comment_id = comments[0].comment_id
 
         self.app.post(
             route_path('repo_commit_comment_delete',
                        repo_name=backend.repo_name,
                        commit_id=commit_id,
                        comment_id=comment_id),
             params={'csrf_token': self.csrf_token})
 
         comments = ChangesetComment.query().all()
         assert len(comments) == 0
 
         response = self.app.get(
             route_path('repo_commit',
                        repo_name=backend.repo_name, commit_id=commit_id))
         assert_comment_links(response, 0, 0)
 
     def test_edit(self, backend):
         self.log_user()
         commit_id = backend.repo.get_commit('300').raw_id
         text = u'CommentOnCommit'
 
         params = {'text': text, 'csrf_token': self.csrf_token}
         self.app.post(
             route_path(
                 'repo_commit_comment_create',
                 repo_name=backend.repo_name, commit_id=commit_id),
             params=params)
 
         comments = ChangesetComment.query().all()
         assert len(comments) == 1
         comment_id = comments[0].comment_id
         test_text = 'test_text'
         self.app.post(
             route_path(
                 'repo_commit_comment_edit',
                 repo_name=backend.repo_name,
                 commit_id=commit_id,
                 comment_id=comment_id,
             ),
             params={
                 'csrf_token': self.csrf_token,
                 'text': test_text,
                 'version': '0',
             })
 
         text_form_db = ChangesetComment.query().filter(
             ChangesetComment.comment_id == comment_id).first().text
         assert test_text == text_form_db
 
     def test_edit_without_change(self, backend):
         self.log_user()
         commit_id = backend.repo.get_commit('300').raw_id
         text = u'CommentOnCommit'
 
         params = {'text': text, 'csrf_token': self.csrf_token}
         self.app.post(
             route_path(
                 'repo_commit_comment_create',
                 repo_name=backend.repo_name, commit_id=commit_id),
             params=params)
 
         comments = ChangesetComment.query().all()
         assert len(comments) == 1
         comment_id = comments[0].comment_id
 
         response = self.app.post(
             route_path(
                 'repo_commit_comment_edit',
                 repo_name=backend.repo_name,
                 commit_id=commit_id,
                 comment_id=comment_id,
             ),
             params={
                 'csrf_token': self.csrf_token,
                 'text': text,
                 'version': '0',
             },
             status=404,
         )
         assert response.status_int == 404
 
     def test_edit_try_edit_already_edited(self, backend):
         self.log_user()
         commit_id = backend.repo.get_commit('300').raw_id
         text = u'CommentOnCommit'
 
         params = {'text': text, 'csrf_token': self.csrf_token}
         self.app.post(
             route_path(
                 'repo_commit_comment_create',
                 repo_name=backend.repo_name, commit_id=commit_id
             ),
             params=params,
         )
 
         comments = ChangesetComment.query().all()
         assert len(comments) == 1
         comment_id = comments[0].comment_id
         test_text = 'test_text'
         self.app.post(
             route_path(
                 'repo_commit_comment_edit',
                 repo_name=backend.repo_name,
                 commit_id=commit_id,
                 comment_id=comment_id,
             ),
             params={
                 'csrf_token': self.csrf_token,
                 'text': test_text,
                 'version': '0',
             }
         )
         test_text_v2 = 'test_v2'
         response = self.app.post(
             route_path(
                 'repo_commit_comment_edit',
                 repo_name=backend.repo_name,
                 commit_id=commit_id,
                 comment_id=comment_id,
             ),
             params={
                 'csrf_token': self.csrf_token,
                 'text': test_text_v2,
                 'version': '0',
             },
             status=409,
         )
         assert response.status_int == 409
 
         text_form_db = ChangesetComment.query().filter(
             ChangesetComment.comment_id == comment_id).first().text
 
         assert test_text == text_form_db
         assert test_text_v2 != text_form_db
 
     def test_edit_forbidden_for_immutable_comments(self, backend):
         self.log_user()
         commit_id = backend.repo.get_commit('300').raw_id
         text = u'CommentOnCommit'
 
         params = {'text': text, 'csrf_token': self.csrf_token, 'version': '0'}
         self.app.post(
             route_path(
                 'repo_commit_comment_create',
                 repo_name=backend.repo_name,
                 commit_id=commit_id,
             ),
             params=params
         )
 
         comments = ChangesetComment.query().all()
         assert len(comments) == 1
         comment_id = comments[0].comment_id
 
         comment = ChangesetComment.get(comment_id)
         comment.immutable_state = ChangesetComment.OP_IMMUTABLE
         Session().add(comment)
         Session().commit()
 
         response = self.app.post(
             route_path(
                 'repo_commit_comment_edit',
                 repo_name=backend.repo_name,
                 commit_id=commit_id,
                 comment_id=comment_id,
             ),
             params={
                 'csrf_token': self.csrf_token,
                 'text': 'test_text',
             },
             status=403,
         )
         assert response.status_int == 403
 
     def test_delete_forbidden_for_immutable_comments(self, backend):
         self.log_user()
         commit_id = backend.repo.get_commit('300').raw_id
         text = u'CommentOnCommit'
 
         params = {'text': text, 'csrf_token': self.csrf_token}
         self.app.post(
             route_path(
                 'repo_commit_comment_create',
                 repo_name=backend.repo_name, commit_id=commit_id),
             params=params)
 
         comments = ChangesetComment.query().all()
         assert len(comments) == 1
         comment_id = comments[0].comment_id
 
         comment = ChangesetComment.get(comment_id)
         comment.immutable_state = ChangesetComment.OP_IMMUTABLE
         Session().add(comment)
         Session().commit()
 
         self.app.post(
             route_path('repo_commit_comment_delete',
                        repo_name=backend.repo_name,
                        commit_id=commit_id,
                        comment_id=comment_id),
             params={'csrf_token': self.csrf_token},
             status=403)
 
     @pytest.mark.parametrize('renderer, text_input, output', [
         ('rst', 'plain text', '<p>plain text</p>'),
         ('rst', 'header\n======', '<h1 class="title">header</h1>'),
         ('rst', '*italics*', '<em>italics</em>'),
         ('rst', '**bold**', '<strong>bold</strong>'),
         ('markdown', 'plain text', '<p>plain text</p>'),
         ('markdown', '# header', '<h1>header</h1>'),
         ('markdown', '*italics*', '<em>italics</em>'),
         ('markdown', '**bold**', '<strong>bold</strong>'),
     ], ids=['rst-plain', 'rst-header', 'rst-italics', 'rst-bold', 'md-plain',
             'md-header', 'md-italics', 'md-bold', ])
     def test_preview(self, renderer, text_input, output, backend, xhr_header):
         self.log_user()
         params = {
             'renderer': renderer,
             'text': text_input,
             'csrf_token': self.csrf_token
         }
         commit_id = '0' * 16  # fake this for tests
         response = self.app.post(
             route_path('repo_commit_comment_preview',
                        repo_name=backend.repo_name, commit_id=commit_id,),
             params=params,
             extra_environ=xhr_header)
 
         response.mustcontain(output)
 
 
 def assert_comment_links(response, comments, inline_comments):
-    if comments == 1:
-        comments_text = "%d General" % comments
-    else:
-        comments_text = "%d General" % comments
-
-    if inline_comments == 1:
-        inline_comments_text = "%d Inline" % inline_comments
-    else:
-        inline_comments_text = "%d Inline" % inline_comments
+    response.mustcontain(
+        '<span class="display-none" id="general-comments-count">{}</span>'.format(comments))
+    response.mustcontain(
+        '<span class="display-none" id="inline-comments-count">{}</span>'.format(inline_comments))
 
-    if comments:
-        response.mustcontain('<a href="#comments">%s</a>,' % comments_text)
-    else:
-        response.mustcontain(comments_text)
 
-    if inline_comments:
-        response.mustcontain(
-            'id="inline-comments-counter">%s' % inline_comments_text)
-    else:
-        response.mustcontain(inline_comments_text)
+

rhodecode/apps/repository/tests/test_repo_compare.py

@@ -1,667 +1,672 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import mock
 import pytest
 import lxml.html
 
 from rhodecode.lib.vcs.exceptions import RepositoryRequirementError
 from rhodecode.tests import assert_session_flash
 from rhodecode.tests.utils import AssertResponse, commit_change
 
 
 def route_path(name, params=None, **kwargs):
     import urllib
 
     base_url = {
         'repo_compare_select': '/{repo_name}/compare',
         'repo_compare': '/{repo_name}/compare/{source_ref_type}@{source_ref}...{target_ref_type}@{target_ref}',
     }[name].format(**kwargs)
 
     if params:
         base_url = '{}?{}'.format(base_url, urllib.urlencode(params))
     return base_url
 
 
 @pytest.mark.usefixtures("autologin_user", "app")
 class TestCompareView(object):
 
     def test_compare_index_is_reached_at_least_once(self, backend):
         repo = backend.repo
         self.app.get(
             route_path('repo_compare_select', repo_name=repo.repo_name))
 
     @pytest.mark.xfail_backends("svn", reason="Requires pull")
     def test_compare_remote_with_different_commit_indexes(self, backend):
         # Preparing the following repository structure:
         #
         # Origin repository has two commits:
         #
         #    0    1
         #    A -- D
         #
         # The fork of it has a few more commits and "D" has a commit index
         # which does not exist in origin.
         #
         #    0    1    2    3    4
         #    A --   --   -- D -- E
         #      \- B -- C
         #
 
         fork = backend.create_repo()
 
         # prepare fork
         commit0 = commit_change(
             fork.repo_name, filename='file1', content='A',
             message='A', vcs_type=backend.alias, parent=None, newfile=True)
 
         commit1 = commit_change(
             fork.repo_name, filename='file1', content='B',
             message='B, child of A', vcs_type=backend.alias, parent=commit0)
 
         commit_change(  # commit 2
             fork.repo_name, filename='file1', content='C',
             message='C, child of B', vcs_type=backend.alias, parent=commit1)
 
         commit3 = commit_change(
             fork.repo_name, filename='file1', content='D',
             message='D, child of A', vcs_type=backend.alias, parent=commit0)
 
         commit4 = commit_change(
             fork.repo_name, filename='file1', content='E',
             message='E, child of D', vcs_type=backend.alias, parent=commit3)
 
         # prepare origin repository, taking just the history up to D
         origin = backend.create_repo()
 
         origin_repo = origin.scm_instance(cache=False)
         origin_repo.config.clear_section('hooks')
         origin_repo.pull(fork.repo_full_path, commit_ids=[commit3.raw_id])
         origin_repo = origin.scm_instance(cache=False)  # cache rebuild
 
         # Verify test fixture setup
         # This does not work for git
         if backend.alias != 'git':
             assert 5 == len(fork.scm_instance().commit_ids)
             assert 2 == len(origin_repo.commit_ids)
 
         # Comparing the revisions
         response = self.app.get(
             route_path('repo_compare',
                        repo_name=origin.repo_name,
                        source_ref_type="rev", source_ref=commit3.raw_id,
                        target_ref_type="rev", target_ref=commit4.raw_id,
                        params=dict(merge='1', target_repo=fork.repo_name)
                        ))
 
         compare_page = ComparePage(response)
         compare_page.contains_commits([commit4])
 
     @pytest.mark.xfail_backends("svn", reason="Depends on branch support")
     def test_compare_forks_on_branch_extra_commits(self, backend):
         repo1 = backend.create_repo()
 
         # commit something !
         commit0 = commit_change(
             repo1.repo_name, filename='file1', content='line1\n',
             message='commit1', vcs_type=backend.alias, parent=None,
             newfile=True)
 
         # fork this repo
         repo2 = backend.create_fork()
 
         # add two extra commit into fork
         commit1 = commit_change(
             repo2.repo_name, filename='file1', content='line1\nline2\n',
             message='commit2', vcs_type=backend.alias, parent=commit0)
 
         commit2 = commit_change(
             repo2.repo_name, filename='file1', content='line1\nline2\nline3\n',
             message='commit3', vcs_type=backend.alias, parent=commit1)
 
         commit_id1 = repo1.scm_instance().DEFAULT_BRANCH_NAME
         commit_id2 = repo2.scm_instance().DEFAULT_BRANCH_NAME
 
         response = self.app.get(
             route_path('repo_compare',
                        repo_name=repo1.repo_name,
                        source_ref_type="branch", source_ref=commit_id2,
                        target_ref_type="branch", target_ref=commit_id1,
                        params=dict(merge='1', target_repo=repo2.repo_name)
                        ))
 
         response.mustcontain('%s@%s' % (repo1.repo_name, commit_id2))
         response.mustcontain('%s@%s' % (repo2.repo_name, commit_id1))
 
         compare_page = ComparePage(response)
         compare_page.contains_change_summary(1, 2, 0)
         compare_page.contains_commits([commit1, commit2])
 
         anchor = 'a_c-{}-826e8142e6ba'.format(commit0.short_id)
         compare_page.contains_file_links_and_anchors([('file1', anchor), ])
 
         # Swap is removed when comparing branches since it's a PR feature and
         # it is then a preview mode
         compare_page.swap_is_hidden()
         compare_page.target_source_are_disabled()
 
     @pytest.mark.xfail_backends("svn", reason="Depends on branch support")
     def test_compare_forks_on_branch_extra_commits_origin_has_incomming(self, backend):
         repo1 = backend.create_repo()
 
         # commit something !
         commit0 = commit_change(
             repo1.repo_name, filename='file1', content='line1\n',
             message='commit1', vcs_type=backend.alias, parent=None,
             newfile=True)
 
         # fork this repo
         repo2 = backend.create_fork()
 
         # now commit something to origin repo
         commit_change(
             repo1.repo_name, filename='file2', content='line1file2\n',
             message='commit2', vcs_type=backend.alias, parent=commit0,
             newfile=True)
 
         # add two extra commit into fork
         commit1 = commit_change(
             repo2.repo_name, filename='file1', content='line1\nline2\n',
             message='commit2', vcs_type=backend.alias, parent=commit0)
 
         commit2 = commit_change(
             repo2.repo_name, filename='file1', content='line1\nline2\nline3\n',
             message='commit3', vcs_type=backend.alias, parent=commit1)
 
         commit_id1 = repo1.scm_instance().DEFAULT_BRANCH_NAME
         commit_id2 = repo2.scm_instance().DEFAULT_BRANCH_NAME
 
         response = self.app.get(
             route_path('repo_compare',
                        repo_name=repo1.repo_name,
                        source_ref_type="branch", source_ref=commit_id2,
                        target_ref_type="branch", target_ref=commit_id1,
                        params=dict(merge='1', target_repo=repo2.repo_name),
                        ))
 
         response.mustcontain('%s@%s' % (repo1.repo_name, commit_id2))
         response.mustcontain('%s@%s' % (repo2.repo_name, commit_id1))
 
         compare_page = ComparePage(response)
         compare_page.contains_change_summary(1, 2, 0)
         compare_page.contains_commits([commit1, commit2])
         anchor = 'a_c-{}-826e8142e6ba'.format(commit0.short_id)
         compare_page.contains_file_links_and_anchors([('file1', anchor), ])
 
         # Swap is removed when comparing branches since it's a PR feature and
         # it is then a preview mode
         compare_page.swap_is_hidden()
         compare_page.target_source_are_disabled()
 
     @pytest.mark.xfail_backends("svn")
     # TODO(marcink): no svn support for compare two seperate repos
     def test_compare_of_unrelated_forks(self, backend):
         orig = backend.create_repo(number_of_commits=1)
         fork = backend.create_repo(number_of_commits=1)
 
         response = self.app.get(
             route_path('repo_compare',
                        repo_name=orig.repo_name,
                        source_ref_type="rev", source_ref="tip",
                        target_ref_type="rev", target_ref="tip",
                        params=dict(merge='1', target_repo=fork.repo_name),
                        ),
             status=302)
         response = response.follow()
         response.mustcontain("Repositories unrelated.")
 
     @pytest.mark.xfail_backends("svn")
     def test_compare_cherry_pick_commits_from_bottom(self, backend):
 
         # repo1:
         #     commit0:
         #     commit1:
         # repo1-fork- in which we will cherry pick bottom commits
         #     commit0:
         #     commit1:
         #     commit2: x
         #     commit3: x
         #     commit4: x
         #     commit5:
         # make repo1, and commit1+commit2
 
         repo1 = backend.create_repo()
 
         # commit something !
         commit0 = commit_change(
             repo1.repo_name, filename='file1', content='line1\n',
             message='commit1', vcs_type=backend.alias, parent=None,
             newfile=True)
         commit1 = commit_change(
             repo1.repo_name, filename='file1', content='line1\nline2\n',
             message='commit2', vcs_type=backend.alias, parent=commit0)
 
         # fork this repo
         repo2 = backend.create_fork()
 
         # now make commit3-6
         commit2 = commit_change(
             repo1.repo_name, filename='file1', content='line1\nline2\nline3\n',
             message='commit3', vcs_type=backend.alias, parent=commit1)
         commit3 = commit_change(
             repo1.repo_name, filename='file1',
             content='line1\nline2\nline3\nline4\n', message='commit4',
             vcs_type=backend.alias, parent=commit2)
         commit4 = commit_change(
             repo1.repo_name, filename='file1',
             content='line1\nline2\nline3\nline4\nline5\n', message='commit5',
             vcs_type=backend.alias, parent=commit3)
         commit_change(  # commit 5
             repo1.repo_name, filename='file1',
             content='line1\nline2\nline3\nline4\nline5\nline6\n',
             message='commit6', vcs_type=backend.alias, parent=commit4)
 
         response = self.app.get(
             route_path('repo_compare',
                        repo_name=repo2.repo_name,
                        # parent of commit2, in target repo2
                        source_ref_type="rev", source_ref=commit1.raw_id,
                        target_ref_type="rev", target_ref=commit4.raw_id,
                        params=dict(merge='1', target_repo=repo1.repo_name),
                        ))
         response.mustcontain('%s@%s' % (repo2.repo_name, commit1.short_id))
         response.mustcontain('%s@%s' % (repo1.repo_name, commit4.short_id))
 
         # files
         compare_page = ComparePage(response)
         compare_page.contains_change_summary(1, 3, 0)
         compare_page.contains_commits([commit2, commit3, commit4])
         anchor = 'a_c-{}-826e8142e6ba'.format(commit1.short_id)
         compare_page.contains_file_links_and_anchors([('file1', anchor),])
 
     @pytest.mark.xfail_backends("svn")
     def test_compare_cherry_pick_commits_from_top(self, backend):
         # repo1:
         #     commit0:
         #     commit1:
         # repo1-fork- in which we will cherry pick bottom commits
         #     commit0:
         #     commit1:
         #     commit2:
         #     commit3: x
         #     commit4: x
         #     commit5: x
 
         # make repo1, and commit1+commit2
         repo1 = backend.create_repo()
 
         # commit something !
         commit0 = commit_change(
             repo1.repo_name, filename='file1', content='line1\n',
             message='commit1', vcs_type=backend.alias, parent=None,
             newfile=True)
         commit1 = commit_change(
             repo1.repo_name, filename='file1', content='line1\nline2\n',
             message='commit2', vcs_type=backend.alias, parent=commit0)
 
         # fork this repo
         backend.create_fork()
 
         # now make commit3-6
         commit2 = commit_change(
             repo1.repo_name, filename='file1', content='line1\nline2\nline3\n',
             message='commit3', vcs_type=backend.alias, parent=commit1)
         commit3 = commit_change(
             repo1.repo_name, filename='file1',
             content='line1\nline2\nline3\nline4\n', message='commit4',
             vcs_type=backend.alias, parent=commit2)
         commit4 = commit_change(
             repo1.repo_name, filename='file1',
             content='line1\nline2\nline3\nline4\nline5\n', message='commit5',
             vcs_type=backend.alias, parent=commit3)
         commit5 = commit_change(
             repo1.repo_name, filename='file1',
             content='line1\nline2\nline3\nline4\nline5\nline6\n',
             message='commit6', vcs_type=backend.alias, parent=commit4)
 
         response = self.app.get(
             route_path('repo_compare',
                        repo_name=repo1.repo_name,
                        # parent of commit3, not in source repo2
                        source_ref_type="rev", source_ref=commit2.raw_id,
                        target_ref_type="rev", target_ref=commit5.raw_id,
                        params=dict(merge='1'),))
 
         response.mustcontain('%s@%s' % (repo1.repo_name, commit2.short_id))
         response.mustcontain('%s@%s' % (repo1.repo_name, commit5.short_id))
 
         compare_page = ComparePage(response)
         compare_page.contains_change_summary(1, 3, 0)
         compare_page.contains_commits([commit3, commit4, commit5])
 
         # files
         anchor = 'a_c-{}-826e8142e6ba'.format(commit2.short_id)
         compare_page.contains_file_links_and_anchors([('file1', anchor),])
 
     @pytest.mark.xfail_backends("svn")
     def test_compare_remote_branches(self, backend):
         repo1 = backend.repo
         repo2 = backend.create_fork()
 
         commit_id1 = repo1.get_commit(commit_idx=3).raw_id
         commit_id1_short = repo1.get_commit(commit_idx=3).short_id
         commit_id2 = repo1.get_commit(commit_idx=6).raw_id
         commit_id2_short = repo1.get_commit(commit_idx=6).short_id
 
         response = self.app.get(
             route_path('repo_compare',
                        repo_name=repo1.repo_name,
                        source_ref_type="rev", source_ref=commit_id1,
                        target_ref_type="rev", target_ref=commit_id2,
                        params=dict(merge='1', target_repo=repo2.repo_name),
                        ))
 
         response.mustcontain('%s@%s' % (repo1.repo_name, commit_id1))
         response.mustcontain('%s@%s' % (repo2.repo_name, commit_id2))
 
         compare_page = ComparePage(response)
 
         # outgoing commits between those commits
         compare_page.contains_commits(
             [repo2.get_commit(commit_idx=x) for x in [4, 5, 6]])
 
         # files
         compare_page.contains_file_links_and_anchors([
             ('vcs/backends/hg.py', 'a_c-{}-9c390eb52cd6'.format(commit_id2_short)),
             ('vcs/backends/__init__.py', 'a_c-{}-41b41c1f2796'.format(commit_id1_short)),
             ('vcs/backends/base.py', 'a_c-{}-2f574d260608'.format(commit_id1_short)),
         ])
 
     @pytest.mark.xfail_backends("svn")
     def test_source_repo_new_commits_after_forking_simple_diff(self, backend):
         repo1 = backend.create_repo()
         r1_name = repo1.repo_name
 
         commit0 = commit_change(
             repo=r1_name, filename='file1',
             content='line1', message='commit1', vcs_type=backend.alias,
             newfile=True)
         assert repo1.scm_instance().commit_ids == [commit0.raw_id]
 
         # fork the repo1
         repo2 = backend.create_fork()
         assert repo2.scm_instance().commit_ids == [commit0.raw_id]
 
         self.r2_id = repo2.repo_id
         r2_name = repo2.repo_name
 
         commit1 = commit_change(
             repo=r2_name, filename='file1-fork',
             content='file1-line1-from-fork', message='commit1-fork',
             vcs_type=backend.alias, parent=repo2.scm_instance()[-1],
             newfile=True)
 
         commit2 = commit_change(
             repo=r2_name, filename='file2-fork',
             content='file2-line1-from-fork', message='commit2-fork',
             vcs_type=backend.alias, parent=commit1,
             newfile=True)
 
         commit_change(  # commit 3
             repo=r2_name, filename='file3-fork',
             content='file3-line1-from-fork', message='commit3-fork',
             vcs_type=backend.alias, parent=commit2, newfile=True)
 
         # compare !
         commit_id1 = repo1.scm_instance().DEFAULT_BRANCH_NAME
         commit_id2 = repo2.scm_instance().DEFAULT_BRANCH_NAME
 
         response = self.app.get(
             route_path('repo_compare',
                        repo_name=r2_name,
                        source_ref_type="branch", source_ref=commit_id1,
                        target_ref_type="branch", target_ref=commit_id2,
                        params=dict(merge='1', target_repo=r1_name),
                        ))
 
         response.mustcontain('%s@%s' % (r2_name, commit_id1))
         response.mustcontain('%s@%s' % (r1_name, commit_id2))
         response.mustcontain('No files')
         response.mustcontain('No commits in this compare')
 
         commit0 = commit_change(
             repo=r1_name, filename='file2',
             content='line1-added-after-fork', message='commit2-parent',
             vcs_type=backend.alias, parent=None, newfile=True)
 
         # compare !
         response = self.app.get(
             route_path('repo_compare',
                        repo_name=r2_name,
                        source_ref_type="branch", source_ref=commit_id1,
                        target_ref_type="branch", target_ref=commit_id2,
                        params=dict(merge='1', target_repo=r1_name),
                        ))
 
         response.mustcontain('%s@%s' % (r2_name, commit_id1))
         response.mustcontain('%s@%s' % (r1_name, commit_id2))
 
         response.mustcontain("""commit2-parent""")
         response.mustcontain("""line1-added-after-fork""")
         compare_page = ComparePage(response)
         compare_page.contains_change_summary(1, 1, 0)
 
     @pytest.mark.xfail_backends("svn")
     def test_compare_commits(self, backend, xhr_header):
         commit0 = backend.repo.get_commit(commit_idx=0)
         commit1 = backend.repo.get_commit(commit_idx=1)
 
         response = self.app.get(
             route_path('repo_compare',
                        repo_name=backend.repo_name,
                        source_ref_type="rev", source_ref=commit0.raw_id,
                        target_ref_type="rev", target_ref=commit1.raw_id,
                        params=dict(merge='1')
                        ),
             extra_environ=xhr_header, )
 
         # outgoing commits between those commits
         compare_page = ComparePage(response)
         compare_page.contains_commits(commits=[commit1])
 
     def test_errors_when_comparing_unknown_source_repo(self, backend):
         repo = backend.repo
         badrepo = 'badrepo'
 
         response = self.app.get(
             route_path('repo_compare',
                        repo_name=badrepo,
                        source_ref_type="rev", source_ref='tip',
                        target_ref_type="rev", target_ref='tip',
                        params=dict(merge='1', target_repo=repo.repo_name)
                        ),
             status=404)
 
     def test_errors_when_comparing_unknown_target_repo(self, backend):
         repo = backend.repo
         badrepo = 'badrepo'
 
         response = self.app.get(
             route_path('repo_compare',
                        repo_name=repo.repo_name,
                        source_ref_type="rev", source_ref='tip',
                        target_ref_type="rev", target_ref='tip',
                        params=dict(merge='1', target_repo=badrepo),
                        ),
             status=302)
         redirected = response.follow()
         redirected.mustcontain(
             'Could not find the target repo: `{}`'.format(badrepo))
 
     def test_compare_not_in_preview_mode(self, backend_stub):
         commit0 = backend_stub.repo.get_commit(commit_idx=0)
         commit1 = backend_stub.repo.get_commit(commit_idx=1)
 
         response = self.app.get(
             route_path('repo_compare',
                        repo_name=backend_stub.repo_name,
                        source_ref_type="rev", source_ref=commit0.raw_id,
                        target_ref_type="rev", target_ref=commit1.raw_id,
                        ))
 
         # outgoing commits between those commits
         compare_page = ComparePage(response)
         compare_page.swap_is_visible()
         compare_page.target_source_are_enabled()
 
     def test_compare_of_fork_with_largefiles(self, backend_hg, settings_util):
         orig = backend_hg.create_repo(number_of_commits=1)
         fork = backend_hg.create_fork()
 
         settings_util.create_repo_rhodecode_ui(
             orig, 'extensions', value='', key='largefiles', active=False)
         settings_util.create_repo_rhodecode_ui(
             fork, 'extensions', value='', key='largefiles', active=True)
 
         compare_module = ('rhodecode.lib.vcs.backends.hg.repository.'
                           'MercurialRepository.compare')
         with mock.patch(compare_module) as compare_mock:
             compare_mock.side_effect = RepositoryRequirementError()
 
             response = self.app.get(
                 route_path('repo_compare',
                            repo_name=orig.repo_name,
                            source_ref_type="rev", source_ref="tip",
                            target_ref_type="rev", target_ref="tip",
                            params=dict(merge='1', target_repo=fork.repo_name),
                            ),
                 status=302)
 
             assert_session_flash(
                 response,
                 'Could not compare repos with different large file settings')
 
 
 @pytest.mark.usefixtures("autologin_user")
 class TestCompareControllerSvn(object):
 
     def test_supports_references_with_path(self, app, backend_svn):
         repo = backend_svn['svn-simple-layout']
         commit_id = repo.get_commit(commit_idx=-1).raw_id
         response = app.get(
             route_path('repo_compare',
                        repo_name=repo.repo_name,
                        source_ref_type="tag",
                        source_ref="%s@%s" % ('tags/v0.1', commit_id),
                        target_ref_type="tag",
                        target_ref="%s@%s" % ('tags/v0.2', commit_id),
                        params=dict(merge='1'),
                        ),
             status=200)
 
         # Expecting no commits, since both paths are at the same revision
         response.mustcontain('No commits in this compare')
 
         # Should find only one file changed when comparing those two tags
         response.mustcontain('example.py')
         compare_page = ComparePage(response)
         compare_page.contains_change_summary(1, 5, 1)
 
     def test_shows_commits_if_different_ids(self, app, backend_svn):
         repo = backend_svn['svn-simple-layout']
         source_id = repo.get_commit(commit_idx=-6).raw_id
         target_id = repo.get_commit(commit_idx=-1).raw_id
         response = app.get(
             route_path('repo_compare',
                        repo_name=repo.repo_name,
                        source_ref_type="tag",
                        source_ref="%s@%s" % ('tags/v0.1', source_id),
                        target_ref_type="tag",
                        target_ref="%s@%s" % ('tags/v0.2', target_id),
                        params=dict(merge='1')
                        ),
             status=200)
 
         # It should show commits
         assert 'No commits in this compare' not in response.body
 
         # Should find only one file changed when comparing those two tags
         response.mustcontain('example.py')
         compare_page = ComparePage(response)
         compare_page.contains_change_summary(1, 5, 1)
 
 
 class ComparePage(AssertResponse):
     """
     Abstracts the page template from the tests
     """
 
     def contains_file_links_and_anchors(self, files):
         doc = lxml.html.fromstring(self.response.body)
         for filename, file_id in files:
             self.contains_one_anchor(file_id)
             diffblock = doc.cssselect('[data-f-path="%s"]' % filename)
             assert len(diffblock) == 2
-            assert len(diffblock[0].cssselect('a[href="#%s"]' % file_id)) == 1
+            for lnk in diffblock[0].cssselect('a'):
+                if 'permalink' in lnk.text:
+                    assert '#{}'.format(file_id) in lnk.attrib['href']
+                    break
+            else:
+                pytest.fail('Unable to find permalink')
 
     def contains_change_summary(self, files_changed, inserted, deleted):
         template = (
             '{files_changed} file{plural} changed: '
             '<span class="op-added">{inserted} inserted</span>, <span class="op-deleted">{deleted} deleted</span>')
         self.response.mustcontain(template.format(
             files_changed=files_changed,
             plural="s" if files_changed > 1 else "",
             inserted=inserted,
             deleted=deleted))
 
     def contains_commits(self, commits, ancestors=None):
         response = self.response
 
         for commit in commits:
             # Expecting to see the commit message in an element which
             # has the ID "c-{commit.raw_id}"
             self.element_contains('#c-' + commit.raw_id, commit.message)
             self.contains_one_link(
                 'r%s:%s' % (commit.idx, commit.short_id),
                 self._commit_url(commit))
 
         if ancestors:
             response.mustcontain('Ancestor')
             for ancestor in ancestors:
                 self.contains_one_link(
                     ancestor.short_id, self._commit_url(ancestor))
 
     def _commit_url(self, commit):
         return '/%s/changeset/%s' % (commit.repository.name, commit.raw_id)
 
     def swap_is_hidden(self):
         assert '<a id="btn-swap"' not in self.response.text
 
     def swap_is_visible(self):
         assert '<a id="btn-swap"' in self.response.text
 
     def target_source_are_disabled(self):
         response = self.response
         response.mustcontain("var enable_fields = false;")
         response.mustcontain('.select2("enable", enable_fields)')
 
     def target_source_are_enabled(self):
         response = self.response
         response.mustcontain("var enable_fields = true;")

rhodecode/apps/repository/tests/test_repo_pullrequests.py

@@ -1,1652 +1,1658 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import mock
 import pytest
 
 import rhodecode
 from rhodecode.lib.vcs.backends.base import MergeResponse, MergeFailureReason
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.lib import helpers as h
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.db import (
     PullRequest, ChangesetStatus, UserLog, Notification, ChangesetComment, Repository)
 from rhodecode.model.meta import Session
 from rhodecode.model.pull_request import PullRequestModel
 from rhodecode.model.user import UserModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.tests import (
     assert_session_flash, TEST_USER_ADMIN_LOGIN, TEST_USER_REGULAR_LOGIN)
 
 
 def route_path(name, params=None, **kwargs):
     import urllib
 
     base_url = {
         'repo_changelog': '/{repo_name}/changelog',
         'repo_changelog_file': '/{repo_name}/changelog/{commit_id}/{f_path}',
         'repo_commits': '/{repo_name}/commits',
         'repo_commits_file': '/{repo_name}/commits/{commit_id}/{f_path}',
         'pullrequest_show': '/{repo_name}/pull-request/{pull_request_id}',
         'pullrequest_show_all': '/{repo_name}/pull-request',
         'pullrequest_show_all_data': '/{repo_name}/pull-request-data',
         'pullrequest_repo_refs': '/{repo_name}/pull-request/refs/{target_repo_name:.*?[^/]}',
         'pullrequest_repo_targets': '/{repo_name}/pull-request/repo-destinations',
         'pullrequest_new': '/{repo_name}/pull-request/new',
         'pullrequest_create': '/{repo_name}/pull-request/create',
         'pullrequest_update': '/{repo_name}/pull-request/{pull_request_id}/update',
         'pullrequest_merge': '/{repo_name}/pull-request/{pull_request_id}/merge',
         'pullrequest_delete': '/{repo_name}/pull-request/{pull_request_id}/delete',
         'pullrequest_comment_create': '/{repo_name}/pull-request/{pull_request_id}/comment',
         'pullrequest_comment_delete': '/{repo_name}/pull-request/{pull_request_id}/comment/{comment_id}/delete',
         'pullrequest_comment_edit': '/{repo_name}/pull-request/{pull_request_id}/comment/{comment_id}/edit',
     }[name].format(**kwargs)
 
     if params:
         base_url = '{}?{}'.format(base_url, urllib.urlencode(params))
     return base_url
 
 
 @pytest.mark.usefixtures('app', 'autologin_user')
 @pytest.mark.backends("git", "hg")
 class TestPullrequestsView(object):
 
     def test_index(self, backend):
         self.app.get(route_path(
             'pullrequest_new',
             repo_name=backend.repo_name))
 
     def test_option_menu_create_pull_request_exists(self, backend):
         repo_name = backend.repo_name
         response = self.app.get(h.route_path('repo_summary', repo_name=repo_name))
 
         create_pr_link = '<a href="%s">Create Pull Request</a>' % route_path(
             'pullrequest_new', repo_name=repo_name)
         response.mustcontain(create_pr_link)
 
     def test_create_pr_form_with_raw_commit_id(self, backend):
         repo = backend.repo
 
         self.app.get(
             route_path('pullrequest_new', repo_name=repo.repo_name,
                        commit=repo.get_commit().raw_id),
             status=200)
 
     @pytest.mark.parametrize('pr_merge_enabled', [True, False])
     @pytest.mark.parametrize('range_diff', ["0", "1"])
     def test_show(self, pr_util, pr_merge_enabled, range_diff):
         pull_request = pr_util.create_pull_request(
             mergeable=pr_merge_enabled, enable_notifications=False)
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id,
             params={'range-diff': range_diff}))
 
         for commit_id in pull_request.revisions:
             response.mustcontain(commit_id)
 
         response.mustcontain(pull_request.target_ref_parts.type)
         response.mustcontain(pull_request.target_ref_parts.name)
 
         response.mustcontain('class="pull-request-merge"')
 
         if pr_merge_enabled:
             response.mustcontain('Pull request reviewer approval is pending')
         else:
             response.mustcontain('Server-side pull request merging is disabled.')
 
         if range_diff == "1":
             response.mustcontain('Turn off: Show the diff as commit range')
 
     def test_show_versions_of_pr(self, backend, csrf_token):
         commits = [
             {'message': 'initial-commit',
              'added': [FileNode('test-file.txt', 'LINE1\n')]},
 
             {'message': 'commit-1',
              'changed': [FileNode('test-file.txt', 'LINE1\nLINE2\n')]},
             # Above is the initial version of PR that changes a single line
 
             # from now on we'll add 3x commit adding a nother line on each step
             {'message': 'commit-2',
              'changed': [FileNode('test-file.txt', 'LINE1\nLINE2\nLINE3\n')]},
 
             {'message': 'commit-3',
              'changed': [FileNode('test-file.txt', 'LINE1\nLINE2\nLINE3\nLINE4\n')]},
 
             {'message': 'commit-4',
              'changed': [FileNode('test-file.txt', 'LINE1\nLINE2\nLINE3\nLINE4\nLINE5\n')]},
         ]
 
         commit_ids = backend.create_master_repo(commits)
         target = backend.create_repo(heads=['initial-commit'])
         source = backend.create_repo(heads=['commit-1'])
         source_repo_name = source.repo_name
         target_repo_name = target.repo_name
 
         target_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name, commit_id=commit_ids['initial-commit'])
         source_ref = 'branch:{branch}:{commit_id}'.format(
              branch=backend.default_branch_name, commit_id=commit_ids['commit-1'])
 
         response = self.app.post(
             route_path('pullrequest_create', repo_name=source.repo_name),
             [
-                ('source_repo', source.repo_name),
+                ('source_repo', source_repo_name),
                 ('source_ref', source_ref),
-                ('target_repo', target.repo_name),
+                ('target_repo', target_repo_name),
                 ('target_ref',  target_ref),
                 ('common_ancestor', commit_ids['initial-commit']),
                 ('pullrequest_title', 'Title'),
                 ('pullrequest_desc', 'Description'),
                 ('description_renderer', 'markdown'),
                 ('__start__', 'review_members:sequence'),
                     ('__start__', 'reviewer:mapping'),
                         ('user_id', '1'),
                         ('__start__', 'reasons:sequence'),
                             ('reason', 'Some reason'),
                         ('__end__', 'reasons:sequence'),
                         ('__start__', 'rules:sequence'),
                         ('__end__', 'rules:sequence'),
                         ('mandatory', 'False'),
                     ('__end__', 'reviewer:mapping'),
                 ('__end__', 'review_members:sequence'),
                 ('__start__', 'revisions:sequence'),
                     ('revisions', commit_ids['commit-1']),
                 ('__end__', 'revisions:sequence'),
                 ('user', ''),
                 ('csrf_token', csrf_token),
             ],
             status=302)
 
         location = response.headers['Location']
 
         pull_request_id = location.rsplit('/', 1)[1]
         assert pull_request_id != 'new'
         pull_request = PullRequest.get(int(pull_request_id))
 
         pull_request_id = pull_request.pull_request_id
 
         # Show initial version of PR
         response = self.app.get(
             route_path('pullrequest_show',
                        repo_name=target_repo_name,
                        pull_request_id=pull_request_id))
 
         response.mustcontain('commit-1')
         response.mustcontain(no=['commit-2'])
         response.mustcontain(no=['commit-3'])
         response.mustcontain(no=['commit-4'])
 
         response.mustcontain('cb-addition"></span><span>LINE2</span>')
         response.mustcontain(no=['LINE3'])
         response.mustcontain(no=['LINE4'])
         response.mustcontain(no=['LINE5'])
 
         # update PR #1
         source_repo = Repository.get_by_repo_name(source_repo_name)
         backend.pull_heads(source_repo, heads=['commit-2'])
         response = self.app.post(
             route_path('pullrequest_update',
                        repo_name=target_repo_name, pull_request_id=pull_request_id),
             params={'update_commits': 'true', 'csrf_token': csrf_token})
 
         # update PR #2
         source_repo = Repository.get_by_repo_name(source_repo_name)
         backend.pull_heads(source_repo, heads=['commit-3'])
         response = self.app.post(
             route_path('pullrequest_update',
                        repo_name=target_repo_name, pull_request_id=pull_request_id),
             params={'update_commits': 'true', 'csrf_token': csrf_token})
 
         # update PR #3
         source_repo = Repository.get_by_repo_name(source_repo_name)
         backend.pull_heads(source_repo, heads=['commit-4'])
         response = self.app.post(
             route_path('pullrequest_update',
                        repo_name=target_repo_name, pull_request_id=pull_request_id),
             params={'update_commits': 'true', 'csrf_token': csrf_token})
 
         # Show final version !
         response = self.app.get(
             route_path('pullrequest_show',
                        repo_name=target_repo_name,
                        pull_request_id=pull_request_id))
 
         # 3 updates, and the latest == 4
         response.mustcontain('4 versions available for this pull request')
         response.mustcontain(no=['rhodecode diff rendering error'])
 
         # initial show must have 3 commits, and 3 adds
         response.mustcontain('commit-1')
         response.mustcontain('commit-2')
         response.mustcontain('commit-3')
         response.mustcontain('commit-4')
 
         response.mustcontain('cb-addition"></span><span>LINE2</span>')
         response.mustcontain('cb-addition"></span><span>LINE3</span>')
         response.mustcontain('cb-addition"></span><span>LINE4</span>')
         response.mustcontain('cb-addition"></span><span>LINE5</span>')
 
         # fetch versions
         pr = PullRequest.get(pull_request_id)
         versions = [x.pull_request_version_id for x in pr.versions.all()]
         assert len(versions) == 3
 
         # show v1,v2,v3,v4
         def cb_line(text):
             return 'cb-addition"></span><span>{}</span>'.format(text)
 
         def cb_context(text):
             return '<span class="cb-code"><span class="cb-action cb-context">' \
                    '</span><span>{}</span></span>'.format(text)
 
         commit_tests = {
             # in response, not in response
             1: (['commit-1'], ['commit-2', 'commit-3', 'commit-4']),
             2: (['commit-1', 'commit-2'], ['commit-3', 'commit-4']),
             3: (['commit-1', 'commit-2', 'commit-3'], ['commit-4']),
             4: (['commit-1', 'commit-2', 'commit-3', 'commit-4'], []),
         }
         diff_tests = {
             1: (['LINE2'], ['LINE3', 'LINE4', 'LINE5']),
             2: (['LINE2', 'LINE3'], ['LINE4', 'LINE5']),
             3: (['LINE2', 'LINE3', 'LINE4'], ['LINE5']),
             4: (['LINE2', 'LINE3', 'LINE4', 'LINE5'], []),
         }
         for idx, ver in enumerate(versions, 1):
 
             response = self.app.get(
                 route_path('pullrequest_show',
                            repo_name=target_repo_name,
                            pull_request_id=pull_request_id,
                            params={'version': ver}))
 
             response.mustcontain(no=['rhodecode diff rendering error'])
             response.mustcontain('Showing changes at v{}'.format(idx))
 
             yes, no = commit_tests[idx]
             for y in yes:
                 response.mustcontain(y)
             for n in no:
                 response.mustcontain(no=n)
 
             yes, no = diff_tests[idx]
             for y in yes:
                 response.mustcontain(cb_line(y))
             for n in no:
                 response.mustcontain(no=n)
 
         # show diff between versions
         diff_compare_tests = {
             1: (['LINE3'], ['LINE1', 'LINE2']),
             2: (['LINE3', 'LINE4'], ['LINE1', 'LINE2']),
             3: (['LINE3', 'LINE4', 'LINE5'], ['LINE1', 'LINE2']),
         }
         for idx, ver in enumerate(versions, 1):
             adds, context = diff_compare_tests[idx]
 
             to_ver = ver+1
             if idx == 3:
                 to_ver = 'latest'
 
             response = self.app.get(
                 route_path('pullrequest_show',
                            repo_name=target_repo_name,
                            pull_request_id=pull_request_id,
                            params={'from_version': versions[0], 'version': to_ver}))
 
             response.mustcontain(no=['rhodecode diff rendering error'])
 
             for a in adds:
                 response.mustcontain(cb_line(a))
             for c in context:
                 response.mustcontain(cb_context(c))
 
         # test version v2 -> v3
         response = self.app.get(
             route_path('pullrequest_show',
                        repo_name=target_repo_name,
                        pull_request_id=pull_request_id,
                        params={'from_version': versions[1], 'version': versions[2]}))
 
         response.mustcontain(cb_context('LINE1'))
         response.mustcontain(cb_context('LINE2'))
         response.mustcontain(cb_context('LINE3'))
         response.mustcontain(cb_line('LINE4'))
 
     def test_close_status_visibility(self, pr_util, user_util, csrf_token):
         # Logout
         response = self.app.post(
             h.route_path('logout'),
             params={'csrf_token': csrf_token})
         # Login as regular user
         response = self.app.post(h.route_path('login'),
                                  {'username': TEST_USER_REGULAR_LOGIN,
                                   'password': 'test12'})
 
         pull_request = pr_util.create_pull_request(
             author=TEST_USER_REGULAR_LOGIN)
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
 
         response.mustcontain('Server-side pull request merging is disabled.')
 
         assert_response = response.assert_response()
         # for regular user without a merge permissions, we don't see it
         assert_response.no_element_exists('#close-pull-request-action')
 
         user_util.grant_user_permission_to_repo(
             pull_request.target_repo,
             UserModel().get_by_username(TEST_USER_REGULAR_LOGIN),
             'repository.write')
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
 
         response.mustcontain('Server-side pull request merging is disabled.')
 
         assert_response = response.assert_response()
         # now regular user has a merge permissions, we have CLOSE button
         assert_response.one_element_exists('#close-pull-request-action')
 
     def test_show_invalid_commit_id(self, pr_util):
         # Simulating invalid revisions which will cause a lookup error
         pull_request = pr_util.create_pull_request()
         pull_request.revisions = ['invalid']
         Session().add(pull_request)
         Session().commit()
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
 
         for commit_id in pull_request.revisions:
             response.mustcontain(commit_id)
 
     def test_show_invalid_source_reference(self, pr_util):
         pull_request = pr_util.create_pull_request()
         pull_request.source_ref = 'branch:b:invalid'
         Session().add(pull_request)
         Session().commit()
 
         self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
 
     def test_edit_title_description(self, pr_util, csrf_token):
         pull_request = pr_util.create_pull_request()
         pull_request_id = pull_request.pull_request_id
 
         response = self.app.post(
             route_path('pullrequest_update',
                 repo_name=pull_request.target_repo.repo_name,
                 pull_request_id=pull_request_id),
             params={
                 'edit_pull_request': 'true',
                 'title': 'New title',
                 'description': 'New description',
                 'csrf_token': csrf_token})
 
         assert_session_flash(
             response, u'Pull request title & description updated.',
             category='success')
 
         pull_request = PullRequest.get(pull_request_id)
         assert pull_request.title == 'New title'
         assert pull_request.description == 'New description'
 
     def test_edit_title_description_closed(self, pr_util, csrf_token):
         pull_request = pr_util.create_pull_request()
         pull_request_id = pull_request.pull_request_id
         repo_name = pull_request.target_repo.repo_name
         pr_util.close()
 
         response = self.app.post(
             route_path('pullrequest_update',
                        repo_name=repo_name, pull_request_id=pull_request_id),
             params={
                 'edit_pull_request': 'true',
                 'title': 'New title',
                 'description': 'New description',
                 'csrf_token': csrf_token}, status=200)
         assert_session_flash(
             response, u'Cannot update closed pull requests.',
             category='error')
 
     def test_update_invalid_source_reference(self, pr_util, csrf_token):
         from rhodecode.lib.vcs.backends.base import UpdateFailureReason
 
         pull_request = pr_util.create_pull_request()
         pull_request.source_ref = 'branch:invalid-branch:invalid-commit-id'
         Session().add(pull_request)
         Session().commit()
 
         pull_request_id = pull_request.pull_request_id
 
         response = self.app.post(
             route_path('pullrequest_update',
                 repo_name=pull_request.target_repo.repo_name,
                 pull_request_id=pull_request_id),
             params={'update_commits': 'true', 'csrf_token': csrf_token})
 
         expected_msg = str(PullRequestModel.UPDATE_STATUS_MESSAGES[
             UpdateFailureReason.MISSING_SOURCE_REF])
         assert_session_flash(response, expected_msg, category='error')
 
     def test_missing_target_reference(self, pr_util, csrf_token):
         from rhodecode.lib.vcs.backends.base import MergeFailureReason
         pull_request = pr_util.create_pull_request(
             approved=True, mergeable=True)
         unicode_reference = u'branch:invalid-branch:invalid-commit-id'
         pull_request.target_ref = unicode_reference
         Session().add(pull_request)
         Session().commit()
 
         pull_request_id = pull_request.pull_request_id
         pull_request_url = route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.repo_name,
             pull_request_id=pull_request_id)
 
         response = self.app.get(pull_request_url)
         target_ref_id = 'invalid-branch'
         merge_resp = MergeResponse(
             True, True, '', MergeFailureReason.MISSING_TARGET_REF,
             metadata={'target_ref': PullRequest.unicode_to_reference(unicode_reference)})
         response.assert_response().element_contains(
             'div[data-role="merge-message"]', merge_resp.merge_status_message)
 
     def test_comment_and_close_pull_request_custom_message_approved(
             self, pr_util, csrf_token, xhr_header):
 
         pull_request = pr_util.create_pull_request(approved=True)
         pull_request_id = pull_request.pull_request_id
         author = pull_request.user_id
         repo = pull_request.target_repo.repo_id
 
         self.app.post(
             route_path('pullrequest_comment_create',
                        repo_name=pull_request.target_repo.scm_instance().name,
                        pull_request_id=pull_request_id),
             params={
                 'close_pull_request': '1',
                 'text': 'Closing a PR',
                 'csrf_token': csrf_token},
             extra_environ=xhr_header,)
 
         journal = UserLog.query()\
             .filter(UserLog.user_id == author)\
             .filter(UserLog.repository_id == repo) \
             .order_by(UserLog.user_log_id.asc()) \
             .all()
         assert journal[-1].action == 'repo.pull_request.close'
 
         pull_request = PullRequest.get(pull_request_id)
         assert pull_request.is_closed()
 
         status = ChangesetStatusModel().get_status(
             pull_request.source_repo, pull_request=pull_request)
         assert status == ChangesetStatus.STATUS_APPROVED
         comments = ChangesetComment().query() \
             .filter(ChangesetComment.pull_request == pull_request) \
             .order_by(ChangesetComment.comment_id.asc())\
             .all()
         assert comments[-1].text == 'Closing a PR'
 
     def test_comment_force_close_pull_request_rejected(
             self, pr_util, csrf_token, xhr_header):
         pull_request = pr_util.create_pull_request()
         pull_request_id = pull_request.pull_request_id
         PullRequestModel().update_reviewers(
             pull_request_id, [(1, ['reason'], False, []), (2, ['reason2'], False, [])],
             pull_request.author)
         author = pull_request.user_id
         repo = pull_request.target_repo.repo_id
 
         self.app.post(
             route_path('pullrequest_comment_create',
                 repo_name=pull_request.target_repo.scm_instance().name,
                 pull_request_id=pull_request_id),
             params={
                 'close_pull_request': '1',
                 'csrf_token': csrf_token},
             extra_environ=xhr_header)
 
         pull_request = PullRequest.get(pull_request_id)
 
         journal = UserLog.query()\
             .filter(UserLog.user_id == author, UserLog.repository_id == repo) \
             .order_by(UserLog.user_log_id.asc()) \
             .all()
         assert journal[-1].action == 'repo.pull_request.close'
 
         # check only the latest status, not the review status
         status = ChangesetStatusModel().get_status(
             pull_request.source_repo, pull_request=pull_request)
         assert status == ChangesetStatus.STATUS_REJECTED
 
     def test_comment_and_close_pull_request(
             self, pr_util, csrf_token, xhr_header):
         pull_request = pr_util.create_pull_request()
         pull_request_id = pull_request.pull_request_id
 
         response = self.app.post(
             route_path('pullrequest_comment_create',
                        repo_name=pull_request.target_repo.scm_instance().name,
                        pull_request_id=pull_request.pull_request_id),
             params={
                 'close_pull_request': 'true',
                 'csrf_token': csrf_token},
             extra_environ=xhr_header)
 
         assert response.json
 
         pull_request = PullRequest.get(pull_request_id)
         assert pull_request.is_closed()
 
         # check only the latest status, not the review status
         status = ChangesetStatusModel().get_status(
             pull_request.source_repo, pull_request=pull_request)
         assert status == ChangesetStatus.STATUS_REJECTED
 
     def test_comment_and_close_pull_request_try_edit_comment(
             self, pr_util, csrf_token, xhr_header
     ):
         pull_request = pr_util.create_pull_request()
         pull_request_id = pull_request.pull_request_id
         target_scm = pull_request.target_repo.scm_instance()
         target_scm_name = target_scm.name
 
         response = self.app.post(
             route_path(
                 'pullrequest_comment_create',
                 repo_name=target_scm_name,
                 pull_request_id=pull_request_id,
             ),
             params={
                 'close_pull_request': 'true',
                 'csrf_token': csrf_token,
             },
             extra_environ=xhr_header)
 
         assert response.json
 
         pull_request = PullRequest.get(pull_request_id)
         target_scm = pull_request.target_repo.scm_instance()
         target_scm_name = target_scm.name
         assert pull_request.is_closed()
 
         # check only the latest status, not the review status
         status = ChangesetStatusModel().get_status(
             pull_request.source_repo, pull_request=pull_request)
         assert status == ChangesetStatus.STATUS_REJECTED
 
         comment_id = response.json.get('comment_id', None)
         test_text = 'test'
         response = self.app.post(
             route_path(
                 'pullrequest_comment_edit',
                 repo_name=target_scm_name,
                 pull_request_id=pull_request_id,
                 comment_id=comment_id,
             ),
             extra_environ=xhr_header,
             params={
                 'csrf_token': csrf_token,
                 'text': test_text,
             },
             status=403,
         )
         assert response.status_int == 403
 
     def test_comment_and_comment_edit(self, pr_util, csrf_token, xhr_header):
         pull_request = pr_util.create_pull_request()
         target_scm = pull_request.target_repo.scm_instance()
         target_scm_name = target_scm.name
 
         response = self.app.post(
             route_path(
                 'pullrequest_comment_create',
                 repo_name=target_scm_name,
                 pull_request_id=pull_request.pull_request_id),
             params={
                 'csrf_token': csrf_token,
                 'text': 'init',
             },
             extra_environ=xhr_header,
         )
         assert response.json
 
         comment_id = response.json.get('comment_id', None)
         assert comment_id
         test_text = 'test'
         self.app.post(
             route_path(
                 'pullrequest_comment_edit',
                 repo_name=target_scm_name,
                 pull_request_id=pull_request.pull_request_id,
                 comment_id=comment_id,
             ),
             extra_environ=xhr_header,
             params={
                 'csrf_token': csrf_token,
                 'text': test_text,
                 'version': '0',
             },
 
         )
         text_form_db = ChangesetComment.query().filter(
             ChangesetComment.comment_id == comment_id).first().text
         assert test_text == text_form_db
 
     def test_comment_and_comment_edit(self, pr_util, csrf_token, xhr_header):
         pull_request = pr_util.create_pull_request()
         target_scm = pull_request.target_repo.scm_instance()
         target_scm_name = target_scm.name
 
         response = self.app.post(
             route_path(
                 'pullrequest_comment_create',
                 repo_name=target_scm_name,
                 pull_request_id=pull_request.pull_request_id),
             params={
                 'csrf_token': csrf_token,
                 'text': 'init',
             },
             extra_environ=xhr_header,
         )
         assert response.json
 
         comment_id = response.json.get('comment_id', None)
         assert comment_id
         test_text = 'init'
         response = self.app.post(
             route_path(
                 'pullrequest_comment_edit',
                 repo_name=target_scm_name,
                 pull_request_id=pull_request.pull_request_id,
                 comment_id=comment_id,
             ),
             extra_environ=xhr_header,
             params={
                 'csrf_token': csrf_token,
                 'text': test_text,
                 'version': '0',
             },
             status=404,
 
         )
         assert response.status_int == 404
 
     def test_comment_and_try_edit_already_edited(self, pr_util, csrf_token, xhr_header):
         pull_request = pr_util.create_pull_request()
         target_scm = pull_request.target_repo.scm_instance()
         target_scm_name = target_scm.name
 
         response = self.app.post(
             route_path(
                 'pullrequest_comment_create',
                 repo_name=target_scm_name,
                 pull_request_id=pull_request.pull_request_id),
             params={
                 'csrf_token': csrf_token,
                 'text': 'init',
             },
             extra_environ=xhr_header,
         )
         assert response.json
         comment_id = response.json.get('comment_id', None)
         assert comment_id
 
         test_text = 'test'
         self.app.post(
             route_path(
                 'pullrequest_comment_edit',
                 repo_name=target_scm_name,
                 pull_request_id=pull_request.pull_request_id,
                 comment_id=comment_id,
             ),
             extra_environ=xhr_header,
             params={
                 'csrf_token': csrf_token,
                 'text': test_text,
                 'version': '0',
             },
 
         )
         test_text_v2 = 'test_v2'
         response = self.app.post(
             route_path(
                 'pullrequest_comment_edit',
                 repo_name=target_scm_name,
                 pull_request_id=pull_request.pull_request_id,
                 comment_id=comment_id,
             ),
             extra_environ=xhr_header,
             params={
                 'csrf_token': csrf_token,
                 'text': test_text_v2,
                 'version': '0',
             },
             status=409,
         )
         assert response.status_int == 409
 
         text_form_db = ChangesetComment.query().filter(
             ChangesetComment.comment_id == comment_id).first().text
 
         assert test_text == text_form_db
         assert test_text_v2 != text_form_db
 
     def test_comment_and_comment_edit_permissions_forbidden(
             self, autologin_regular_user, user_regular, user_admin, pr_util,
             csrf_token, xhr_header):
         pull_request = pr_util.create_pull_request(
             author=user_admin.username, enable_notifications=False)
         comment = CommentsModel().create(
             text='test',
             repo=pull_request.target_repo.scm_instance().name,
             user=user_admin,
             pull_request=pull_request,
         )
         response = self.app.post(
             route_path(
                 'pullrequest_comment_edit',
                 repo_name=pull_request.target_repo.scm_instance().name,
                 pull_request_id=pull_request.pull_request_id,
                 comment_id=comment.comment_id,
             ),
             extra_environ=xhr_header,
             params={
                 'csrf_token': csrf_token,
                 'text': 'test_text',
             },
             status=403,
         )
         assert response.status_int == 403
 
     def test_create_pull_request(self, backend, csrf_token):
         commits = [
             {'message': 'ancestor'},
             {'message': 'change'},
             {'message': 'change2'},
         ]
         commit_ids = backend.create_master_repo(commits)
         target = backend.create_repo(heads=['ancestor'])
         source = backend.create_repo(heads=['change2'])
 
         response = self.app.post(
             route_path('pullrequest_create', repo_name=source.repo_name),
             [
                 ('source_repo', source.repo_name),
                 ('source_ref', 'branch:default:' + commit_ids['change2']),
                 ('target_repo', target.repo_name),
                 ('target_ref',  'branch:default:' + commit_ids['ancestor']),
                 ('common_ancestor', commit_ids['ancestor']),
                 ('pullrequest_title', 'Title'),
                 ('pullrequest_desc', 'Description'),
                 ('description_renderer', 'markdown'),
                 ('__start__', 'review_members:sequence'),
                     ('__start__', 'reviewer:mapping'),
                         ('user_id', '1'),
                         ('__start__', 'reasons:sequence'),
                             ('reason', 'Some reason'),
                         ('__end__', 'reasons:sequence'),
                         ('__start__', 'rules:sequence'),
                         ('__end__', 'rules:sequence'),
                         ('mandatory', 'False'),
                     ('__end__', 'reviewer:mapping'),
                 ('__end__', 'review_members:sequence'),
                 ('__start__', 'revisions:sequence'),
                     ('revisions', commit_ids['change']),
                     ('revisions', commit_ids['change2']),
                 ('__end__', 'revisions:sequence'),
                 ('user', ''),
                 ('csrf_token', csrf_token),
             ],
             status=302)
 
         location = response.headers['Location']
         pull_request_id = location.rsplit('/', 1)[1]
         assert pull_request_id != 'new'
         pull_request = PullRequest.get(int(pull_request_id))
 
         # check that we have now both revisions
         assert pull_request.revisions == [commit_ids['change2'], commit_ids['change']]
         assert pull_request.source_ref == 'branch:default:' + commit_ids['change2']
         expected_target_ref = 'branch:default:' + commit_ids['ancestor']
         assert pull_request.target_ref == expected_target_ref
 
     def test_reviewer_notifications(self, backend, csrf_token):
         # We have to use the app.post for this test so it will create the
         # notifications properly with the new PR
         commits = [
             {'message': 'ancestor',
              'added': [FileNode('file_A', content='content_of_ancestor')]},
             {'message': 'change',
              'added': [FileNode('file_a', content='content_of_change')]},
             {'message': 'change-child'},
             {'message': 'ancestor-child', 'parents': ['ancestor'],
              'added': [
                 FileNode('file_B', content='content_of_ancestor_child')]},
             {'message': 'ancestor-child-2'},
         ]
         commit_ids = backend.create_master_repo(commits)
         target = backend.create_repo(heads=['ancestor-child'])
         source = backend.create_repo(heads=['change'])
 
         response = self.app.post(
             route_path('pullrequest_create', repo_name=source.repo_name),
             [
                 ('source_repo', source.repo_name),
                 ('source_ref', 'branch:default:' + commit_ids['change']),
                 ('target_repo', target.repo_name),
                 ('target_ref',  'branch:default:' + commit_ids['ancestor-child']),
                 ('common_ancestor', commit_ids['ancestor']),
                 ('pullrequest_title', 'Title'),
                 ('pullrequest_desc', 'Description'),
                 ('description_renderer', 'markdown'),
                 ('__start__', 'review_members:sequence'),
                     ('__start__', 'reviewer:mapping'),
                         ('user_id', '2'),
                         ('__start__', 'reasons:sequence'),
                             ('reason', 'Some reason'),
                         ('__end__', 'reasons:sequence'),
                         ('__start__', 'rules:sequence'),
                         ('__end__', 'rules:sequence'),
                         ('mandatory', 'False'),
                     ('__end__', 'reviewer:mapping'),
                 ('__end__', 'review_members:sequence'),
                 ('__start__', 'revisions:sequence'),
                     ('revisions', commit_ids['change']),
                 ('__end__', 'revisions:sequence'),
                 ('user', ''),
                 ('csrf_token', csrf_token),
             ],
             status=302)
 
         location = response.headers['Location']
 
         pull_request_id = location.rsplit('/', 1)[1]
         assert pull_request_id != 'new'
         pull_request = PullRequest.get(int(pull_request_id))
 
         # Check that a notification was made
         notifications = Notification.query()\
             .filter(Notification.created_by == pull_request.author.user_id,
                     Notification.type_ == Notification.TYPE_PULL_REQUEST,
                     Notification.subject.contains(
                         "requested a pull request review. !%s" % pull_request_id))
         assert len(notifications.all()) == 1
 
         # Change reviewers and check that a notification was made
         PullRequestModel().update_reviewers(
             pull_request.pull_request_id, [(1, [], False, [])],
             pull_request.author)
         assert len(notifications.all()) == 2
 
     def test_create_pull_request_stores_ancestor_commit_id(self, backend,
                                                            csrf_token):
         commits = [
             {'message': 'ancestor',
              'added': [FileNode('file_A', content='content_of_ancestor')]},
             {'message': 'change',
              'added': [FileNode('file_a', content='content_of_change')]},
             {'message': 'change-child'},
             {'message': 'ancestor-child', 'parents': ['ancestor'],
              'added': [
                 FileNode('file_B', content='content_of_ancestor_child')]},
             {'message': 'ancestor-child-2'},
         ]
         commit_ids = backend.create_master_repo(commits)
         target = backend.create_repo(heads=['ancestor-child'])
         source = backend.create_repo(heads=['change'])
 
         response = self.app.post(
             route_path('pullrequest_create', repo_name=source.repo_name),
             [
                 ('source_repo', source.repo_name),
                 ('source_ref', 'branch:default:' + commit_ids['change']),
                 ('target_repo', target.repo_name),
                 ('target_ref',  'branch:default:' + commit_ids['ancestor-child']),
                 ('common_ancestor', commit_ids['ancestor']),
                 ('pullrequest_title', 'Title'),
                 ('pullrequest_desc', 'Description'),
                 ('description_renderer', 'markdown'),
                 ('__start__', 'review_members:sequence'),
                     ('__start__', 'reviewer:mapping'),
                         ('user_id', '1'),
                         ('__start__', 'reasons:sequence'),
                             ('reason', 'Some reason'),
                         ('__end__', 'reasons:sequence'),
                         ('__start__', 'rules:sequence'),
                         ('__end__', 'rules:sequence'),
                         ('mandatory', 'False'),
                     ('__end__', 'reviewer:mapping'),
                 ('__end__', 'review_members:sequence'),
                 ('__start__', 'revisions:sequence'),
                     ('revisions', commit_ids['change']),
                 ('__end__', 'revisions:sequence'),
                 ('user', ''),
                 ('csrf_token', csrf_token),
             ],
             status=302)
 
         location = response.headers['Location']
 
         pull_request_id = location.rsplit('/', 1)[1]
         assert pull_request_id != 'new'
         pull_request = PullRequest.get(int(pull_request_id))
 
         # target_ref has to point to the ancestor's commit_id in order to
         # show the correct diff
         expected_target_ref = 'branch:default:' + commit_ids['ancestor']
         assert pull_request.target_ref == expected_target_ref
 
         # Check generated diff contents
         response = response.follow()
         response.mustcontain(no=['content_of_ancestor'])
         response.mustcontain(no=['content_of_ancestor-child'])
         response.mustcontain('content_of_change')
 
     def test_merge_pull_request_enabled(self, pr_util, csrf_token):
         # Clear any previous calls to rcextensions
         rhodecode.EXTENSIONS.calls.clear()
 
         pull_request = pr_util.create_pull_request(
             approved=True, mergeable=True)
         pull_request_id = pull_request.pull_request_id
         repo_name = pull_request.target_repo.scm_instance().name,
 
         url = route_path('pullrequest_merge',
                          repo_name=str(repo_name[0]),
                          pull_request_id=pull_request_id)
         response = self.app.post(url, params={'csrf_token': csrf_token}).follow()
 
         pull_request = PullRequest.get(pull_request_id)
 
         assert response.status_int == 200
         assert pull_request.is_closed()
         assert_pull_request_status(
             pull_request, ChangesetStatus.STATUS_APPROVED)
 
         # Check the relevant log entries were added
         user_logs = UserLog.query().order_by(UserLog.user_log_id.desc()).limit(3)
         actions = [log.action for log in user_logs]
         pr_commit_ids = PullRequestModel()._get_commit_ids(pull_request)
         expected_actions = [
              u'repo.pull_request.close',
              u'repo.pull_request.merge',
              u'repo.pull_request.comment.create'
         ]
         assert actions == expected_actions
 
         user_logs = UserLog.query().order_by(UserLog.user_log_id.desc()).limit(4)
         actions = [log for log in user_logs]
         assert actions[-1].action == 'user.push'
         assert actions[-1].action_data['commit_ids'] == pr_commit_ids
 
         # Check post_push rcextension was really executed
         push_calls = rhodecode.EXTENSIONS.calls['_push_hook']
         assert len(push_calls) == 1
         unused_last_call_args, last_call_kwargs = push_calls[0]
         assert last_call_kwargs['action'] == 'push'
         assert last_call_kwargs['commit_ids'] == pr_commit_ids
 
     def test_merge_pull_request_disabled(self, pr_util, csrf_token):
         pull_request = pr_util.create_pull_request(mergeable=False)
         pull_request_id = pull_request.pull_request_id
         pull_request = PullRequest.get(pull_request_id)
 
         response = self.app.post(
             route_path('pullrequest_merge',
                 repo_name=pull_request.target_repo.scm_instance().name,
                 pull_request_id=pull_request.pull_request_id),
             params={'csrf_token': csrf_token}).follow()
 
         assert response.status_int == 200
         response.mustcontain(
             'Merge is not currently possible because of below failed checks.')
         response.mustcontain('Server-side pull request merging is disabled.')
 
     @pytest.mark.skip_backends('svn')
     def test_merge_pull_request_not_approved(self, pr_util, csrf_token):
         pull_request = pr_util.create_pull_request(mergeable=True)
         pull_request_id = pull_request.pull_request_id
         repo_name = pull_request.target_repo.scm_instance().name
 
         response = self.app.post(
             route_path('pullrequest_merge',
                        repo_name=repo_name, pull_request_id=pull_request_id),
             params={'csrf_token': csrf_token}).follow()
 
         assert response.status_int == 200
 
         response.mustcontain(
             'Merge is not currently possible because of below failed checks.')
         response.mustcontain('Pull request reviewer approval is pending.')
 
     def test_merge_pull_request_renders_failure_reason(
             self, user_regular, csrf_token, pr_util):
         pull_request = pr_util.create_pull_request(mergeable=True, approved=True)
         pull_request_id = pull_request.pull_request_id
         repo_name = pull_request.target_repo.scm_instance().name
 
         merge_resp = MergeResponse(True, False, 'STUB_COMMIT_ID',
                                    MergeFailureReason.PUSH_FAILED,
                                    metadata={'target': 'shadow repo',
                                              'merge_commit': 'xxx'})
         model_patcher = mock.patch.multiple(
             PullRequestModel,
             merge_repo=mock.Mock(return_value=merge_resp),
             merge_status=mock.Mock(return_value=(None, True, 'WRONG_MESSAGE')))
 
         with model_patcher:
             response = self.app.post(
                 route_path('pullrequest_merge',
                            repo_name=repo_name,
                            pull_request_id=pull_request_id),
                 params={'csrf_token': csrf_token}, status=302)
 
             merge_resp = MergeResponse(True, True, '', MergeFailureReason.PUSH_FAILED,
                                        metadata={'target': 'shadow repo',
                                                  'merge_commit': 'xxx'})
         assert_session_flash(response, merge_resp.merge_status_message)
 
     def test_update_source_revision(self, backend, csrf_token):
         commits = [
             {'message': 'ancestor'},
             {'message': 'change'},
             {'message': 'change-2'},
         ]
         commit_ids = backend.create_master_repo(commits)
         target = backend.create_repo(heads=['ancestor'])
         source = backend.create_repo(heads=['change'])
 
         # create pr from a in source to A in target
         pull_request = PullRequest()
 
         pull_request.source_repo = source
         pull_request.source_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name, commit_id=commit_ids['change'])
 
         pull_request.target_repo = target
         pull_request.target_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name, commit_id=commit_ids['ancestor'])
 
         pull_request.revisions = [commit_ids['change']]
         pull_request.title = u"Test"
         pull_request.description = u"Description"
         pull_request.author = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN)
         pull_request.pull_request_state = PullRequest.STATE_CREATED
         Session().add(pull_request)
         Session().commit()
         pull_request_id = pull_request.pull_request_id
 
         # source has ancestor - change - change-2
         backend.pull_heads(source, heads=['change-2'])
+        target_repo_name = target.repo_name
 
         # update PR
         self.app.post(
             route_path('pullrequest_update',
-                       repo_name=target.repo_name, pull_request_id=pull_request_id),
+                       repo_name=target_repo_name, pull_request_id=pull_request_id),
             params={'update_commits': 'true', 'csrf_token': csrf_token})
 
         response = self.app.get(
             route_path('pullrequest_show',
-                       repo_name=target.repo_name,
+                       repo_name=target_repo_name,
                        pull_request_id=pull_request.pull_request_id))
 
         assert response.status_int == 200
         response.mustcontain('Pull request updated to')
         response.mustcontain('with 1 added, 0 removed commits.')
 
         # check that we have now both revisions
         pull_request = PullRequest.get(pull_request_id)
         assert pull_request.revisions == [commit_ids['change-2'], commit_ids['change']]
 
     def test_update_target_revision(self, backend, csrf_token):
         commits = [
             {'message': 'ancestor'},
             {'message': 'change'},
             {'message': 'ancestor-new', 'parents': ['ancestor']},
             {'message': 'change-rebased'},
         ]
         commit_ids = backend.create_master_repo(commits)
         target = backend.create_repo(heads=['ancestor'])
         source = backend.create_repo(heads=['change'])
 
         # create pr from a in source to A in target
         pull_request = PullRequest()
 
         pull_request.source_repo = source
         pull_request.source_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name, commit_id=commit_ids['change'])
 
         pull_request.target_repo = target
         pull_request.target_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name, commit_id=commit_ids['ancestor'])
 
         pull_request.revisions = [commit_ids['change']]
         pull_request.title = u"Test"
         pull_request.description = u"Description"
         pull_request.author = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN)
         pull_request.pull_request_state = PullRequest.STATE_CREATED
 
         Session().add(pull_request)
         Session().commit()
         pull_request_id = pull_request.pull_request_id
 
         # target has ancestor - ancestor-new
         # source has ancestor - ancestor-new - change-rebased
         backend.pull_heads(target, heads=['ancestor-new'])
         backend.pull_heads(source, heads=['change-rebased'])
+        target_repo_name = target.repo_name
 
         # update PR
         url = route_path('pullrequest_update',
-                         repo_name=target.repo_name,
+                         repo_name=target_repo_name,
                          pull_request_id=pull_request_id)
         self.app.post(url,
                       params={'update_commits': 'true', 'csrf_token': csrf_token},
                       status=200)
 
         # check that we have now both revisions
         pull_request = PullRequest.get(pull_request_id)
         assert pull_request.revisions == [commit_ids['change-rebased']]
         assert pull_request.target_ref == 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name, commit_id=commit_ids['ancestor-new'])
 
         response = self.app.get(
             route_path('pullrequest_show',
-                       repo_name=target.repo_name,
+                       repo_name=target_repo_name,
                        pull_request_id=pull_request.pull_request_id))
         assert response.status_int == 200
         response.mustcontain('Pull request updated to')
         response.mustcontain('with 1 added, 1 removed commits.')
 
     def test_update_target_revision_with_removal_of_1_commit_git(self, backend_git, csrf_token):
         backend = backend_git
         commits = [
             {'message': 'master-commit-1'},
             {'message': 'master-commit-2-change-1'},
             {'message': 'master-commit-3-change-2'},
 
             {'message': 'feat-commit-1', 'parents': ['master-commit-1']},
             {'message': 'feat-commit-2'},
         ]
         commit_ids = backend.create_master_repo(commits)
         target = backend.create_repo(heads=['master-commit-3-change-2'])
         source = backend.create_repo(heads=['feat-commit-2'])
 
         # create pr from a in source to A in target
         pull_request = PullRequest()
         pull_request.source_repo = source
 
         pull_request.source_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name,
             commit_id=commit_ids['master-commit-3-change-2'])
 
         pull_request.target_repo = target
         pull_request.target_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name, commit_id=commit_ids['feat-commit-2'])
 
         pull_request.revisions = [
             commit_ids['feat-commit-1'],
             commit_ids['feat-commit-2']
         ]
         pull_request.title = u"Test"
         pull_request.description = u"Description"
         pull_request.author = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN)
         pull_request.pull_request_state = PullRequest.STATE_CREATED
         Session().add(pull_request)
         Session().commit()
         pull_request_id = pull_request.pull_request_id
 
         # PR is created, now we simulate a force-push into target,
         # that drops a 2 last commits
         vcsrepo = target.scm_instance()
         vcsrepo.config.clear_section('hooks')
         vcsrepo.run_git_command(['reset', '--soft', 'HEAD~2'])
+        target_repo_name = target.repo_name
 
         # update PR
         url = route_path('pullrequest_update',
-                         repo_name=target.repo_name,
+                         repo_name=target_repo_name,
                          pull_request_id=pull_request_id)
         self.app.post(url,
                       params={'update_commits': 'true', 'csrf_token': csrf_token},
                       status=200)
 
-        response = self.app.get(route_path('pullrequest_new', repo_name=target.repo_name))
+        response = self.app.get(route_path('pullrequest_new', repo_name=target_repo_name))
         assert response.status_int == 200
         response.mustcontain('Pull request updated to')
         response.mustcontain('with 0 added, 0 removed commits.')
 
     def test_update_of_ancestor_reference(self, backend, csrf_token):
         commits = [
             {'message': 'ancestor'},
             {'message': 'change'},
             {'message': 'change-2'},
             {'message': 'ancestor-new', 'parents': ['ancestor']},
             {'message': 'change-rebased'},
         ]
         commit_ids = backend.create_master_repo(commits)
         target = backend.create_repo(heads=['ancestor'])
         source = backend.create_repo(heads=['change'])
 
         # create pr from a in source to A in target
         pull_request = PullRequest()
         pull_request.source_repo = source
 
         pull_request.source_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name, commit_id=commit_ids['change'])
         pull_request.target_repo = target
         pull_request.target_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name, commit_id=commit_ids['ancestor'])
         pull_request.revisions = [commit_ids['change']]
         pull_request.title = u"Test"
         pull_request.description = u"Description"
         pull_request.author = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN)
         pull_request.pull_request_state = PullRequest.STATE_CREATED
         Session().add(pull_request)
         Session().commit()
         pull_request_id = pull_request.pull_request_id
 
         # target has ancestor - ancestor-new
         # source has ancestor - ancestor-new - change-rebased
         backend.pull_heads(target, heads=['ancestor-new'])
         backend.pull_heads(source, heads=['change-rebased'])
+        target_repo_name = target.repo_name
 
         # update PR
         self.app.post(
             route_path('pullrequest_update',
-                       repo_name=target.repo_name, pull_request_id=pull_request_id),
+                       repo_name=target_repo_name, pull_request_id=pull_request_id),
             params={'update_commits': 'true', 'csrf_token': csrf_token},
             status=200)
 
         # Expect the target reference to be updated correctly
         pull_request = PullRequest.get(pull_request_id)
         assert pull_request.revisions == [commit_ids['change-rebased']]
         expected_target_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name,
             commit_id=commit_ids['ancestor-new'])
         assert pull_request.target_ref == expected_target_ref
 
     def test_remove_pull_request_branch(self, backend_git, csrf_token):
         branch_name = 'development'
         commits = [
             {'message': 'initial-commit'},
             {'message': 'old-feature'},
             {'message': 'new-feature', 'branch': branch_name},
         ]
         repo = backend_git.create_repo(commits)
         repo_name = repo.repo_name
         commit_ids = backend_git.commit_ids
 
         pull_request = PullRequest()
         pull_request.source_repo = repo
         pull_request.target_repo = repo
         pull_request.source_ref = 'branch:{branch}:{commit_id}'.format(
             branch=branch_name, commit_id=commit_ids['new-feature'])
         pull_request.target_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend_git.default_branch_name, commit_id=commit_ids['old-feature'])
         pull_request.revisions = [commit_ids['new-feature']]
         pull_request.title = u"Test"
         pull_request.description = u"Description"
         pull_request.author = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN)
         pull_request.pull_request_state = PullRequest.STATE_CREATED
         Session().add(pull_request)
         Session().commit()
 
         pull_request_id = pull_request.pull_request_id
 
         vcs = repo.scm_instance()
         vcs.remove_ref('refs/heads/{}'.format(branch_name))
         # NOTE(marcink): run GC to ensure the commits are gone
         vcs.run_gc()
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=repo_name,
             pull_request_id=pull_request_id))
 
         assert response.status_int == 200
 
         response.assert_response().element_contains(
             '#changeset_compare_view_content .alert strong',
             'Missing commits')
         response.assert_response().element_contains(
             '#changeset_compare_view_content .alert',
             'This pull request cannot be displayed, because one or more'
             ' commits no longer exist in the source repository.')
 
     def test_strip_commits_from_pull_request(
             self, backend, pr_util, csrf_token):
         commits = [
             {'message': 'initial-commit'},
             {'message': 'old-feature'},
             {'message': 'new-feature', 'parents': ['initial-commit']},
         ]
         pull_request = pr_util.create_pull_request(
             commits, target_head='initial-commit', source_head='new-feature',
             revisions=['new-feature'])
 
         vcs = pr_util.source_repository.scm_instance()
         if backend.alias == 'git':
             vcs.strip(pr_util.commit_ids['new-feature'], branch_name='master')
         else:
             vcs.strip(pr_util.commit_ids['new-feature'])
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pr_util.target_repository.repo_name,
             pull_request_id=pull_request.pull_request_id))
 
         assert response.status_int == 200
 
         response.assert_response().element_contains(
             '#changeset_compare_view_content .alert strong',
             'Missing commits')
         response.assert_response().element_contains(
             '#changeset_compare_view_content .alert',
             'This pull request cannot be displayed, because one or more'
             ' commits no longer exist in the source repository.')
         response.assert_response().element_contains(
             '#update_commits',
             'Update commits')
 
     def test_strip_commits_and_update(
             self, backend, pr_util, csrf_token):
         commits = [
             {'message': 'initial-commit'},
             {'message': 'old-feature'},
             {'message': 'new-feature', 'parents': ['old-feature']},
         ]
         pull_request = pr_util.create_pull_request(
             commits, target_head='old-feature', source_head='new-feature',
             revisions=['new-feature'], mergeable=True)
+        pr_id = pull_request.pull_request_id
+        target_repo_name = pull_request.target_repo.repo_name
 
         vcs = pr_util.source_repository.scm_instance()
         if backend.alias == 'git':
             vcs.strip(pr_util.commit_ids['new-feature'], branch_name='master')
         else:
             vcs.strip(pr_util.commit_ids['new-feature'])
 
         url = route_path('pullrequest_update',
-                         repo_name=pull_request.target_repo.repo_name,
-                         pull_request_id=pull_request.pull_request_id)
+                         repo_name=target_repo_name,
+                         pull_request_id=pr_id)
         response = self.app.post(url,
                                  params={'update_commits': 'true',
                                          'csrf_token': csrf_token})
 
         assert response.status_int == 200
         assert response.body == '{"response": true, "redirect_url": null}'
 
         # Make sure that after update, it won't raise 500 errors
         response = self.app.get(route_path(
             'pullrequest_show',
-            repo_name=pr_util.target_repository.repo_name,
-            pull_request_id=pull_request.pull_request_id))
+            repo_name=target_repo_name,
+            pull_request_id=pr_id))
 
         assert response.status_int == 200
         response.assert_response().element_contains(
             '#changeset_compare_view_content .alert strong',
             'Missing commits')
 
     def test_branch_is_a_link(self, pr_util):
         pull_request = pr_util.create_pull_request()
         pull_request.source_ref = 'branch:origin:1234567890abcdef'
         pull_request.target_ref = 'branch:target:abcdef1234567890'
         Session().add(pull_request)
         Session().commit()
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
         assert response.status_int == 200
 
         source = response.assert_response().get_element('.pr-source-info')
         source_parent = source.getparent()
         assert len(source_parent) == 1
 
         target = response.assert_response().get_element('.pr-target-info')
         target_parent = target.getparent()
         assert len(target_parent) == 1
 
         expected_origin_link = route_path(
             'repo_commits',
             repo_name=pull_request.source_repo.scm_instance().name,
             params=dict(branch='origin'))
         expected_target_link = route_path(
             'repo_commits',
             repo_name=pull_request.target_repo.scm_instance().name,
             params=dict(branch='target'))
         assert source_parent.attrib['href'] == expected_origin_link
         assert target_parent.attrib['href'] == expected_target_link
 
     def test_bookmark_is_not_a_link(self, pr_util):
         pull_request = pr_util.create_pull_request()
         pull_request.source_ref = 'bookmark:origin:1234567890abcdef'
         pull_request.target_ref = 'bookmark:target:abcdef1234567890'
         Session().add(pull_request)
         Session().commit()
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
         assert response.status_int == 200
 
         source = response.assert_response().get_element('.pr-source-info')
         assert source.text.strip() == 'bookmark:origin'
         assert source.getparent().attrib.get('href') is None
 
         target = response.assert_response().get_element('.pr-target-info')
         assert target.text.strip() == 'bookmark:target'
         assert target.getparent().attrib.get('href') is None
 
     def test_tag_is_not_a_link(self, pr_util):
         pull_request = pr_util.create_pull_request()
         pull_request.source_ref = 'tag:origin:1234567890abcdef'
         pull_request.target_ref = 'tag:target:abcdef1234567890'
         Session().add(pull_request)
         Session().commit()
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
         assert response.status_int == 200
 
         source = response.assert_response().get_element('.pr-source-info')
         assert source.text.strip() == 'tag:origin'
         assert source.getparent().attrib.get('href') is None
 
         target = response.assert_response().get_element('.pr-target-info')
         assert target.text.strip() == 'tag:target'
         assert target.getparent().attrib.get('href') is None
 
     @pytest.mark.parametrize('mergeable', [True, False])
     def test_shadow_repository_link(
             self, mergeable, pr_util, http_host_only_stub):
         """
         Check that the pull request summary page displays a link to the shadow
         repository if the pull request is mergeable. If it is not mergeable
         the link should not be displayed.
         """
         pull_request = pr_util.create_pull_request(
             mergeable=mergeable, enable_notifications=False)
         target_repo = pull_request.target_repo.scm_instance()
         pr_id = pull_request.pull_request_id
         shadow_url = '{host}/{repo}/pull-request/{pr_id}/repository'.format(
             host=http_host_only_stub, repo=target_repo.name, pr_id=pr_id)
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=target_repo.name,
             pull_request_id=pr_id))
 
         if mergeable:
             response.assert_response().element_value_contains(
                 'input.pr-mergeinfo', shadow_url)
             response.assert_response().element_value_contains(
                 'input.pr-mergeinfo ', 'pr-merge')
         else:
             response.assert_response().no_element_exists('.pr-mergeinfo')
 
 
 @pytest.mark.usefixtures('app')
 @pytest.mark.backends("git", "hg")
 class TestPullrequestsControllerDelete(object):
     def test_pull_request_delete_button_permissions_admin(
             self, autologin_user, user_admin, pr_util):
         pull_request = pr_util.create_pull_request(
             author=user_admin.username, enable_notifications=False)
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
 
         response.mustcontain('id="delete_pullrequest"')
         response.mustcontain('Confirm to delete this pull request')
 
     def test_pull_request_delete_button_permissions_owner(
             self, autologin_regular_user, user_regular, pr_util):
         pull_request = pr_util.create_pull_request(
             author=user_regular.username, enable_notifications=False)
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
 
         response.mustcontain('id="delete_pullrequest"')
         response.mustcontain('Confirm to delete this pull request')
 
     def test_pull_request_delete_button_permissions_forbidden(
             self, autologin_regular_user, user_regular, user_admin, pr_util):
         pull_request = pr_util.create_pull_request(
             author=user_admin.username, enable_notifications=False)
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
         response.mustcontain(no=['id="delete_pullrequest"'])
         response.mustcontain(no=['Confirm to delete this pull request'])
 
     def test_pull_request_delete_button_permissions_can_update_cannot_delete(
             self, autologin_regular_user, user_regular, user_admin, pr_util,
             user_util):
 
         pull_request = pr_util.create_pull_request(
             author=user_admin.username, enable_notifications=False)
 
         user_util.grant_user_permission_to_repo(
             pull_request.target_repo, user_regular,
             'repository.write')
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
 
         response.mustcontain('id="open_edit_pullrequest"')
         response.mustcontain('id="delete_pullrequest"')
         response.mustcontain(no=['Confirm to delete this pull request'])
 
     def test_delete_comment_returns_404_if_comment_does_not_exist(
             self, autologin_user, pr_util, user_admin, csrf_token, xhr_header):
 
         pull_request = pr_util.create_pull_request(
             author=user_admin.username, enable_notifications=False)
 
         self.app.post(
             route_path(
             'pullrequest_comment_delete',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id,
             comment_id=1024404),
             extra_environ=xhr_header,
             params={'csrf_token': csrf_token},
             status=404
         )
 
     def test_delete_comment(
             self, autologin_user, pr_util, user_admin, csrf_token, xhr_header):
 
         pull_request = pr_util.create_pull_request(
             author=user_admin.username, enable_notifications=False)
         comment = pr_util.create_comment()
         comment_id = comment.comment_id
 
         response = self.app.post(
             route_path(
             'pullrequest_comment_delete',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id,
             comment_id=comment_id),
             extra_environ=xhr_header,
             params={'csrf_token': csrf_token},
             status=200
         )
         assert response.body == 'true'
 
     @pytest.mark.parametrize('url_type', [
         'pullrequest_new',
         'pullrequest_create',
         'pullrequest_update',
         'pullrequest_merge',
     ])
     def test_pull_request_is_forbidden_on_archived_repo(
             self, autologin_user, backend, xhr_header, user_util, url_type):
 
         # create a temporary repo
         source = user_util.create_repo(repo_type=backend.alias)
         repo_name = source.repo_name
         repo = Repository.get_by_repo_name(repo_name)
         repo.archived = True
         Session().commit()
 
         response = self.app.get(
             route_path(url_type, repo_name=repo_name, pull_request_id=1), status=302)
 
         msg = 'Action not supported for archived repository.'
         assert_session_flash(response, msg)
 
 
 def assert_pull_request_status(pull_request, expected_status):
     status = ChangesetStatusModel().calculated_review_status(pull_request=pull_request)
     assert status == expected_status
 
 
 @pytest.mark.parametrize('route', ['pullrequest_new', 'pullrequest_create'])
 @pytest.mark.usefixtures("autologin_user")
 def test_forbidde_to_repo_summary_for_svn_repositories(backend_svn, app, route):
     app.get(route_path(route, repo_name=backend_svn.repo_name), status=404)

rhodecode/apps/repository/views/repo_commits.py

@@ -1,723 +1,781 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
-
 import logging
+import collections
 
 from pyramid.httpexceptions import (
     HTTPNotFound, HTTPBadRequest, HTTPFound, HTTPForbidden, HTTPConflict)
 from pyramid.view import view_config
 from pyramid.renderers import render
 from pyramid.response import Response
 
 from rhodecode.apps._base import RepoAppView
 from rhodecode.apps.file_store import utils as store_utils
 from rhodecode.apps.file_store.exceptions import FileNotAllowedException, FileOverSizeException
 
 from rhodecode.lib import diffs, codeblocks
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAnyDecorator, NotAnonymous, CSRFRequired)
-
+from rhodecode.lib.ext_json import json
 from rhodecode.lib.compat import OrderedDict
 from rhodecode.lib.diffs import (
     cache_diff, load_cached_diff, diff_cache_exist, get_diff_context,
     get_diff_whitespace_flag)
 from rhodecode.lib.exceptions import StatusChangeOnClosedPullRequestError, CommentVersionMismatch
 import rhodecode.lib.helpers as h
-from rhodecode.lib.utils2 import safe_unicode, str2bool
+from rhodecode.lib.utils2 import safe_unicode, str2bool, StrictAttributeDict
 from rhodecode.lib.vcs.backends.base import EmptyCommit
 from rhodecode.lib.vcs.exceptions import (
     RepositoryError, CommitDoesNotExistError)
 from rhodecode.model.db import ChangesetComment, ChangesetStatus, FileStore, \
     ChangesetCommentHistory
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.meta import Session
 from rhodecode.model.settings import VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
 
 def _update_with_GET(params, request):
     for k in ['diff1', 'diff2', 'diff']:
         params[k] += request.GET.getall(k)
 
 
 class RepoCommitsView(RepoAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context(include_app_defaults=True)
         c.rhodecode_repo = self.rhodecode_vcs_repo
 
         return c
 
     def _is_diff_cache_enabled(self, target_repo):
         caching_enabled = self._get_general_setting(
             target_repo, 'rhodecode_diff_cache')
         log.debug('Diff caching enabled: %s', caching_enabled)
         return caching_enabled
 
     def _commit(self, commit_id_range, method):
         _ = self.request.translate
         c = self.load_default_context()
         c.fulldiff = self.request.GET.get('fulldiff')
 
         # fetch global flags of ignore ws or context lines
         diff_context = get_diff_context(self.request)
         hide_whitespace_changes = get_diff_whitespace_flag(self.request)
 
         # diff_limit will cut off the whole diff if the limit is applied
         # otherwise it will just hide the big files from the front-end
         diff_limit = c.visual.cut_off_limit_diff
         file_limit = c.visual.cut_off_limit_file
 
         # get ranges of commit ids if preset
         commit_range = commit_id_range.split('...')[:2]
 
         try:
             pre_load = ['affected_files', 'author', 'branch', 'date',
                         'message', 'parents']
             if self.rhodecode_vcs_repo.alias == 'hg':
                 pre_load += ['hidden', 'obsolete', 'phase']
 
             if len(commit_range) == 2:
                 commits = self.rhodecode_vcs_repo.get_commits(
                     start_id=commit_range[0], end_id=commit_range[1],
                     pre_load=pre_load, translate_tags=False)
                 commits = list(commits)
             else:
                 commits = [self.rhodecode_vcs_repo.get_commit(
                     commit_id=commit_id_range, pre_load=pre_load)]
 
             c.commit_ranges = commits
             if not c.commit_ranges:
                 raise RepositoryError('The commit range returned an empty result')
         except CommitDoesNotExistError as e:
             msg = _('No such commit exists. Org exception: `{}`').format(e)
             h.flash(msg, category='error')
             raise HTTPNotFound()
         except Exception:
             log.exception("General failure")
             raise HTTPNotFound()
+        single_commit = len(c.commit_ranges) == 1
 
         c.changes = OrderedDict()
         c.lines_added = 0
         c.lines_deleted = 0
 
         # auto collapse if we have more than limit
         collapse_limit = diffs.DiffProcessor._collapse_commits_over
         c.collapse_all_commits = len(c.commit_ranges) > collapse_limit
 
         c.commit_statuses = ChangesetStatus.STATUSES
         c.inline_comments = []
         c.files = []
 
-        c.statuses = []
         c.comments = []
         c.unresolved_comments = []
         c.resolved_comments = []
-        if len(c.commit_ranges) == 1:
+
+        # Single commit
+        if single_commit:
             commit = c.commit_ranges[0]
             c.comments = CommentsModel().get_comments(
                 self.db_repo.repo_id,
                 revision=commit.raw_id)
-            c.statuses.append(ChangesetStatusModel().get_status(
-                self.db_repo.repo_id, commit.raw_id))
+
             # comments from PR
             statuses = ChangesetStatusModel().get_statuses(
                 self.db_repo.repo_id, commit.raw_id,
                 with_revisions=True)
-            prs = set(st.pull_request for st in statuses
-                      if st.pull_request is not None)
+
+            prs = set()
+            reviewers = list()
+            reviewers_duplicates = set()  # to not have duplicates from multiple votes
+            for c_status in statuses:
+
+                # extract associated pull-requests from votes
+                if c_status.pull_request:
+                    prs.add(c_status.pull_request)
+
+                # extract reviewers
+                _user_id = c_status.author.user_id
+                if _user_id not in reviewers_duplicates:
+                    reviewers.append(
+                        StrictAttributeDict({
+                            'user': c_status.author,
+
+                            # fake attributed for commit, page that we don't have
+                            # but we share the display with PR page
+                            'mandatory': False,
+                            'reasons': [],
+                            'rule_user_group_data': lambda: None
+                        })
+                    )
+                    reviewers_duplicates.add(_user_id)
+
+            c.allowed_reviewers = reviewers
             # from associated statuses, check the pull requests, and
             # show comments from them
             for pr in prs:
                 c.comments.extend(pr.comments)
 
             c.unresolved_comments = CommentsModel()\
                 .get_commit_unresolved_todos(commit.raw_id)
             c.resolved_comments = CommentsModel()\
                 .get_commit_resolved_todos(commit.raw_id)
 
+            c.inline_comments_flat = CommentsModel()\
+                .get_commit_inline_comments(commit.raw_id)
+
+            review_statuses = ChangesetStatusModel().aggregate_votes_by_user(
+                statuses, reviewers)
+
+            c.commit_review_status = ChangesetStatus.STATUS_NOT_REVIEWED
+
+            c.commit_set_reviewers_data_json = collections.OrderedDict({'reviewers': []})
+
+            for review_obj, member, reasons, mandatory, status in review_statuses:
+                member_reviewer = h.reviewer_as_json(
+                    member, reasons=reasons, mandatory=mandatory,
+                    user_group=None
+                )
+
+                current_review_status = status[0][1].status if status else ChangesetStatus.STATUS_NOT_REVIEWED
+                member_reviewer['review_status'] = current_review_status
+                member_reviewer['review_status_label'] = h.commit_status_lbl(current_review_status)
+                member_reviewer['allowed_to_update'] = False
+                c.commit_set_reviewers_data_json['reviewers'].append(member_reviewer)
+
+            c.commit_set_reviewers_data_json = json.dumps(c.commit_set_reviewers_data_json)
+
+            # NOTE(marcink): this uses the same voting logic as in pull-requests
+            c.commit_review_status = ChangesetStatusModel().calculate_status(review_statuses)
+            c.commit_broadcast_channel = u'/repo${}$/commit/{}'.format(
+                c.repo_name,
+                commit.raw_id
+            )
+
         diff = None
         # Iterate over ranges (default commit view is always one commit)
         for commit in c.commit_ranges:
             c.changes[commit.raw_id] = []
 
             commit2 = commit
             commit1 = commit.first_parent
 
             if method == 'show':
                 inline_comments = CommentsModel().get_inline_comments(
                     self.db_repo.repo_id, revision=commit.raw_id)
-                c.inline_cnt = CommentsModel().get_inline_comments_count(
-                    inline_comments)
+                c.inline_cnt = len(CommentsModel().get_inline_comments_as_list(
+                    inline_comments))
                 c.inline_comments = inline_comments
 
                 cache_path = self.rhodecode_vcs_repo.get_create_shadow_cache_pr_path(
                     self.db_repo)
                 cache_file_path = diff_cache_exist(
                     cache_path, 'diff', commit.raw_id,
                     hide_whitespace_changes, diff_context, c.fulldiff)
 
                 caching_enabled = self._is_diff_cache_enabled(self.db_repo)
                 force_recache = str2bool(self.request.GET.get('force_recache'))
 
                 cached_diff = None
                 if caching_enabled:
                     cached_diff = load_cached_diff(cache_file_path)
 
                 has_proper_diff_cache = cached_diff and cached_diff.get('diff')
                 if not force_recache and has_proper_diff_cache:
                     diffset = cached_diff['diff']
                 else:
                     vcs_diff = self.rhodecode_vcs_repo.get_diff(
                         commit1, commit2,
                         ignore_whitespace=hide_whitespace_changes,
                         context=diff_context)
 
                     diff_processor = diffs.DiffProcessor(
                         vcs_diff, format='newdiff', diff_limit=diff_limit,
                         file_limit=file_limit, show_full_diff=c.fulldiff)
 
                     _parsed = diff_processor.prepare()
 
                     diffset = codeblocks.DiffSet(
                         repo_name=self.db_repo_name,
                         source_node_getter=codeblocks.diffset_node_getter(commit1),
                         target_node_getter=codeblocks.diffset_node_getter(commit2))
 
                     diffset = self.path_filter.render_patchset_filtered(
                         diffset, _parsed, commit1.raw_id, commit2.raw_id)
 
                     # save cached diff
                     if caching_enabled:
                         cache_diff(cache_file_path, diffset, None)
 
                 c.limited_diff = diffset.limited_diff
                 c.changes[commit.raw_id] = diffset
             else:
                 # TODO(marcink): no cache usage here...
                 _diff = self.rhodecode_vcs_repo.get_diff(
                     commit1, commit2,
                     ignore_whitespace=hide_whitespace_changes, context=diff_context)
                 diff_processor = diffs.DiffProcessor(
                     _diff, format='newdiff', diff_limit=diff_limit,
                     file_limit=file_limit, show_full_diff=c.fulldiff)
                 # downloads/raw we only need RAW diff nothing else
                 diff = self.path_filter.get_raw_patch(diff_processor)
                 c.changes[commit.raw_id] = [None, None, None, None, diff, None, None]
 
         # sort comments by how they were generated
         c.comments = sorted(c.comments, key=lambda x: x.comment_id)
+        c.at_version_num = None
 
         if len(c.commit_ranges) == 1:
             c.commit = c.commit_ranges[0]
             c.parent_tmpl = ''.join(
                 '# Parent  %s\n' % x.raw_id for x in c.commit.parents)
 
         if method == 'download':
             response = Response(diff)
             response.content_type = 'text/plain'
             response.content_disposition = (
                 'attachment; filename=%s.diff' % commit_id_range[:12])
             return response
         elif method == 'patch':
             c.diff = safe_unicode(diff)
             patch = render(
                 'rhodecode:templates/changeset/patch_changeset.mako',
                 self._get_template_context(c), self.request)
             response = Response(patch)
             response.content_type = 'text/plain'
             return response
         elif method == 'raw':
             response = Response(diff)
             response.content_type = 'text/plain'
             return response
         elif method == 'show':
             if len(c.commit_ranges) == 1:
                 html = render(
                     'rhodecode:templates/changeset/changeset.mako',
                     self._get_template_context(c), self.request)
                 return Response(html)
             else:
                 c.ancestor = None
                 c.target_repo = self.db_repo
                 html = render(
                     'rhodecode:templates/changeset/changeset_range.mako',
                     self._get_template_context(c), self.request)
                 return Response(html)
 
         raise HTTPBadRequest()
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_commit', request_method='GET',
         renderer=None)
     def repo_commit_show(self):
         commit_id = self.request.matchdict['commit_id']
         return self._commit(commit_id, method='show')
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_commit_raw', request_method='GET',
         renderer=None)
     @view_config(
         route_name='repo_commit_raw_deprecated', request_method='GET',
         renderer=None)
     def repo_commit_raw(self):
         commit_id = self.request.matchdict['commit_id']
         return self._commit(commit_id, method='raw')
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_commit_patch', request_method='GET',
         renderer=None)
     def repo_commit_patch(self):
         commit_id = self.request.matchdict['commit_id']
         return self._commit(commit_id, method='patch')
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_commit_download', request_method='GET',
         renderer=None)
     def repo_commit_download(self):
         commit_id = self.request.matchdict['commit_id']
         return self._commit(commit_id, method='download')
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='repo_commit_comment_create', request_method='POST',
         renderer='json_ext')
     def repo_commit_comment_create(self):
         _ = self.request.translate
         commit_id = self.request.matchdict['commit_id']
 
         c = self.load_default_context()
         status = self.request.POST.get('changeset_status', None)
         text = self.request.POST.get('text')
         comment_type = self.request.POST.get('comment_type')
         resolves_comment_id = self.request.POST.get('resolves_comment_id', None)
 
         if status:
             text = text or (_('Status change %(transition_icon)s %(status)s')
                             % {'transition_icon': '>',
                                'status': ChangesetStatus.get_status_lbl(status)})
 
         multi_commit_ids = []
         for _commit_id in self.request.POST.get('commit_ids', '').split(','):
             if _commit_id not in ['', None, EmptyCommit.raw_id]:
                 if _commit_id not in multi_commit_ids:
                     multi_commit_ids.append(_commit_id)
 
         commit_ids = multi_commit_ids or [commit_id]
 
         comment = None
         for current_id in filter(None, commit_ids):
             comment = CommentsModel().create(
                 text=text,
                 repo=self.db_repo.repo_id,
                 user=self._rhodecode_db_user.user_id,
                 commit_id=current_id,
                 f_path=self.request.POST.get('f_path'),
                 line_no=self.request.POST.get('line'),
                 status_change=(ChangesetStatus.get_status_lbl(status)
                                if status else None),
                 status_change_type=status,
                 comment_type=comment_type,
                 resolves_comment_id=resolves_comment_id,
                 auth_user=self._rhodecode_user
             )
 
             # get status if set !
             if status:
                 # if latest status was from pull request and it's closed
                 # disallow changing status !
                 # dont_allow_on_closed_pull_request = True !
 
                 try:
                     ChangesetStatusModel().set_status(
                         self.db_repo.repo_id,
                         status,
                         self._rhodecode_db_user.user_id,
                         comment,
                         revision=current_id,
                         dont_allow_on_closed_pull_request=True
                     )
                 except StatusChangeOnClosedPullRequestError:
                     msg = _('Changing the status of a commit associated with '
                             'a closed pull request is not allowed')
                     log.exception(msg)
                     h.flash(msg, category='warning')
                     raise HTTPFound(h.route_path(
                         'repo_commit', repo_name=self.db_repo_name,
                         commit_id=current_id))
 
             commit = self.db_repo.get_commit(current_id)
             CommentsModel().trigger_commit_comment_hook(
                 self.db_repo, self._rhodecode_user, 'create',
                 data={'comment': comment, 'commit': commit})
 
         # finalize, commit and redirect
         Session().commit()
 
         data = {
             'target_id': h.safeid(h.safe_unicode(
                 self.request.POST.get('f_path'))),
         }
         if comment:
             c.co = comment
+            c.at_version_num = 0
             rendered_comment = render(
                 'rhodecode:templates/changeset/changeset_comment_block.mako',
                 self._get_template_context(c), self.request)
 
             data.update(comment.get_dict())
             data.update({'rendered_text': rendered_comment})
 
         return data
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='repo_commit_comment_preview', request_method='POST',
         renderer='string', xhr=True)
     def repo_commit_comment_preview(self):
         # Technically a CSRF token is not needed as no state changes with this
         # call. However, as this is a POST is better to have it, so automated
         # tools don't flag it as potential CSRF.
         # Post is required because the payload could be bigger than the maximum
         # allowed by GET.
 
         text = self.request.POST.get('text')
         renderer = self.request.POST.get('renderer') or 'rst'
         if text:
             return h.render(text, renderer=renderer, mentions=True,
                             repo_name=self.db_repo_name)
         return ''
 
     @LoginRequired()
-    @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='repo_commit_comment_history_view', request_method='POST',
         renderer='string', xhr=True)
     def repo_commit_comment_history_view(self):
         c = self.load_default_context()
 
         comment_history_id = self.request.matchdict['comment_history_id']
         comment_history = ChangesetCommentHistory.get_or_404(comment_history_id)
         is_repo_comment = comment_history.comment.repo.repo_id == self.db_repo.repo_id
 
         if is_repo_comment:
             c.comment_history = comment_history
 
             rendered_comment = render(
                 'rhodecode:templates/changeset/comment_history.mako',
                 self._get_template_context(c)
                 , self.request)
             return rendered_comment
         else:
             log.warning('No permissions for user %s to show comment_history_id: %s',
                         self._rhodecode_db_user, comment_history_id)
             raise HTTPNotFound()
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='repo_commit_comment_attachment_upload', request_method='POST',
         renderer='json_ext', xhr=True)
     def repo_commit_comment_attachment_upload(self):
         c = self.load_default_context()
         upload_key = 'attachment'
 
         file_obj = self.request.POST.get(upload_key)
 
         if file_obj is None:
             self.request.response.status = 400
             return {'store_fid': None,
                     'access_path': None,
                     'error': '{} data field is missing'.format(upload_key)}
 
         if not hasattr(file_obj, 'filename'):
             self.request.response.status = 400
             return {'store_fid': None,
                     'access_path': None,
                     'error': 'filename cannot be read from the data field'}
 
         filename = file_obj.filename
         file_display_name = filename
 
         metadata = {
             'user_uploaded': {'username': self._rhodecode_user.username,
                               'user_id': self._rhodecode_user.user_id,
                               'ip': self._rhodecode_user.ip_addr}}
 
         # TODO(marcink): allow .ini configuration for allowed_extensions, and file-size
         allowed_extensions = [
             'gif', '.jpeg', '.jpg', '.png', '.docx', '.gz', '.log', '.pdf',
             '.pptx', '.txt', '.xlsx', '.zip']
         max_file_size = 10 * 1024 * 1024  # 10MB, also validated via dropzone.js
 
         try:
             storage = store_utils.get_file_storage(self.request.registry.settings)
             store_uid, metadata = storage.save_file(
                 file_obj.file, filename, extra_metadata=metadata,
                 extensions=allowed_extensions, max_filesize=max_file_size)
         except FileNotAllowedException:
             self.request.response.status = 400
             permitted_extensions = ', '.join(allowed_extensions)
             error_msg = 'File `{}` is not allowed. ' \
                         'Only following extensions are permitted: {}'.format(
                             filename, permitted_extensions)
             return {'store_fid': None,
                     'access_path': None,
                     'error': error_msg}
         except FileOverSizeException:
             self.request.response.status = 400
             limit_mb = h.format_byte_size_binary(max_file_size)
             return {'store_fid': None,
                     'access_path': None,
                     'error': 'File {} is exceeding allowed limit of {}.'.format(
                         filename, limit_mb)}
 
         try:
             entry = FileStore.create(
                 file_uid=store_uid, filename=metadata["filename"],
                 file_hash=metadata["sha256"], file_size=metadata["size"],
                 file_display_name=file_display_name,
                 file_description=u'comment attachment `{}`'.format(safe_unicode(filename)),
                 hidden=True, check_acl=True, user_id=self._rhodecode_user.user_id,
                 scope_repo_id=self.db_repo.repo_id
             )
             Session().add(entry)
             Session().commit()
             log.debug('Stored upload in DB as %s', entry)
         except Exception:
             log.exception('Failed to store file %s', filename)
             self.request.response.status = 400
             return {'store_fid': None,
                     'access_path': None,
                     'error': 'File {} failed to store in DB.'.format(filename)}
 
         Session().commit()
 
         return {
             'store_fid': store_uid,
             'access_path': h.route_path(
                 'download_file', fid=store_uid),
             'fqn_access_path': h.route_url(
                 'download_file', fid=store_uid),
             'repo_access_path': h.route_path(
                 'repo_artifacts_get', repo_name=self.db_repo_name, uid=store_uid),
             'repo_fqn_access_path': h.route_url(
                 'repo_artifacts_get', repo_name=self.db_repo_name, uid=store_uid),
         }
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='repo_commit_comment_delete', request_method='POST',
         renderer='json_ext')
     def repo_commit_comment_delete(self):
         commit_id = self.request.matchdict['commit_id']
         comment_id = self.request.matchdict['comment_id']
 
         comment = ChangesetComment.get_or_404(comment_id)
         if not comment:
             log.debug('Comment with id:%s not found, skipping', comment_id)
             # comment already deleted in another call probably
             return True
 
         if comment.immutable:
             # don't allow deleting comments that are immutable
             raise HTTPForbidden()
 
         is_repo_admin = h.HasRepoPermissionAny('repository.admin')(self.db_repo_name)
         super_admin = h.HasPermissionAny('hg.admin')()
         comment_owner = (comment.author.user_id == self._rhodecode_db_user.user_id)
         is_repo_comment = comment.repo.repo_id == self.db_repo.repo_id
         comment_repo_admin = is_repo_admin and is_repo_comment
 
         if super_admin or comment_owner or comment_repo_admin:
             CommentsModel().delete(comment=comment, auth_user=self._rhodecode_user)
             Session().commit()
             return True
         else:
             log.warning('No permissions for user %s to delete comment_id: %s',
                         self._rhodecode_db_user, comment_id)
             raise HTTPNotFound()
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='repo_commit_comment_edit', request_method='POST',
         renderer='json_ext')
     def repo_commit_comment_edit(self):
         self.load_default_context()
 
         comment_id = self.request.matchdict['comment_id']
         comment = ChangesetComment.get_or_404(comment_id)
 
         if comment.immutable:
             # don't allow deleting comments that are immutable
             raise HTTPForbidden()
 
         is_repo_admin = h.HasRepoPermissionAny('repository.admin')(self.db_repo_name)
         super_admin = h.HasPermissionAny('hg.admin')()
         comment_owner = (comment.author.user_id == self._rhodecode_db_user.user_id)
         is_repo_comment = comment.repo.repo_id == self.db_repo.repo_id
         comment_repo_admin = is_repo_admin and is_repo_comment
 
         if super_admin or comment_owner or comment_repo_admin:
             text = self.request.POST.get('text')
             version = self.request.POST.get('version')
             if text == comment.text:
                 log.warning(
                     'Comment(repo): '
                     'Trying to create new version '
                     'with the same comment body {}'.format(
                         comment_id,
                     )
                 )
                 raise HTTPNotFound()
 
             if version.isdigit():
                 version = int(version)
             else:
                 log.warning(
                     'Comment(repo): Wrong version type {} {} '
                     'for comment {}'.format(
                         version,
                         type(version),
                         comment_id,
                     )
                 )
                 raise HTTPNotFound()
 
             try:
                 comment_history = CommentsModel().edit(
                     comment_id=comment_id,
                     text=text,
                     auth_user=self._rhodecode_user,
                     version=version,
                 )
             except CommentVersionMismatch:
                 raise HTTPConflict()
 
             if not comment_history:
                 raise HTTPNotFound()
 
             commit_id = self.request.matchdict['commit_id']
             commit = self.db_repo.get_commit(commit_id)
             CommentsModel().trigger_commit_comment_hook(
                 self.db_repo, self._rhodecode_user, 'edit',
                 data={'comment': comment, 'commit': commit})
 
             Session().commit()
             return {
                 'comment_history_id': comment_history.comment_history_id,
                 'comment_id': comment.comment_id,
                 'comment_version': comment_history.version,
                 'comment_author_username': comment_history.author.username,
                 'comment_author_gravatar': h.gravatar_url(comment_history.author.email, 16),
                 'comment_created_on': h.age_component(comment_history.created_on,
                                                       time_is_local=True),
             }
         else:
             log.warning('No permissions for user %s to edit comment_id: %s',
                         self._rhodecode_db_user, comment_id)
             raise HTTPNotFound()
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_commit_data', request_method='GET',
         renderer='json_ext', xhr=True)
     def repo_commit_data(self):
         commit_id = self.request.matchdict['commit_id']
         self.load_default_context()
 
         try:
             return self.rhodecode_vcs_repo.get_commit(commit_id=commit_id)
         except CommitDoesNotExistError as e:
             return EmptyCommit(message=str(e))
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_commit_children', request_method='GET',
         renderer='json_ext', xhr=True)
     def repo_commit_children(self):
         commit_id = self.request.matchdict['commit_id']
         self.load_default_context()
 
         try:
             commit = self.rhodecode_vcs_repo.get_commit(commit_id=commit_id)
             children = commit.children
         except CommitDoesNotExistError:
             children = []
 
         result = {"results": children}
         return result
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='repo_commit_parents', request_method='GET',
         renderer='json_ext')
     def repo_commit_parents(self):
         commit_id = self.request.matchdict['commit_id']
         self.load_default_context()
 
         try:
             commit = self.rhodecode_vcs_repo.get_commit(commit_id=commit_id)
             parents = commit.parents
         except CommitDoesNotExistError:
             parents = []
         result = {"results": parents}
         return result

rhodecode/apps/repository/views/repo_pull_requests.py

@@ -1,1637 +1,1757 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import collections
 
 import formencode
 import formencode.htmlfill
 import peppercorn
 from pyramid.httpexceptions import (
     HTTPFound, HTTPNotFound, HTTPForbidden, HTTPBadRequest, HTTPConflict)
 from pyramid.view import view_config
 from pyramid.renderers import render
 
 from rhodecode.apps._base import RepoAppView, DataGridAppView
 
 from rhodecode.lib import helpers as h, diffs, codeblocks, channelstream
 from rhodecode.lib.base import vcs_operation_context
 from rhodecode.lib.diffs import load_cached_diff, cache_diff, diff_cache_exist
 from rhodecode.lib.exceptions import CommentVersionMismatch
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAny, HasRepoPermissionAnyDecorator,
     NotAnonymous, CSRFRequired)
-from rhodecode.lib.utils2 import str2bool, safe_str, safe_unicode
+from rhodecode.lib.utils2 import str2bool, safe_str, safe_unicode, safe_int
 from rhodecode.lib.vcs.backends.base import EmptyCommit, UpdateFailureReason
 from rhodecode.lib.vcs.exceptions import (
     CommitDoesNotExistError, RepositoryRequirementError, EmptyRepositoryError)
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.db import (
     func, or_, PullRequest, ChangesetComment, ChangesetStatus, Repository)
 from rhodecode.model.forms import PullRequestForm
 from rhodecode.model.meta import Session
 from rhodecode.model.pull_request import PullRequestModel, MergeCheck
 from rhodecode.model.scm import ScmModel
 
 log = logging.getLogger(__name__)
 
 
 class RepoPullRequestsView(RepoAppView, DataGridAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context(include_app_defaults=True)
         c.REVIEW_STATUS_APPROVED = ChangesetStatus.STATUS_APPROVED
         c.REVIEW_STATUS_REJECTED = ChangesetStatus.STATUS_REJECTED
         # backward compat., we use for OLD PRs a plain renderer
         c.renderer = 'plain'
         return c
 
     def _get_pull_requests_list(
             self, repo_name, source, filter_type, opened_by, statuses):
 
         draw, start, limit = self._extract_chunk(self.request)
         search_q, order_by, order_dir = self._extract_ordering(self.request)
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/data_table/_dt_elements.mako')
 
         # pagination
 
         if filter_type == 'awaiting_review':
             pull_requests = PullRequestModel().get_awaiting_review(
                 repo_name, search_q=search_q, source=source, opened_by=opened_by,
                 statuses=statuses, offset=start, length=limit,
                 order_by=order_by, order_dir=order_dir)
             pull_requests_total_count = PullRequestModel().count_awaiting_review(
                 repo_name, search_q=search_q, source=source, statuses=statuses,
                 opened_by=opened_by)
         elif filter_type == 'awaiting_my_review':
             pull_requests = PullRequestModel().get_awaiting_my_review(
                 repo_name, search_q=search_q, source=source, opened_by=opened_by,
                 user_id=self._rhodecode_user.user_id, statuses=statuses,
                 offset=start, length=limit, order_by=order_by,
                 order_dir=order_dir)
             pull_requests_total_count = PullRequestModel().count_awaiting_my_review(
                 repo_name, search_q=search_q, source=source, user_id=self._rhodecode_user.user_id,
                 statuses=statuses, opened_by=opened_by)
         else:
             pull_requests = PullRequestModel().get_all(
                 repo_name, search_q=search_q, source=source, opened_by=opened_by,
                 statuses=statuses, offset=start, length=limit,
                 order_by=order_by, order_dir=order_dir)
             pull_requests_total_count = PullRequestModel().count_all(
                 repo_name, search_q=search_q, source=source, statuses=statuses,
                 opened_by=opened_by)
 
         data = []
         comments_model = CommentsModel()
         for pr in pull_requests:
             comments = comments_model.get_all_comments(
                 self.db_repo.repo_id, pull_request=pr)
 
             data.append({
                 'name': _render('pullrequest_name',
                                 pr.pull_request_id, pr.pull_request_state,
                                 pr.work_in_progress, pr.target_repo.repo_name),
                 'name_raw': pr.pull_request_id,
                 'status': _render('pullrequest_status',
                                   pr.calculated_review_status()),
                 'title': _render('pullrequest_title', pr.title, pr.description),
                 'description': h.escape(pr.description),
                 'updated_on': _render('pullrequest_updated_on',
                                       h.datetime_to_time(pr.updated_on)),
                 'updated_on_raw': h.datetime_to_time(pr.updated_on),
                 'created_on': _render('pullrequest_updated_on',
                                       h.datetime_to_time(pr.created_on)),
                 'created_on_raw': h.datetime_to_time(pr.created_on),
                 'state': pr.pull_request_state,
                 'author': _render('pullrequest_author',
                                   pr.author.full_contact, ),
                 'author_raw': pr.author.full_name,
                 'comments': _render('pullrequest_comments', len(comments)),
                 'comments_raw': len(comments),
                 'closed': pr.is_closed(),
             })
 
         data = ({
             'draw': draw,
             'data': data,
             'recordsTotal': pull_requests_total_count,
             'recordsFiltered': pull_requests_total_count,
         })
         return data
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='pullrequest_show_all', request_method='GET',
         renderer='rhodecode:templates/pullrequests/pullrequests.mako')
     def pull_request_list(self):
         c = self.load_default_context()
 
         req_get = self.request.GET
         c.source = str2bool(req_get.get('source'))
         c.closed = str2bool(req_get.get('closed'))
         c.my = str2bool(req_get.get('my'))
         c.awaiting_review = str2bool(req_get.get('awaiting_review'))
         c.awaiting_my_review = str2bool(req_get.get('awaiting_my_review'))
 
         c.active = 'open'
         if c.my:
             c.active = 'my'
         if c.closed:
             c.active = 'closed'
         if c.awaiting_review and not c.source:
             c.active = 'awaiting'
         if c.source and not c.awaiting_review:
             c.active = 'source'
         if c.awaiting_my_review:
             c.active = 'awaiting_my'
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='pullrequest_show_all_data', request_method='GET',
         renderer='json_ext', xhr=True)
     def pull_request_list_data(self):
         self.load_default_context()
 
         # additional filters
         req_get = self.request.GET
         source = str2bool(req_get.get('source'))
         closed = str2bool(req_get.get('closed'))
         my = str2bool(req_get.get('my'))
         awaiting_review = str2bool(req_get.get('awaiting_review'))
         awaiting_my_review = str2bool(req_get.get('awaiting_my_review'))
 
         filter_type = 'awaiting_review' if awaiting_review \
             else 'awaiting_my_review' if awaiting_my_review \
             else None
 
         opened_by = None
         if my:
             opened_by = [self._rhodecode_user.user_id]
 
         statuses = [PullRequest.STATUS_NEW, PullRequest.STATUS_OPEN]
         if closed:
             statuses = [PullRequest.STATUS_CLOSED]
 
         data = self._get_pull_requests_list(
             repo_name=self.db_repo_name, source=source,
             filter_type=filter_type, opened_by=opened_by, statuses=statuses)
 
         return data
 
     def _is_diff_cache_enabled(self, target_repo):
         caching_enabled = self._get_general_setting(
             target_repo, 'rhodecode_diff_cache')
         log.debug('Diff caching enabled: %s', caching_enabled)
         return caching_enabled
 
     def _get_diffset(self, source_repo_name, source_repo,
                      ancestor_commit,
                      source_ref_id, target_ref_id,
                      target_commit, source_commit, diff_limit, file_limit,
                      fulldiff, hide_whitespace_changes, diff_context, use_ancestor=True):
 
         if use_ancestor:
             # we might want to not use it for versions
             target_ref_id = ancestor_commit.raw_id
 
         vcs_diff = PullRequestModel().get_diff(
             source_repo, source_ref_id, target_ref_id,
             hide_whitespace_changes, diff_context)
 
         diff_processor = diffs.DiffProcessor(
             vcs_diff, format='newdiff', diff_limit=diff_limit,
             file_limit=file_limit, show_full_diff=fulldiff)
 
         _parsed = diff_processor.prepare()
 
         diffset = codeblocks.DiffSet(
             repo_name=self.db_repo_name,
             source_repo_name=source_repo_name,
             source_node_getter=codeblocks.diffset_node_getter(target_commit),
             target_node_getter=codeblocks.diffset_node_getter(source_commit),
         )
         diffset = self.path_filter.render_patchset_filtered(
             diffset, _parsed, target_commit.raw_id, source_commit.raw_id)
 
         return diffset
 
     def _get_range_diffset(self, source_scm, source_repo,
                            commit1, commit2, diff_limit, file_limit,
                            fulldiff, hide_whitespace_changes, diff_context):
         vcs_diff = source_scm.get_diff(
             commit1, commit2,
             ignore_whitespace=hide_whitespace_changes,
             context=diff_context)
 
         diff_processor = diffs.DiffProcessor(
             vcs_diff, format='newdiff', diff_limit=diff_limit,
             file_limit=file_limit, show_full_diff=fulldiff)
 
         _parsed = diff_processor.prepare()
 
         diffset = codeblocks.DiffSet(
             repo_name=source_repo.repo_name,
             source_node_getter=codeblocks.diffset_node_getter(commit1),
             target_node_getter=codeblocks.diffset_node_getter(commit2))
 
         diffset = self.path_filter.render_patchset_filtered(
             diffset, _parsed, commit1.raw_id, commit2.raw_id)
 
         return diffset
 
+    def register_comments_vars(self, c, pull_request, versions):
+        comments_model = CommentsModel()
+
+        # GENERAL COMMENTS with versions #
+        q = comments_model._all_general_comments_of_pull_request(pull_request)
+        q = q.order_by(ChangesetComment.comment_id.asc())
+        general_comments = q
+
+        # pick comments we want to render at current version
+        c.comment_versions = comments_model.aggregate_comments(
+            general_comments, versions, c.at_version_num)
+
+        # INLINE COMMENTS with versions  #
+        q = comments_model._all_inline_comments_of_pull_request(pull_request)
+        q = q.order_by(ChangesetComment.comment_id.asc())
+        inline_comments = q
+
+        c.inline_versions = comments_model.aggregate_comments(
+            inline_comments, versions, c.at_version_num, inline=True)
+
+        # Comments inline+general
+        if c.at_version:
+            c.inline_comments_flat = c.inline_versions[c.at_version_num]['display']
+            c.comments = c.comment_versions[c.at_version_num]['display']
+        else:
+            c.inline_comments_flat = c.inline_versions[c.at_version_num]['until']
+            c.comments = c.comment_versions[c.at_version_num]['until']
+
+        return general_comments, inline_comments
+
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='pullrequest_show', request_method='GET',
         renderer='rhodecode:templates/pullrequests/pullrequest_show.mako')
     def pull_request_show(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         pull_request_id = pull_request.pull_request_id
 
         c.state_progressing = pull_request.is_state_changing()
+        c.pr_broadcast_channel = '/repo${}$/pr/{}'.format(
+            pull_request.target_repo.repo_name, pull_request.pull_request_id)
 
         _new_state = {
             'created': PullRequest.STATE_CREATED,
         }.get(self.request.GET.get('force_state'))
 
         if c.is_super_admin and _new_state:
             with pull_request.set_state(PullRequest.STATE_UPDATING, final_state=_new_state):
                 h.flash(
                     _('Pull Request state was force changed to `{}`').format(_new_state),
                     category='success')
                 Session().commit()
 
             raise HTTPFound(h.route_path(
                 'pullrequest_show', repo_name=self.db_repo_name,
                 pull_request_id=pull_request_id))
 
         version = self.request.GET.get('version')
         from_version = self.request.GET.get('from_version') or version
         merge_checks = self.request.GET.get('merge_checks')
         c.fulldiff = str2bool(self.request.GET.get('fulldiff'))
+        force_refresh = str2bool(self.request.GET.get('force_refresh'))
+        c.range_diff_on = self.request.GET.get('range-diff') == "1"
 
         # fetch global flags of ignore ws or context lines
         diff_context = diffs.get_diff_context(self.request)
         hide_whitespace_changes = diffs.get_diff_whitespace_flag(self.request)
 
-        force_refresh = str2bool(self.request.GET.get('force_refresh'))
-
         (pull_request_latest,
          pull_request_at_ver,
          pull_request_display_obj,
          at_version) = PullRequestModel().get_pr_version(
             pull_request_id, version=version)
+
         pr_closed = pull_request_latest.is_closed()
 
         if pr_closed and (version or from_version):
-            # not allow to browse versions
+            # not allow to browse versions for closed PR
             raise HTTPFound(h.route_path(
                 'pullrequest_show', repo_name=self.db_repo_name,
                 pull_request_id=pull_request_id))
 
         versions = pull_request_display_obj.versions()
         # used to store per-commit range diffs
         c.changes = collections.OrderedDict()
-        c.range_diff_on = self.request.GET.get('range-diff') == "1"
 
         c.at_version = at_version
         c.at_version_num = (at_version
-                            if at_version and at_version != 'latest'
+                            if at_version and at_version != PullRequest.LATEST_VER
                             else None)
-        c.at_version_pos = ChangesetComment.get_index_from_version(
+
+        c.at_version_index = ChangesetComment.get_index_from_version(
             c.at_version_num, versions)
 
         (prev_pull_request_latest,
          prev_pull_request_at_ver,
          prev_pull_request_display_obj,
          prev_at_version) = PullRequestModel().get_pr_version(
             pull_request_id, version=from_version)
 
         c.from_version = prev_at_version
         c.from_version_num = (prev_at_version
-                              if prev_at_version and prev_at_version != 'latest'
+                              if prev_at_version and prev_at_version != PullRequest.LATEST_VER
                               else None)
-        c.from_version_pos = ChangesetComment.get_index_from_version(
+        c.from_version_index = ChangesetComment.get_index_from_version(
             c.from_version_num, versions)
 
         # define if we're in COMPARE mode or VIEW at version mode
         compare = at_version != prev_at_version
 
         # pull_requests repo_name we opened it against
         # ie. target_repo must match
         if self.db_repo_name != pull_request_at_ver.target_repo.repo_name:
+            log.warning('Mismatch between the current repo: %s, and target %s',
+                        self.db_repo_name, pull_request_at_ver.target_repo.repo_name)
             raise HTTPNotFound()
 
-        c.shadow_clone_url = PullRequestModel().get_shadow_clone_url(
-            pull_request_at_ver)
+        c.shadow_clone_url = PullRequestModel().get_shadow_clone_url(pull_request_at_ver)
 
         c.pull_request = pull_request_display_obj
         c.renderer = pull_request_at_ver.description_renderer or c.renderer
         c.pull_request_latest = pull_request_latest
 
-        if compare or (at_version and not at_version == 'latest'):
+        # inject latest version
+        latest_ver = PullRequest.get_pr_display_object(pull_request_latest, pull_request_latest)
+        c.versions = versions + [latest_ver]
+
+        if compare or (at_version and not at_version == PullRequest.LATEST_VER):
             c.allowed_to_change_status = False
             c.allowed_to_update = False
             c.allowed_to_merge = False
             c.allowed_to_delete = False
             c.allowed_to_comment = False
             c.allowed_to_close = False
         else:
             can_change_status = PullRequestModel().check_user_change_status(
                 pull_request_at_ver, self._rhodecode_user)
             c.allowed_to_change_status = can_change_status and not pr_closed
 
             c.allowed_to_update = PullRequestModel().check_user_update(
                 pull_request_latest, self._rhodecode_user) and not pr_closed
             c.allowed_to_merge = PullRequestModel().check_user_merge(
                 pull_request_latest, self._rhodecode_user) and not pr_closed
             c.allowed_to_delete = PullRequestModel().check_user_delete(
                 pull_request_latest, self._rhodecode_user) and not pr_closed
             c.allowed_to_comment = not pr_closed
             c.allowed_to_close = c.allowed_to_merge and not pr_closed
 
         c.forbid_adding_reviewers = False
         c.forbid_author_to_review = False
         c.forbid_commit_author_to_review = False
 
         if pull_request_latest.reviewer_data and \
                         'rules' in pull_request_latest.reviewer_data:
             rules = pull_request_latest.reviewer_data['rules'] or {}
             try:
-                c.forbid_adding_reviewers = rules.get(
-                    'forbid_adding_reviewers')
-                c.forbid_author_to_review = rules.get(
-                    'forbid_author_to_review')
-                c.forbid_commit_author_to_review = rules.get(
-                    'forbid_commit_author_to_review')
+                c.forbid_adding_reviewers = rules.get('forbid_adding_reviewers')
+                c.forbid_author_to_review = rules.get('forbid_author_to_review')
+                c.forbid_commit_author_to_review = rules.get('forbid_commit_author_to_review')
             except Exception:
                 pass
 
         # check merge capabilities
         _merge_check = MergeCheck.validate(
             pull_request_latest, auth_user=self._rhodecode_user,
             translator=self.request.translate,
             force_shadow_repo_refresh=force_refresh)
 
         c.pr_merge_errors = _merge_check.error_details
         c.pr_merge_possible = not _merge_check.failed
         c.pr_merge_message = _merge_check.merge_msg
         c.pr_merge_source_commit = _merge_check.source_commit
         c.pr_merge_target_commit = _merge_check.target_commit
 
         c.pr_merge_info = MergeCheck.get_merge_conditions(
             pull_request_latest, translator=self.request.translate)
 
         c.pull_request_review_status = _merge_check.review_status
         if merge_checks:
             self.request.override_renderer = \
                 'rhodecode:templates/pullrequests/pullrequest_merge_checks.mako'
             return self._get_template_context(c)
 
-        comments_model = CommentsModel()
+        c.allowed_reviewers = [obj.user_id for obj in pull_request.reviewers if obj.user]
 
         # reviewers and statuses
-        c.pull_request_reviewers = pull_request_at_ver.reviewers_statuses()
-        allowed_reviewers = [x[0].user_id for x in c.pull_request_reviewers]
+        c.pull_request_default_reviewers_data_json = json.dumps(pull_request.reviewer_data)
+        c.pull_request_set_reviewers_data_json = collections.OrderedDict({'reviewers': []})
 
-        # GENERAL COMMENTS with versions #
-        q = comments_model._all_general_comments_of_pull_request(pull_request_latest)
-        q = q.order_by(ChangesetComment.comment_id.asc())
-        general_comments = q
+        for review_obj, member, reasons, mandatory, status in pull_request_at_ver.reviewers_statuses():
+            member_reviewer = h.reviewer_as_json(
+                member, reasons=reasons, mandatory=mandatory,
+                user_group=review_obj.rule_user_group_data()
+            )
 
-        # pick comments we want to render at current version
-        c.comment_versions = comments_model.aggregate_comments(
-            general_comments, versions, c.at_version_num)
-        c.comments = c.comment_versions[c.at_version_num]['until']
+            current_review_status = status[0][1].status if status else ChangesetStatus.STATUS_NOT_REVIEWED
+            member_reviewer['review_status'] = current_review_status
+            member_reviewer['review_status_label'] = h.commit_status_lbl(current_review_status)
+            member_reviewer['allowed_to_update'] = c.allowed_to_update
+            c.pull_request_set_reviewers_data_json['reviewers'].append(member_reviewer)
 
-        # INLINE COMMENTS with versions  #
-        q = comments_model._all_inline_comments_of_pull_request(pull_request_latest)
-        q = q.order_by(ChangesetComment.comment_id.asc())
-        inline_comments = q
+        c.pull_request_set_reviewers_data_json = json.dumps(c.pull_request_set_reviewers_data_json)
 
-        c.inline_versions = comments_model.aggregate_comments(
-            inline_comments, versions, c.at_version_num, inline=True)
+        general_comments, inline_comments = \
+            self.register_comments_vars(c, pull_request_latest, versions)
 
         # TODOs
         c.unresolved_comments = CommentsModel() \
-            .get_pull_request_unresolved_todos(pull_request)
+            .get_pull_request_unresolved_todos(pull_request_latest)
         c.resolved_comments = CommentsModel() \
-            .get_pull_request_resolved_todos(pull_request)
-
-        # inject latest version
-        latest_ver = PullRequest.get_pr_display_object(
-            pull_request_latest, pull_request_latest)
-
-        c.versions = versions + [latest_ver]
+            .get_pull_request_resolved_todos(pull_request_latest)
 
         # if we use version, then do not show later comments
         # than current version
         display_inline_comments = collections.defaultdict(
             lambda: collections.defaultdict(list))
         for co in inline_comments:
             if c.at_version_num:
                 # pick comments that are at least UPTO given version, so we
                 # don't render comments for higher version
                 should_render = co.pull_request_version_id and \
                                 co.pull_request_version_id <= c.at_version_num
             else:
                 # showing all, for 'latest'
                 should_render = True
 
             if should_render:
                 display_inline_comments[co.f_path][co.line_no].append(co)
 
         # load diff data into template context, if we use compare mode then
         # diff is calculated based on changes between versions of PR
 
         source_repo = pull_request_at_ver.source_repo
         source_ref_id = pull_request_at_ver.source_ref_parts.commit_id
 
         target_repo = pull_request_at_ver.target_repo
         target_ref_id = pull_request_at_ver.target_ref_parts.commit_id
 
         if compare:
             # in compare switch the diff base to latest commit from prev version
             target_ref_id = prev_pull_request_display_obj.revisions[0]
 
         # despite opening commits for bookmarks/branches/tags, we always
         # convert this to rev to prevent changes after bookmark or branch change
         c.source_ref_type = 'rev'
         c.source_ref = source_ref_id
 
         c.target_ref_type = 'rev'
         c.target_ref = target_ref_id
 
         c.source_repo = source_repo
         c.target_repo = target_repo
 
         c.commit_ranges = []
         source_commit = EmptyCommit()
         target_commit = EmptyCommit()
         c.missing_requirements = False
 
         source_scm = source_repo.scm_instance()
         target_scm = target_repo.scm_instance()
 
         shadow_scm = None
         try:
             shadow_scm = pull_request_latest.get_shadow_repo()
         except Exception:
             log.debug('Failed to get shadow repo', exc_info=True)
         # try first the existing source_repo, and then shadow
         # repo if we can obtain one
         commits_source_repo = source_scm
         if shadow_scm:
             commits_source_repo = shadow_scm
 
         c.commits_source_repo = commits_source_repo
         c.ancestor = None  # set it to None, to hide it from PR view
 
         # empty version means latest, so we keep this to prevent
         # double caching
-        version_normalized = version or 'latest'
-        from_version_normalized = from_version or 'latest'
+        version_normalized = version or PullRequest.LATEST_VER
+        from_version_normalized = from_version or PullRequest.LATEST_VER
 
         cache_path = self.rhodecode_vcs_repo.get_create_shadow_cache_pr_path(target_repo)
         cache_file_path = diff_cache_exist(
             cache_path, 'pull_request', pull_request_id, version_normalized,
             from_version_normalized, source_ref_id, target_ref_id,
             hide_whitespace_changes, diff_context, c.fulldiff)
 
         caching_enabled = self._is_diff_cache_enabled(c.target_repo)
         force_recache = self.get_recache_flag()
 
         cached_diff = None
         if caching_enabled:
             cached_diff = load_cached_diff(cache_file_path)
 
         has_proper_commit_cache = (
                 cached_diff and cached_diff.get('commits')
                 and len(cached_diff.get('commits', [])) == 5
                 and cached_diff.get('commits')[0]
                 and cached_diff.get('commits')[3])
 
         if not force_recache and not c.range_diff_on and has_proper_commit_cache:
             diff_commit_cache = \
                 (ancestor_commit, commit_cache, missing_requirements,
                  source_commit, target_commit) = cached_diff['commits']
         else:
             # NOTE(marcink): we reach potentially unreachable errors when a PR has
             # merge errors resulting in potentially hidden commits in the shadow repo.
             maybe_unreachable = _merge_check.MERGE_CHECK in _merge_check.error_details \
                                 and _merge_check.merge_response
             maybe_unreachable = maybe_unreachable \
                                 and _merge_check.merge_response.metadata.get('unresolved_files')
             log.debug("Using unreachable commits due to MERGE_CHECK in merge simulation")
             diff_commit_cache = \
                 (ancestor_commit, commit_cache, missing_requirements,
                  source_commit, target_commit) = self.get_commits(
                     commits_source_repo,
                     pull_request_at_ver,
                     source_commit,
                     source_ref_id,
                     source_scm,
                     target_commit,
                     target_ref_id,
                     target_scm,
                 maybe_unreachable=maybe_unreachable)
 
         # register our commit range
         for comm in commit_cache.values():
             c.commit_ranges.append(comm)
 
         c.missing_requirements = missing_requirements
         c.ancestor_commit = ancestor_commit
         c.statuses = source_repo.statuses(
             [x.raw_id for x in c.commit_ranges])
 
         # auto collapse if we have more than limit
         collapse_limit = diffs.DiffProcessor._collapse_commits_over
         c.collapse_all_commits = len(c.commit_ranges) > collapse_limit
         c.compare_mode = compare
 
         # diff_limit is the old behavior, will cut off the whole diff
         # if the limit is applied  otherwise will just hide the
         # big files from the front-end
         diff_limit = c.visual.cut_off_limit_diff
         file_limit = c.visual.cut_off_limit_file
 
         c.missing_commits = False
         if (c.missing_requirements
             or isinstance(source_commit, EmptyCommit)
             or source_commit == target_commit):
 
             c.missing_commits = True
         else:
             c.inline_comments = display_inline_comments
 
             use_ancestor = True
             if from_version_normalized != version_normalized:
                 use_ancestor = False
 
             has_proper_diff_cache = cached_diff and cached_diff.get('commits')
             if not force_recache and has_proper_diff_cache:
                 c.diffset = cached_diff['diff']
             else:
                 try:
                     c.diffset = self._get_diffset(
                         c.source_repo.repo_name, commits_source_repo,
                         c.ancestor_commit,
                         source_ref_id, target_ref_id,
                         target_commit, source_commit,
                         diff_limit, file_limit, c.fulldiff,
                         hide_whitespace_changes, diff_context,
                         use_ancestor=use_ancestor
-					)
+                    )
 
                     # save cached diff
                     if caching_enabled:
                         cache_diff(cache_file_path, c.diffset, diff_commit_cache)
                 except CommitDoesNotExistError:
                     log.exception('Failed to generate diffset')
                     c.missing_commits = True
 
             if not c.missing_commits:
 
                 c.limited_diff = c.diffset.limited_diff
 
                 # calculate removed files that are bound to comments
                 comment_deleted_files = [
                     fname for fname in display_inline_comments
                     if fname not in c.diffset.file_stats]
 
                 c.deleted_files_comments = collections.defaultdict(dict)
                 for fname, per_line_comments in display_inline_comments.items():
                     if fname in comment_deleted_files:
                         c.deleted_files_comments[fname]['stats'] = 0
                         c.deleted_files_comments[fname]['comments'] = list()
                         for lno, comments in per_line_comments.items():
                             c.deleted_files_comments[fname]['comments'].extend(comments)
 
                 # maybe calculate the range diff
                 if c.range_diff_on:
                     # TODO(marcink): set whitespace/context
                     context_lcl = 3
                     ign_whitespace_lcl = False
 
                     for commit in c.commit_ranges:
                         commit2 = commit
                         commit1 = commit.first_parent
 
                         range_diff_cache_file_path = diff_cache_exist(
                             cache_path, 'diff', commit.raw_id,
                             ign_whitespace_lcl, context_lcl, c.fulldiff)
 
                         cached_diff = None
                         if caching_enabled:
                             cached_diff = load_cached_diff(range_diff_cache_file_path)
 
                         has_proper_diff_cache = cached_diff and cached_diff.get('diff')
                         if not force_recache and has_proper_diff_cache:
                             diffset = cached_diff['diff']
                         else:
                             diffset = self._get_range_diffset(
                                 commits_source_repo, source_repo,
                                 commit1, commit2, diff_limit, file_limit,
                                 c.fulldiff, ign_whitespace_lcl, context_lcl
                             )
 
                         # save cached diff
                         if caching_enabled:
                             cache_diff(range_diff_cache_file_path, diffset, None)
 
                         c.changes[commit.raw_id] = diffset
 
         # this is a hack to properly display links, when creating PR, the
         # compare view and others uses different notation, and
         # compare_commits.mako renders links based on the target_repo.
         # We need to swap that here to generate it properly on the html side
         c.target_repo = c.source_repo
 
         c.commit_statuses = ChangesetStatus.STATUSES
 
         c.show_version_changes = not pr_closed
         if c.show_version_changes:
             cur_obj = pull_request_at_ver
             prev_obj = prev_pull_request_at_ver
 
             old_commit_ids = prev_obj.revisions
             new_commit_ids = cur_obj.revisions
             commit_changes = PullRequestModel()._calculate_commit_id_changes(
                 old_commit_ids, new_commit_ids)
             c.commit_changes_summary = commit_changes
 
             # calculate the diff for commits between versions
             c.commit_changes = []
 
             def mark(cs, fw):
                 return list(h.itertools.izip_longest([], cs, fillvalue=fw))
 
             for c_type, raw_id in mark(commit_changes.added, 'a') \
                                 + mark(commit_changes.removed, 'r') \
                                 + mark(commit_changes.common, 'c'):
 
                 if raw_id in commit_cache:
                     commit = commit_cache[raw_id]
                 else:
                     try:
                         commit = commits_source_repo.get_commit(raw_id)
                     except CommitDoesNotExistError:
                         # in case we fail extracting still use "dummy" commit
                         # for display in commit diff
                         commit = h.AttributeDict(
                             {'raw_id': raw_id,
                              'message': 'EMPTY or MISSING COMMIT'})
                 c.commit_changes.append([c_type, commit])
 
             # current user review statuses for each version
             c.review_versions = {}
-            if self._rhodecode_user.user_id in allowed_reviewers:
+            if self._rhodecode_user.user_id in c.allowed_reviewers:
                 for co in general_comments:
                     if co.author.user_id == self._rhodecode_user.user_id:
                         status = co.status_change
                         if status:
                             _ver_pr = status[0].comment.pull_request_version_id
                             c.review_versions[_ver_pr] = status[0]
 
         return self._get_template_context(c)
 
     def get_commits(
             self, commits_source_repo, pull_request_at_ver, source_commit,
             source_ref_id, source_scm, target_commit, target_ref_id, target_scm,
             maybe_unreachable=False):
 
         commit_cache = collections.OrderedDict()
         missing_requirements = False
 
         try:
             pre_load = ["author", "date", "message", "branch", "parents"]
 
             pull_request_commits = pull_request_at_ver.revisions
             log.debug('Loading %s commits from %s',
                       len(pull_request_commits), commits_source_repo)
 
             for rev in pull_request_commits:
                 comm = commits_source_repo.get_commit(commit_id=rev, pre_load=pre_load,
                                                       maybe_unreachable=maybe_unreachable)
                 commit_cache[comm.raw_id] = comm
 
             # Order here matters, we first need to get target, and then
             # the source
             target_commit = commits_source_repo.get_commit(
                 commit_id=safe_str(target_ref_id))
 
             source_commit = commits_source_repo.get_commit(
                 commit_id=safe_str(source_ref_id), maybe_unreachable=True)
         except CommitDoesNotExistError:
             log.warning('Failed to get commit from `{}` repo'.format(
                 commits_source_repo), exc_info=True)
         except RepositoryRequirementError:
             log.warning('Failed to get all required data from repo', exc_info=True)
             missing_requirements = True
 
         pr_ancestor_id = pull_request_at_ver.common_ancestor_id
 
         try:
             ancestor_commit = source_scm.get_commit(pr_ancestor_id)
         except Exception:
             ancestor_commit = None
 
         return ancestor_commit, commit_cache, missing_requirements, source_commit, target_commit
 
     def assure_not_empty_repo(self):
         _ = self.request.translate
 
         try:
             self.db_repo.scm_instance().get_commit()
         except EmptyRepositoryError:
             h.flash(h.literal(_('There are no commits yet')),
                     category='warning')
             raise HTTPFound(
                 h.route_path('repo_summary', repo_name=self.db_repo.repo_name))
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='pullrequest_new', request_method='GET',
         renderer='rhodecode:templates/pullrequests/pullrequest.mako')
     def pull_request_new(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         self.assure_not_empty_repo()
         source_repo = self.db_repo
 
         commit_id = self.request.GET.get('commit')
         branch_ref = self.request.GET.get('branch')
         bookmark_ref = self.request.GET.get('bookmark')
 
         try:
             source_repo_data = PullRequestModel().generate_repo_data(
                 source_repo, commit_id=commit_id,
                 branch=branch_ref, bookmark=bookmark_ref,
                 translator=self.request.translate)
         except CommitDoesNotExistError as e:
             log.exception(e)
             h.flash(_('Commit does not exist'), 'error')
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=source_repo.repo_name))
 
         default_target_repo = source_repo
 
         if source_repo.parent and c.has_origin_repo_read_perm:
             parent_vcs_obj = source_repo.parent.scm_instance()
             if parent_vcs_obj and not parent_vcs_obj.is_empty():
                 # change default if we have a parent repo
                 default_target_repo = source_repo.parent
 
         target_repo_data = PullRequestModel().generate_repo_data(
             default_target_repo, translator=self.request.translate)
 
         selected_source_ref = source_repo_data['refs']['selected_ref']
         title_source_ref = ''
         if selected_source_ref:
             title_source_ref = selected_source_ref.split(':', 2)[1]
         c.default_title = PullRequestModel().generate_pullrequest_title(
             source=source_repo.repo_name,
             source_ref=title_source_ref,
             target=default_target_repo.repo_name
         )
 
         c.default_repo_data = {
             'source_repo_name': source_repo.repo_name,
             'source_refs_json': json.dumps(source_repo_data),
             'target_repo_name': default_target_repo.repo_name,
             'target_refs_json': json.dumps(target_repo_data),
         }
         c.default_source_ref = selected_source_ref
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='pullrequest_repo_refs', request_method='GET',
         renderer='json_ext', xhr=True)
     def pull_request_repo_refs(self):
         self.load_default_context()
         target_repo_name = self.request.matchdict['target_repo_name']
         repo = Repository.get_by_repo_name(target_repo_name)
         if not repo:
             raise HTTPNotFound()
 
         target_perm = HasRepoPermissionAny(
             'repository.read', 'repository.write', 'repository.admin')(
             target_repo_name)
         if not target_perm:
             raise HTTPNotFound()
 
         return PullRequestModel().generate_repo_data(
             repo, translator=self.request.translate)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='pullrequest_repo_targets', request_method='GET',
         renderer='json_ext', xhr=True)
     def pullrequest_repo_targets(self):
         _ = self.request.translate
         filter_query = self.request.GET.get('query')
 
         # get the parents
         parent_target_repos = []
         if self.db_repo.parent:
             parents_query = Repository.query() \
                 .order_by(func.length(Repository.repo_name)) \
                 .filter(Repository.fork_id == self.db_repo.parent.repo_id)
 
             if filter_query:
                 ilike_expression = u'%{}%'.format(safe_unicode(filter_query))
                 parents_query = parents_query.filter(
                     Repository.repo_name.ilike(ilike_expression))
             parents = parents_query.limit(20).all()
 
             for parent in parents:
                 parent_vcs_obj = parent.scm_instance()
                 if parent_vcs_obj and not parent_vcs_obj.is_empty():
                     parent_target_repos.append(parent)
 
         # get other forks, and repo itself
         query = Repository.query() \
             .order_by(func.length(Repository.repo_name)) \
             .filter(
                 or_(Repository.repo_id == self.db_repo.repo_id,  # repo itself
                     Repository.fork_id == self.db_repo.repo_id)  # forks of this repo
             )  \
             .filter(~Repository.repo_id.in_([x.repo_id for x in parent_target_repos]))
 
         if filter_query:
             ilike_expression = u'%{}%'.format(safe_unicode(filter_query))
             query = query.filter(Repository.repo_name.ilike(ilike_expression))
 
         limit = max(20 - len(parent_target_repos), 5)  # not less then 5
         target_repos = query.limit(limit).all()
 
         all_target_repos = target_repos + parent_target_repos
 
         repos = []
         # This checks permissions to the repositories
         for obj in ScmModel().get_repos(all_target_repos):
             repos.append({
                 'id': obj['name'],
                 'text': obj['name'],
                 'type': 'repo',
                 'repo_id': obj['dbrepo']['repo_id'],
                 'repo_type': obj['dbrepo']['repo_type'],
                 'private': obj['dbrepo']['private'],
 
             })
 
         data = {
             'more': False,
             'results': [{
                 'text': _('Repositories'),
                 'children': repos
             }] if repos else []
         }
         return data
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
+    @view_config(
+        route_name='pullrequest_comments', request_method='POST',
+        renderer='string', xhr=True)
+    def pullrequest_comments(self):
+        self.load_default_context()
+
+        pull_request = PullRequest.get_or_404(
+            self.request.matchdict['pull_request_id'])
+        pull_request_id = pull_request.pull_request_id
+        version = self.request.GET.get('version')
+
+        _render = self.request.get_partial_renderer(
+            'rhodecode:templates/base/sidebar.mako')
+        c = _render.get_call_context()
+
+        (pull_request_latest,
+         pull_request_at_ver,
+         pull_request_display_obj,
+         at_version) = PullRequestModel().get_pr_version(
+            pull_request_id, version=version)
+        versions = pull_request_display_obj.versions()
+        latest_ver = PullRequest.get_pr_display_object(pull_request_latest, pull_request_latest)
+        c.versions = versions + [latest_ver]
+
+        c.at_version = at_version
+        c.at_version_num = (at_version
+                            if at_version and at_version != PullRequest.LATEST_VER
+                            else None)
+
+        self.register_comments_vars(c, pull_request_latest, versions)
+        all_comments = c.inline_comments_flat + c.comments
+
+        existing_ids = filter(
+            lambda e: e, map(safe_int, self.request.POST.getall('comments[]')))
+        return _render('comments_table', all_comments, len(all_comments),
+                       existing_ids=existing_ids)
+
+    @LoginRequired()
+    @NotAnonymous()
+    @HasRepoPermissionAnyDecorator(
+        'repository.read', 'repository.write', 'repository.admin')
+    @view_config(
+        route_name='pullrequest_todos', request_method='POST',
+        renderer='string', xhr=True)
+    def pullrequest_todos(self):
+        self.load_default_context()
+
+        pull_request = PullRequest.get_or_404(
+            self.request.matchdict['pull_request_id'])
+        pull_request_id = pull_request.pull_request_id
+        version = self.request.GET.get('version')
+
+        _render = self.request.get_partial_renderer(
+            'rhodecode:templates/base/sidebar.mako')
+        c = _render.get_call_context()
+        (pull_request_latest,
+         pull_request_at_ver,
+         pull_request_display_obj,
+         at_version) = PullRequestModel().get_pr_version(
+            pull_request_id, version=version)
+        versions = pull_request_display_obj.versions()
+        latest_ver = PullRequest.get_pr_display_object(pull_request_latest, pull_request_latest)
+        c.versions = versions + [latest_ver]
+
+        c.at_version = at_version
+        c.at_version_num = (at_version
+                            if at_version and at_version != PullRequest.LATEST_VER
+                            else None)
+
+        c.unresolved_comments = CommentsModel() \
+            .get_pull_request_unresolved_todos(pull_request)
+        c.resolved_comments = CommentsModel() \
+            .get_pull_request_resolved_todos(pull_request)
+
+        all_comments = c.unresolved_comments + c.resolved_comments
+        existing_ids = filter(
+            lambda e: e, map(safe_int, self.request.POST.getall('comments[]')))
+        return _render('comments_table', all_comments, len(c.unresolved_comments),
+                       todo_comments=True, existing_ids=existing_ids)
+
+    @LoginRequired()
+    @NotAnonymous()
+    @HasRepoPermissionAnyDecorator(
+        'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='pullrequest_create', request_method='POST',
         renderer=None)
     def pull_request_create(self):
         _ = self.request.translate
         self.assure_not_empty_repo()
         self.load_default_context()
 
         controls = peppercorn.parse(self.request.POST.items())
 
         try:
             form = PullRequestForm(
                 self.request.translate, self.db_repo.repo_id)()
             _form = form.to_python(controls)
         except formencode.Invalid as errors:
             if errors.error_dict.get('revisions'):
                 msg = 'Revisions: %s' % errors.error_dict['revisions']
             elif errors.error_dict.get('pullrequest_title'):
                 msg = errors.error_dict.get('pullrequest_title')
             else:
                 msg = _('Error creating pull request: {}').format(errors)
             log.exception(msg)
             h.flash(msg, 'error')
 
             # would rather just go back to form ...
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name))
 
         source_repo = _form['source_repo']
         source_ref = _form['source_ref']
         target_repo = _form['target_repo']
         target_ref = _form['target_ref']
         commit_ids = _form['revisions'][::-1]
         common_ancestor_id = _form['common_ancestor']
 
         # find the ancestor for this pr
         source_db_repo = Repository.get_by_repo_name(_form['source_repo'])
         target_db_repo = Repository.get_by_repo_name(_form['target_repo'])
 
         if not (source_db_repo or target_db_repo):
             h.flash(_('source_repo or target repo not found'), category='error')
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name))
 
         # re-check permissions again here
         # source_repo we must have read permissions
 
         source_perm = HasRepoPermissionAny(
             'repository.read', 'repository.write', 'repository.admin')(
             source_db_repo.repo_name)
         if not source_perm:
             msg = _('Not Enough permissions to source repo `{}`.'.format(
                 source_db_repo.repo_name))
             h.flash(msg, category='error')
             # copy the args back to redirect
             org_query = self.request.GET.mixed()
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name,
                              _query=org_query))
 
         # target repo we must have read permissions, and also later on
         # we want to check branch permissions here
         target_perm = HasRepoPermissionAny(
             'repository.read', 'repository.write', 'repository.admin')(
             target_db_repo.repo_name)
         if not target_perm:
             msg = _('Not Enough permissions to target repo `{}`.'.format(
                 target_db_repo.repo_name))
             h.flash(msg, category='error')
             # copy the args back to redirect
             org_query = self.request.GET.mixed()
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name,
                              _query=org_query))
 
         source_scm = source_db_repo.scm_instance()
         target_scm = target_db_repo.scm_instance()
 
         source_commit = source_scm.get_commit(source_ref.split(':')[-1])
         target_commit = target_scm.get_commit(target_ref.split(':')[-1])
 
         ancestor = source_scm.get_common_ancestor(
             source_commit.raw_id, target_commit.raw_id, target_scm)
 
         # recalculate target ref based on ancestor
         target_ref_type, target_ref_name, __ = _form['target_ref'].split(':')
         target_ref = ':'.join((target_ref_type, target_ref_name, ancestor))
 
         get_default_reviewers_data, validate_default_reviewers = \
             PullRequestModel().get_reviewer_functions()
 
         # recalculate reviewers logic, to make sure we can validate this
         reviewer_rules = get_default_reviewers_data(
             self._rhodecode_db_user, source_db_repo,
             source_commit, target_db_repo, target_commit)
 
         given_reviewers = _form['review_members']
         reviewers = validate_default_reviewers(
             given_reviewers, reviewer_rules)
 
         pullrequest_title = _form['pullrequest_title']
         title_source_ref = source_ref.split(':', 2)[1]
         if not pullrequest_title:
             pullrequest_title = PullRequestModel().generate_pullrequest_title(
                 source=source_repo,
                 source_ref=title_source_ref,
                 target=target_repo
             )
 
         description = _form['pullrequest_desc']
         description_renderer = _form['description_renderer']
 
         try:
             pull_request = PullRequestModel().create(
                 created_by=self._rhodecode_user.user_id,
                 source_repo=source_repo,
                 source_ref=source_ref,
                 target_repo=target_repo,
                 target_ref=target_ref,
                 revisions=commit_ids,
                 common_ancestor_id=common_ancestor_id,
                 reviewers=reviewers,
                 title=pullrequest_title,
                 description=description,
                 description_renderer=description_renderer,
                 reviewer_data=reviewer_rules,
                 auth_user=self._rhodecode_user
             )
             Session().commit()
 
             h.flash(_('Successfully opened new pull request'),
                     category='success')
         except Exception:
             msg = _('Error occurred during creation of this pull request.')
             log.exception(msg)
             h.flash(msg, category='error')
 
             # copy the args back to redirect
             org_query = self.request.GET.mixed()
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name,
                              _query=org_query))
 
         raise HTTPFound(
             h.route_path('pullrequest_show', repo_name=target_repo,
                          pull_request_id=pull_request.pull_request_id))
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='pullrequest_update', request_method='POST',
         renderer='json_ext')
     def pull_request_update(self):
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         _ = self.request.translate
 
-        self.load_default_context()
+        c = self.load_default_context()
         redirect_url = None
 
         if pull_request.is_closed():
             log.debug('update: forbidden because pull request is closed')
             msg = _(u'Cannot update closed pull requests.')
             h.flash(msg, category='error')
             return {'response': True,
                     'redirect_url': redirect_url}
 
         is_state_changing = pull_request.is_state_changing()
+        c.pr_broadcast_channel = '/repo${}$/pr/{}'.format(
+            pull_request.target_repo.repo_name, pull_request.pull_request_id)
 
         # only owner or admin can update it
         allowed_to_update = PullRequestModel().check_user_update(
             pull_request, self._rhodecode_user)
         if allowed_to_update:
             controls = peppercorn.parse(self.request.POST.items())
             force_refresh = str2bool(self.request.POST.get('force_refresh'))
 
             if 'review_members' in controls:
                 self._update_reviewers(
                     pull_request, controls['review_members'],
                     pull_request.reviewer_data)
             elif str2bool(self.request.POST.get('update_commits', 'false')):
                 if is_state_changing:
                     log.debug('commits update: forbidden because pull request is in state %s',
                               pull_request.pull_request_state)
                     msg = _(u'Cannot update pull requests commits in state other than `{}`. '
                             u'Current state is: `{}`').format(
                         PullRequest.STATE_CREATED, pull_request.pull_request_state)
                     h.flash(msg, category='error')
                     return {'response': True,
                             'redirect_url': redirect_url}
 
-                self._update_commits(pull_request)
+                self._update_commits(c, pull_request)
                 if force_refresh:
                     redirect_url = h.route_path(
                         'pullrequest_show', repo_name=self.db_repo_name,
                         pull_request_id=pull_request.pull_request_id,
                         _query={"force_refresh": 1})
             elif str2bool(self.request.POST.get('edit_pull_request', 'false')):
                 self._edit_pull_request(pull_request)
             else:
                 raise HTTPBadRequest()
 
             return {'response': True,
                     'redirect_url': redirect_url}
         raise HTTPForbidden()
 
     def _edit_pull_request(self, pull_request):
         _ = self.request.translate
 
         try:
             PullRequestModel().edit(
                 pull_request,
                 self.request.POST.get('title'),
                 self.request.POST.get('description'),
                 self.request.POST.get('description_renderer'),
                 self._rhodecode_user)
         except ValueError:
             msg = _(u'Cannot update closed pull requests.')
             h.flash(msg, category='error')
             return
         else:
             Session().commit()
 
         msg = _(u'Pull request title & description updated.')
         h.flash(msg, category='success')
         return
 
-    def _update_commits(self, pull_request):
+    def _update_commits(self, c, pull_request):
         _ = self.request.translate
 
         with pull_request.set_state(PullRequest.STATE_UPDATING):
             resp = PullRequestModel().update_commits(
                 pull_request, self._rhodecode_db_user)
 
         if resp.executed:
 
             if resp.target_changed and resp.source_changed:
                 changed = 'target and source repositories'
             elif resp.target_changed and not resp.source_changed:
                 changed = 'target repository'
             elif not resp.target_changed and resp.source_changed:
                 changed = 'source repository'
             else:
                 changed = 'nothing'
 
             msg = _(u'Pull request updated to "{source_commit_id}" with '
                     u'{count_added} added, {count_removed} removed commits. '
                     u'Source of changes: {change_source}')
             msg = msg.format(
                 source_commit_id=pull_request.source_ref_parts.commit_id,
                 count_added=len(resp.changes.added),
                 count_removed=len(resp.changes.removed),
                 change_source=changed)
             h.flash(msg, category='success')
 
-            channel = '/repo${}$/pr/{}'.format(
-                pull_request.target_repo.repo_name, pull_request.pull_request_id)
             message = msg + (
                 ' - <a onclick="window.location.reload()">'
                 '<strong>{}</strong></a>'.format(_('Reload page')))
+
+            message_obj = {
+                'message': message,
+                'level': 'success',
+                'topic': '/notifications'
+            }
+
             channelstream.post_message(
-                channel, message, self._rhodecode_user.username,
+                c.pr_broadcast_channel, message_obj, self._rhodecode_user.username,
                 registry=self.request.registry)
         else:
             msg = PullRequestModel.UPDATE_STATUS_MESSAGES[resp.reason]
             warning_reasons = [
                 UpdateFailureReason.NO_CHANGE,
                 UpdateFailureReason.WRONG_REF_TYPE,
             ]
             category = 'warning' if resp.reason in warning_reasons else 'error'
             h.flash(msg, category=category)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='pullrequest_merge', request_method='POST',
         renderer='json_ext')
     def pull_request_merge(self):
         """
         Merge will perform a server-side merge of the specified
         pull request, if the pull request is approved and mergeable.
         After successful merging, the pull request is automatically
         closed, with a relevant comment.
         """
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         _ = self.request.translate
 
         if pull_request.is_state_changing():
             log.debug('show: forbidden because pull request is in state %s',
                       pull_request.pull_request_state)
             msg = _(u'Cannot merge pull requests in state other than `{}`. '
                     u'Current state is: `{}`').format(PullRequest.STATE_CREATED,
                                                       pull_request.pull_request_state)
             h.flash(msg, category='error')
             raise HTTPFound(
                 h.route_path('pullrequest_show',
                              repo_name=pull_request.target_repo.repo_name,
                              pull_request_id=pull_request.pull_request_id))
 
         self.load_default_context()
 
         with pull_request.set_state(PullRequest.STATE_UPDATING):
             check = MergeCheck.validate(
                 pull_request, auth_user=self._rhodecode_user,
                 translator=self.request.translate)
         merge_possible = not check.failed
 
         for err_type, error_msg in check.errors:
             h.flash(error_msg, category=err_type)
 
         if merge_possible:
             log.debug("Pre-conditions checked, trying to merge.")
             extras = vcs_operation_context(
                 self.request.environ, repo_name=pull_request.target_repo.repo_name,
                 username=self._rhodecode_db_user.username, action='push',
                 scm=pull_request.target_repo.repo_type)
             with pull_request.set_state(PullRequest.STATE_UPDATING):
                 self._merge_pull_request(
                     pull_request, self._rhodecode_db_user, extras)
         else:
             log.debug("Pre-conditions failed, NOT merging.")
 
         raise HTTPFound(
             h.route_path('pullrequest_show',
                          repo_name=pull_request.target_repo.repo_name,
                          pull_request_id=pull_request.pull_request_id))
 
     def _merge_pull_request(self, pull_request, user, extras):
         _ = self.request.translate
         merge_resp = PullRequestModel().merge_repo(pull_request, user, extras=extras)
 
         if merge_resp.executed:
             log.debug("The merge was successful, closing the pull request.")
             PullRequestModel().close_pull_request(
                 pull_request.pull_request_id, user)
             Session().commit()
             msg = _('Pull request was successfully merged and closed.')
             h.flash(msg, category='success')
         else:
             log.debug(
                 "The merge was not successful. Merge response: %s", merge_resp)
             msg = merge_resp.merge_status_message
             h.flash(msg, category='error')
 
     def _update_reviewers(self, pull_request, review_members, reviewer_rules):
         _ = self.request.translate
 
         get_default_reviewers_data, validate_default_reviewers = \
             PullRequestModel().get_reviewer_functions()
 
         try:
             reviewers = validate_default_reviewers(review_members, reviewer_rules)
         except ValueError as e:
             log.error('Reviewers Validation: {}'.format(e))
             h.flash(e, category='error')
             return
 
         old_calculated_status = pull_request.calculated_review_status()
         PullRequestModel().update_reviewers(
             pull_request, reviewers, self._rhodecode_user)
         h.flash(_('Pull request reviewers updated.'), category='success')
         Session().commit()
 
         # trigger status changed if change in reviewers changes the status
         calculated_status = pull_request.calculated_review_status()
         if old_calculated_status != calculated_status:
             PullRequestModel().trigger_pull_request_hook(
                 pull_request, self._rhodecode_user, 'review_status_change',
                 data={'status': calculated_status})
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='pullrequest_delete', request_method='POST',
         renderer='json_ext')
     def pull_request_delete(self):
         _ = self.request.translate
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         self.load_default_context()
 
         pr_closed = pull_request.is_closed()
         allowed_to_delete = PullRequestModel().check_user_delete(
             pull_request, self._rhodecode_user) and not pr_closed
 
         # only owner can delete it !
         if allowed_to_delete:
             PullRequestModel().delete(pull_request, self._rhodecode_user)
             Session().commit()
             h.flash(_('Successfully deleted pull request'),
                     category='success')
             raise HTTPFound(h.route_path('pullrequest_show_all',
                                          repo_name=self.db_repo_name))
 
         log.warning('user %s tried to delete pull request without access',
                     self._rhodecode_user)
         raise HTTPNotFound()
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='pullrequest_comment_create', request_method='POST',
         renderer='json_ext')
     def pull_request_comment_create(self):
         _ = self.request.translate
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         pull_request_id = pull_request.pull_request_id
 
         if pull_request.is_closed():
             log.debug('comment: forbidden because pull request is closed')
             raise HTTPForbidden()
 
         allowed_to_comment = PullRequestModel().check_user_comment(
             pull_request, self._rhodecode_user)
         if not allowed_to_comment:
             log.debug(
                 'comment: forbidden because pull request is from forbidden repo')
             raise HTTPForbidden()
 
         c = self.load_default_context()
 
         status = self.request.POST.get('changeset_status', None)
         text = self.request.POST.get('text')
         comment_type = self.request.POST.get('comment_type')
         resolves_comment_id = self.request.POST.get('resolves_comment_id', None)
         close_pull_request = self.request.POST.get('close_pull_request')
 
         # the logic here should work like following, if we submit close
         # pr comment, use `close_pull_request_with_comment` function
         # else handle regular comment logic
 
         if close_pull_request:
             # only owner or admin or person with write permissions
             allowed_to_close = PullRequestModel().check_user_update(
                 pull_request, self._rhodecode_user)
             if not allowed_to_close:
                 log.debug('comment: forbidden because not allowed to close '
                           'pull request %s', pull_request_id)
                 raise HTTPForbidden()
 
             # This also triggers `review_status_change`
             comment, status = PullRequestModel().close_pull_request_with_comment(
                 pull_request, self._rhodecode_user, self.db_repo, message=text,
                 auth_user=self._rhodecode_user)
             Session().flush()
 
             PullRequestModel().trigger_pull_request_hook(
                 pull_request, self._rhodecode_user, 'comment',
                 data={'comment': comment})
 
         else:
             # regular comment case, could be inline, or one with status.
             # for that one we check also permissions
 
             allowed_to_change_status = PullRequestModel().check_user_change_status(
                 pull_request, self._rhodecode_user)
 
             if status and allowed_to_change_status:
                 message = (_('Status change %(transition_icon)s %(status)s')
                            % {'transition_icon': '>',
                               'status': ChangesetStatus.get_status_lbl(status)})
                 text = text or message
 
             comment = CommentsModel().create(
                 text=text,
                 repo=self.db_repo.repo_id,
                 user=self._rhodecode_user.user_id,
                 pull_request=pull_request,
                 f_path=self.request.POST.get('f_path'),
                 line_no=self.request.POST.get('line'),
                 status_change=(ChangesetStatus.get_status_lbl(status)
                                if status and allowed_to_change_status else None),
                 status_change_type=(status
                                     if status and allowed_to_change_status else None),
                 comment_type=comment_type,
                 resolves_comment_id=resolves_comment_id,
                 auth_user=self._rhodecode_user
             )
 
             if allowed_to_change_status:
                 # calculate old status before we change it
                 old_calculated_status = pull_request.calculated_review_status()
 
                 # get status if set !
                 if status:
                     ChangesetStatusModel().set_status(
                         self.db_repo.repo_id,
                         status,
                         self._rhodecode_user.user_id,
                         comment,
                         pull_request=pull_request
                     )
 
                 Session().flush()
                 # this is somehow required to get access to some relationship
                 # loaded on comment
                 Session().refresh(comment)
 
                 PullRequestModel().trigger_pull_request_hook(
                     pull_request, self._rhodecode_user, 'comment',
                     data={'comment': comment})
 
                 # we now calculate the status of pull request, and based on that
                 # calculation we set the commits status
                 calculated_status = pull_request.calculated_review_status()
                 if old_calculated_status != calculated_status:
                     PullRequestModel().trigger_pull_request_hook(
                         pull_request, self._rhodecode_user, 'review_status_change',
                         data={'status': calculated_status})
 
         Session().commit()
 
         data = {
             'target_id': h.safeid(h.safe_unicode(
                 self.request.POST.get('f_path'))),
         }
         if comment:
             c.co = comment
+            c.at_version_num = None
             rendered_comment = render(
                 'rhodecode:templates/changeset/changeset_comment_block.mako',
                 self._get_template_context(c), self.request)
 
             data.update(comment.get_dict())
             data.update({'rendered_text': rendered_comment})
 
         return data
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='pullrequest_comment_delete', request_method='POST',
         renderer='json_ext')
     def pull_request_comment_delete(self):
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
 
         comment = ChangesetComment.get_or_404(
             self.request.matchdict['comment_id'])
         comment_id = comment.comment_id
 
         if comment.immutable:
             # don't allow deleting comments that are immutable
             raise HTTPForbidden()
 
         if pull_request.is_closed():
             log.debug('comment: forbidden because pull request is closed')
             raise HTTPForbidden()
 
         if not comment:
             log.debug('Comment with id:%s not found, skipping', comment_id)
             # comment already deleted in another call probably
             return True
 
         if comment.pull_request.is_closed():
             # don't allow deleting comments on closed pull request
             raise HTTPForbidden()
 
         is_repo_admin = h.HasRepoPermissionAny('repository.admin')(self.db_repo_name)
         super_admin = h.HasPermissionAny('hg.admin')()
         comment_owner = comment.author.user_id == self._rhodecode_user.user_id
         is_repo_comment = comment.repo.repo_name == self.db_repo_name
         comment_repo_admin = is_repo_admin and is_repo_comment
 
         if super_admin or comment_owner or comment_repo_admin:
             old_calculated_status = comment.pull_request.calculated_review_status()
             CommentsModel().delete(comment=comment, auth_user=self._rhodecode_user)
             Session().commit()
             calculated_status = comment.pull_request.calculated_review_status()
             if old_calculated_status != calculated_status:
                 PullRequestModel().trigger_pull_request_hook(
                     comment.pull_request, self._rhodecode_user, 'review_status_change',
                     data={'status': calculated_status})
             return True
         else:
             log.warning('No permissions for user %s to delete comment_id: %s',
                         self._rhodecode_db_user, comment_id)
             raise HTTPNotFound()
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='pullrequest_comment_edit', request_method='POST',
         renderer='json_ext')
     def pull_request_comment_edit(self):
         self.load_default_context()
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id']
         )
         comment = ChangesetComment.get_or_404(
             self.request.matchdict['comment_id']
         )
         comment_id = comment.comment_id
 
         if comment.immutable:
             # don't allow deleting comments that are immutable
             raise HTTPForbidden()
 
         if pull_request.is_closed():
             log.debug('comment: forbidden because pull request is closed')
             raise HTTPForbidden()
 
         if not comment:
             log.debug('Comment with id:%s not found, skipping', comment_id)
             # comment already deleted in another call probably
             return True
 
         if comment.pull_request.is_closed():
             # don't allow deleting comments on closed pull request
             raise HTTPForbidden()
 
         is_repo_admin = h.HasRepoPermissionAny('repository.admin')(self.db_repo_name)
         super_admin = h.HasPermissionAny('hg.admin')()
         comment_owner = comment.author.user_id == self._rhodecode_user.user_id
         is_repo_comment = comment.repo.repo_name == self.db_repo_name
         comment_repo_admin = is_repo_admin and is_repo_comment
 
         if super_admin or comment_owner or comment_repo_admin:
             text = self.request.POST.get('text')
             version = self.request.POST.get('version')
             if text == comment.text:
                 log.warning(
                     'Comment(PR): '
                     'Trying to create new version '
                     'with the same comment body {}'.format(
                         comment_id,
                     )
                 )
                 raise HTTPNotFound()
 
             if version.isdigit():
                 version = int(version)
             else:
                 log.warning(
                     'Comment(PR): Wrong version type {} {} '
                     'for comment {}'.format(
                         version,
                         type(version),
                         comment_id,
                     )
                 )
                 raise HTTPNotFound()
 
             try:
                 comment_history = CommentsModel().edit(
                     comment_id=comment_id,
                     text=text,
                     auth_user=self._rhodecode_user,
                     version=version,
                 )
             except CommentVersionMismatch:
                 raise HTTPConflict()
 
             if not comment_history:
                 raise HTTPNotFound()
 
             Session().commit()
 
             PullRequestModel().trigger_pull_request_hook(
                 pull_request, self._rhodecode_user, 'comment_edit',
                 data={'comment': comment})
 
             return {
                 'comment_history_id': comment_history.comment_history_id,
                 'comment_id': comment.comment_id,
                 'comment_version': comment_history.version,
                 'comment_author_username': comment_history.author.username,
                 'comment_author_gravatar': h.gravatar_url(comment_history.author.email, 16),
                 'comment_created_on': h.age_component(comment_history.created_on,
                                                       time_is_local=True),
             }
         else:
             log.warning('No permissions for user %s to edit comment_id: %s',
                         self._rhodecode_db_user, comment_id)
             raise HTTPNotFound()