docs: updated scaling/cluster docs

docs/admin/apache-config.rst

@@ -1,14 +1,14 b''
 .. _apache-ws-ref:
 
 Apache HTTP Server Configuration
 --------------------------------
 
 To set up your Apache Web Server for optimal performance and security, use
 the information in the following sections.
 
 .. toctree::
 
+   apache-conf-example
    apache-diffie-hellman
-   apache-conf-example
    apache-subdirectory
    apache-wsgi-coding

docs/admin/nginx-config-example.rst

@@ -1,143 +1,146 b''
 Nginx Configuration Example
 ---------------------------
 
 Use the following example to configure Nginx as a your web server.
 
 
 .. code-block:: nginx
 
     ## rate limiter for certain pages to prevent brute force attacks
-    limit_req_zone  $binary_remote_addr  zone=dl_limit:10m   rate=1r/s;
+    limit_req_zone  $binary_remote_addr  zone=req_limit:10m   rate=1r/s;
 
     ## custom log format
     log_format log_custom '$remote_addr - $remote_user [$time_local] '
                           '"$request" $status $body_bytes_sent '
                           '"$http_referer" "$http_user_agent" '
                           '$request_time $upstream_response_time $pipe';
 
     ## define upstream (local RhodeCode instance) to connect to
     upstream rc {
         # Url to running RhodeCode instance.
-        # This is shown as `- URL:` in output from rccontrol status.
+        # This is shown as `- URL: <host>` in output from rccontrol status.
         server 127.0.0.1:10002;
 
         # add more instances for load balancing
         # server 127.0.0.1:10003;
         # server 127.0.0.1:10004;
     }
 
     ## HTTP to HTTPS rewrite
     server {
         listen          80;
         server_name     rhodecode.myserver.com;
 
         if ($http_host = rhodecode.myserver.com) {
             rewrite  (.*)  https://rhodecode.myserver.com$1 permanent;
         }
     }
 
     ## Optional gist alias server, for serving nicer GIST urls.
     server {
         listen          443;
         server_name     gist.myserver.com;
         access_log      /var/log/nginx/gist.access.log log_custom;
         error_log       /var/log/nginx/gist.error.log;
 
         ssl on;
         ssl_certificate     gist.rhodecode.myserver.com.crt;
         ssl_certificate_key gist.rhodecode.myserver.com.key;
 
         ssl_session_timeout 5m;
 
         ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
         ssl_prefer_server_ciphers on;
         ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA';
 
         # strict http prevents from https -> http downgrade
         add_header Strict-Transport-Security "max-age=31536000; includeSubdomains;";
 
         # Diffie-Hellman parameter for DHE ciphersuites, recommended 2048 bits
         #ssl_dhparam /etc/nginx/ssl/dhparam.pem;
 
         rewrite ^/(.+)$ https://rhodecode.myserver.com/_admin/gists/$1;
         rewrite (.*)    https://rhodecode.myserver.com/_admin/gists;
     }
 
 
     ## MAIN SSL enabled server
     server {
         listen       443 ssl;
         server_name  rhodecode.myserver.com;
 
         access_log   /var/log/nginx/rhodecode.access.log log_custom;
         error_log    /var/log/nginx/rhodecode.error.log;
 
         ssl on;
         ssl_certificate     rhodecode.myserver.com.crt;
         ssl_certificate_key rhodecode.myserver.com.key;
 
         ssl_session_timeout 5m;
 
         ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
         ssl_prefer_server_ciphers on;
         ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA';
 
         # Diffie-Hellman parameter for DHE ciphersuites, recommended 2048 bits
         #ssl_dhparam /etc/nginx/ssl/dhparam.pem;
 
+        # example of proxy.conf can be found in our docs.
         include     /etc/nginx/proxy.conf;
 
-        ## serve static files by Nginx, recommended for performance
+        ## uncomment to serve static files by Nginx, recommended for performance
         # location /_static/rhodecode {
         #    gzip on;
         #    gzip_min_length  500;
         #    gzip_proxied     any;
         #    gzip_comp_level 4;
         #    gzip_types  text/css text/javascript text/xml text/plain text/x-component application/javascript application/json application/xml application/rss+xml font/truetype font/opentype application/vnd.ms-fontobject image/svg+xml;
         #    gzip_vary on;
         #    gzip_disable     "msie6";
+        #    alias /path/to/.rccontrol/community-1/static;
         #    alias /path/to/.rccontrol/enterprise-1/static;
         # }
 
         ## channelstream websocket handling
         location /_channelstream {
             rewrite /_channelstream/(.*) /$1 break;
 
             proxy_pass                  http://127.0.0.1:9800;
 
             proxy_connect_timeout        10;
             proxy_send_timeout           10m;
             proxy_read_timeout           10m;
             tcp_nodelay                  off;
             proxy_set_header             Host $host;
             proxy_set_header             X-Real-IP $remote_addr;
             proxy_set_header             X-Url-Scheme $scheme;
             proxy_set_header             X-Forwarded-Proto $scheme;
             proxy_set_header             X-Forwarded-For $proxy_add_x_forwarded_for;
             gzip                         off;
             proxy_http_version           1.1;
             proxy_set_header Upgrade     $http_upgrade;
             proxy_set_header Connection  "upgrade";
         }
 
+        ## rate limit this endpoint to prevent login page brute-force attacks
         location /_admin/login {
-            ## rate limit this endpoint
-            limit_req  zone=dl_limit  burst=10  nodelay;
+            limit_req  zone=req_limit  burst=10  nodelay;
             try_files $uri @rhode;
         }
 
         location / {
             try_files $uri @rhode;
         }
 
         location @rhode {
             proxy_pass      http://rc;
         }
 
         ## custom 502 error page. Will be displayed while RhodeCode server
         ## is turned off
         error_page 502 /502.html;
         location = /502.html {
+           #root  /path/to/.rccontrol/community-1/static;
            root  /path/to/.rccontrol/enterprise-1/static;
         }
     }
  No newline at end of file

docs/admin/nginx-config.rst

@@ -1,14 +1,14 b''
 .. _nginx-ws-ref:
 
 Nginx HTTP Server Configuration
 -------------------------------
 
 To set up your Nginx Web Server for optimal performance and security, use
 the information in the following sections.
 
 .. toctree::
 
+   nginx-config-example
    nginx-diffie-hellman
-   nginx-config-example
-   nginx-tuning
+   nginx-proxy-conf
    nginx-url-prefix

docs/admin/nginx-proxy-conf.rst

@@ -1,33 +1,34 b''
-.. _nginx-tuning:
+.. _nginx-proxy-conf:
 
-Nginx Tuning
-------------
+Nginx Proxy Config
+------------------
+
 
 Set the following properties in your ``/etc/nginx/proxy.conf`` so it does not
 timeout during large pushes.
 
 .. code-block:: nginx
 
     proxy_redirect              off;
     proxy_set_header            Host $http_host;
 
     ## needed for container auth
     # proxy_set_header            REMOTE_USER $remote_user;
     # proxy_set_header            X-Forwarded-User $remote_user;
 
     proxy_set_header            X-Url-Scheme $scheme;
     proxy_set_header            X-Host $http_host;
     proxy_set_header            X-Real-IP $remote_addr;
     proxy_set_header            X-Forwarded-For $proxy_add_x_forwarded_for;
     proxy_set_header            Proxy-host $proxy_host;
     proxy_buffering             off;
     proxy_connect_timeout       7200;
     proxy_send_timeout          7200;
     proxy_read_timeout          7200;
     proxy_buffers               8 32k;
     # Set this to a larger number if you experience timeouts
     client_max_body_size        1024m;
     client_body_buffer_size     128k;
     large_client_header_buffers 8 64k;
     add_header X-Frame-Options SAMEORIGIN;
     add_header Strict-Transport-Security "max-age=31536000; includeSubdomains;";

docs/admin/tuning-change-large-file-dir.rst

@@ -1,14 +1,14 b''
 .. _hg-lrg-loc:
 
-Change the |hg| Large Files Location
-------------------------------------
+|hg| Large Files Location
+-------------------------
 
 |RCE| manages |hg| larges files from the following default location
 :file:`/home/{user}/repos/.cache/largefiles`. If you wish to change this, use
 the following steps:
 
 1. Open :menuselection:`Admin --> Settings --> VCS` as super-admin.
 
 In section called `Mercurial Settings` you can change where the largefiles
 objects should be stored.
 

docs/admin/tuning-change-lfs-dir.rst

@@ -1,14 +1,14 b''
 .. _git-lfs-loc:
 
-Change the |git| LFS storage Location
--------------------------------------
+|git| LFS storage Location
+--------------------------
 
 |RCE| manages |git| LFS files from the following default location
 :file:`/home/{user}/repos/.cache/lfs_store`. If you wish to change this, use
 the following steps:
 
 1. Open :menuselection:`Admin --> Settings --> VCS` as super-admin.
 
 In section called `Git Settings` you can change where the LFS
 objects should be stored.
 

docs/admin/tuning-gunicorn.rst

@@ -1,124 +1,126 b''
 .. _increase-gunicorn:
 
-Increase Gunicorn Workers
--------------------------
+Configure Gunicorn Workers
+--------------------------
 
 
-|RCE| comes with `Gunicorn`_ packaged in its Nix environment.
-Gunicorn is a Python WSGI HTTP Server for UNIX.
+|RCE| comes with `Gunicorn`_ which is a Python WSGI HTTP Server for UNIX.
 
 To improve |RCE| performance you can increase the number of `Gunicorn`_  workers.
-This allows to handle more connections concurently, and provide better
+This allows to handle more connections concurrently, and provide better
 responsiveness and performance.
 
 By default during installation |RCC|  tries to detect how many CPUs are
 available in the system, and set the number workers based on that information.
 However sometimes it's better to manually set the number of workers.
 
 To do this, use the following steps:
 
 1. Open the :file:`home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file.
-2. In the ``[server:main]`` section, increase the number of Gunicorn
-   ``workers`` using the following formula :math:`(2 * Cores) + 1`.
+2. In the ``[server:main]`` section, change the number of Gunicorn
+   ``workers`` using the following default formula :math:`(2 * Cores) + 1`.
+   We however not recommend using more than 8-12 workers per server. It's better
+   to start using the :ref:`scale-horizontal-cluster` in case that performance
+   with 8-12 workers is not enough.
 
 .. code-block:: ini
 
     use = egg:gunicorn#main
     ## Sets the number of process workers. You must set `instance_id = *`
     ## when this option is set to more than one worker, recommended
     ## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
     ## The `instance_id = *` must be set in the [app:main] section below
     workers = 4
     ## process name
     proc_name = rhodecode
     ## type of worker class, one of sync, gevent
     ## recommended for bigger setup is using of of other than sync one
     worker_class = sync
     ## The maximum number of simultaneous clients. Valid only for Gevent
     #worker_connections = 10
     ## max number of requests that worker will handle before being gracefully
     ## restarted, could prevent memory leaks
     max_requests = 1000
     max_requests_jitter = 30
     ## amount of time a worker can spend with handling a request before it
     ## gets killed and restarted. Set to 6hrs
     timeout = 21600
 
 3. In the ``[app:main]`` section, set the ``instance_id`` property to ``*``.
 
 .. code-block:: ini
 
     # In the [app:main] section
     [app:main]
     # You must set `instance_id = *`
     instance_id = *
 
 4. Change the VCSServer workers too. Open the
    :file:`home/{user}/.rccontrol/{instance-id}/vcsserver.ini` file.
 
 5. In the ``[server:main]`` section, increase the number of Gunicorn
    ``workers`` using the following formula :math:`(2 * Cores) + 1`.
 
 .. code-block:: ini
 
     ## run with gunicorn --log-config vcsserver.ini --paste vcsserver.ini
     use = egg:gunicorn#main
     ## Sets the number of process workers. Recommended
     ## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
     workers = 4
     ## process name
     proc_name = rhodecode_vcsserver
     ## type of worker class, currently `sync` is the only option allowed.
     worker_class = sync
     ## The maximum number of simultaneous clients. Valid only for Gevent
     #worker_connections = 10
     ## max number of requests that worker will handle before being gracefully
     ## restarted, could prevent memory leaks
     max_requests = 1000
     max_requests_jitter = 30
     ## amount of time a worker can spend with handling a request before it
     ## gets killed and restarted. Set to 6hrs
     timeout = 21600
 
 6. Save your changes.
 7. Restart your |RCE| instances, using the following command:
 
 .. code-block:: bash
 
     $ rccontrol restart '*'
 
 
 Gunicorn Gevent Backend
 -----------------------
 
 Gevent is an asynchronous worker type for Gunicorn. It allows accepting multiple
 connections on a single `Gunicorn`_  worker. This means you can handle 100s
 of concurrent clones, or API calls using just few workers. A setting called
 `worker_connections` defines on how many connections each worker can
 handle using `Gevent`.
 
 
 To enable `Gevent` on |RCE| do the following:
 
 
 1. Open the :file:`home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file.
 2. In the ``[server:main]`` section, change `worker_class` for Gunicorn.
 
 
 .. code-block:: ini
 
     ## type of worker class, one of sync, gevent
     ## recommended for bigger setup is using of of other than sync one
     worker_class = gevent
     ## The maximum number of simultaneous clients. Valid only for Gevent
     worker_connections = 30
 
 
 .. note::
 
     `Gevent` is currently only supported for Enterprise/Community instances.
     VCSServer doesn't yet support gevent.
 
 
 
 .. _Gunicorn: http://gunicorn.org/

docs/admin/tuning-increase-db-performance.rst

@@ -1,8 +1,8 b''
 .. _db-session-ref:
 
-Increase Database Performance
------------------------------
+Database Performance
+--------------------
 
 For tuning PostgreSQL we recommend reading: http://www.revsys.com/writings/postgresql-performance.html
 
 For tuning MySQL we recommend reading: http://www.tecmint.com/mysql-mariadb-performance-tuning-and-optimization/
  No newline at end of file

docs/admin/tuning-rhodecode.rst

@@ -1,22 +1,21 b''
 .. _rhodecode-tuning-ref:
 
 Tuning |RCE|
 ============
 
 To customize your |RCE| |version| installation for maximum performance you
 may find some of the following methods useful.
 
 .. toctree::
 
    tuning-gunicorn
    tuning-vcs-memory-cache
    tuning-user-sessions-performance
    tuning-increase-db-performance
-   tuning-scale-horizontally
-   tuning-increase-cache-size
+   tuning-scale-horizontally-cluster
    tuning-mount-cache-memory
    tuning-change-encoding
    tuning-change-large-file-dir
    tuning-change-lfs-dir
    tuning-hg-auth-loop
 

docs/admin/tuning-scale-horizontally-cluster.rst

@@ -1,58 +1,383 b''
-.. _scale-horizontal:
+.. _scale-horizontal-cluster:
+
 
-Scale Horizontally
-------------------
+Scale Horizontally / RhodeCode Cluster
+--------------------------------------
 
 |RCE| is built in a way it support horizontal scaling across multiple machines.
-There are two main pre-requisites for that:
+There are three main pre-requisites for that:
 
-- Shared storage that each machine can access.
-- Shared DB connection across machines.
+- Shared storage that each machine can access. Using NFS or other shared storage system.
+- Shared DB connection across machines. Using `MySQL`/`PostgreSQL` that each node can access.
+- |RCE| user sessions and caches need to use a shared storage (e.g `Redis`_/`Memcached`)
 
 
 Horizontal scaling means adding more machines or workers into your pool of
 resources. Horizontally scaling |RCE| gives a huge performance increase,
-especially under large traffic scenarios with a high number of requests. This
-is very beneficial when |RCE| is serving many users simultaneously,
+especially under large traffic scenarios with a high number of requests.
+This is very beneficial when |RCE| is serving many users simultaneously,
 or if continuous integration servers are automatically pulling and pushing code.
+It also adds High-Availability to your running system.
+
+
+Cluster Overview
+^^^^^^^^^^^^^^^^
+
+Below we'll present a configuration example that will use two separate nodes to serve
+|RCE| in a load-balanced environment. The 3rd node will act as a shared storage/cache
+and handle load-balancing. In addition 3rd node will be used as shared database instance.
+
+This setup can be used both in Docker based configuration or with individual
+physical/virtual machines. Using the 3rd node for Storage/Redis/PostgreSQL/Nginx is
+optional. All those components can be installed on one of the two nodes used for |RCE|.
+We'll use following naming for our nodes:
+
+ - `rc-node-1` (NFS, DB, Cache node)
+ - `rc-node-2` (Worker node1)
+ - `rc-node-3` (Worker node2)
+
+Our shares NFS storage in the example is located on `/home/rcdev/storage` and
+it's RW accessible on **each** node.
+
+In this example we used certain recommended components, however many
+of those can be replaced by other, in case your organization already uses them, for example:
+
+- `MySQL`/`PostgreSQL`: Aren't replaceable and are the two only supported databases.
+- `Nginx`_ on `rc-node-1` can be replaced by: `Hardware Load Balancer (F5)`, `Apache`_, `HA-Proxy` etc.
+- `Nginx`_ on rc-node-2/3 acts as a reverse proxy and can be replaced by other HTTP server
+  acting as reverse proxy such as `Apache`_.
+- `Redis`_ on `rc-node-1` can be replaced by: `Memcached`
+
+
+Here's an overview what components should be installed/setup on each server in our example:
+
+- **rc-node-1**:
+
+ - main storage acting as NFS host.
+ - `nginx` acting as a load-balancer.
+ - `postgresql-server` used for database and sessions.
+ - `redis-server` used for storing shared caches.
+ - optionally `rabbitmq-server` for `Celery` if used.
+ - optionally if `Celery` is used Enterprise/Community instance + VCSServer.
+ - optionally mailserver that can be shared by other instances.
+ - optionally channelstream server to handle live communication for all instances.
+
+
+- **rc-node-2/3**:
+
+ - `nginx` acting as a reverse proxy to handle requests to |RCE|.
+ - 1x RhodeCode Enterprise/Community instance.
+ - 1x VCSServer instance.
+ - optionally for testing connection: postgresql-client, redis-client (redis-tools).
+
+
+Before we start here are few assumptions that should be fulfilled:
+
+- make sure each node can access each other.
+- make sure `Redis`_/`MySQL`/`PostgreSQL`/`RabbitMQ`_ are running on `rc-node-1`
+- make sure both `rc-node-2`/`3` can access NFS storage with RW access
+- make sure rc-node-2/3 can access `Redis`_/`PostgreSQL`, `MySQL` database on `rc-node-1`.
+- make sure `Redis`_/Database/`RabbitMQ`_ are password protected and accessible only from rc-node-2/3.
+
 
 
-If you scale across different machines, each |RCM| instance
-needs to store its data on a shared disk, preferably together with your
-|repos|. This data directory contains template caches, a full text search index,
-and is used for task locking to ensure safety across multiple instances.
-To do this, set the following properties in the :file:`rhodecode.ini` file to
-set the shared location across all |RCM| instances.
+Setup rc-node-2/3
+^^^^^^^^^^^^^^^^^
+
+Initially before `rc-node-1` we'll configure both nodes 2 and 3 to operate as standalone
+nodes with their own hostnames. Use a default installation settings, and use
+the default local addresses (127.0.0.1) to configure VCSServer and Community/Enterprise instances.
+All external connectivity will be handled by the reverse proxy (`Nginx`_ in our example).
+
+This way we can ensure each individual host works,
+accepts connections, or do some operations explicitly on chosen node.
+
+In addition this would allow use to explicitly direct certain traffic to a node, e.g
+CI server will only call directly `rc-node-3`. This should be done similar to normal
+installation so check out `Nginx`_/`Apache`_ configuration example to configure each host.
+Each one should already connect to shared database during installation.
+
+
+1) Assuming our final url will be http://rc-node-1, Configure `instances_id`, `app.base_url`
+
+a) On **rc-node-2** find the following settings and edit :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+
+.. code-block:: ini
+
+    ## required format is: *NAME-
+    instance_id = *rc-node-2-
+    app.base_url = http://rc-node-1
+
+
+b) On **rc-node-3** find the following settings and edit :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+
+.. code-block:: ini
+
+    ## required format is: *NAME-
+    instance_id = *rc-node-3-
+    app.base_url = http://rc-node-1
+
+
+
+2) Configure `User Session` to use a shared database. Example config that should be
+   changed on both node 2 and 3. Edit :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+
+.. code-block:: ini
+
+    ####################################
+    ###       BEAKER SESSION        ####
+    ####################################
+
+    ## Disable the default `file` sessions
+    #beaker.session.type = file
+    #beaker.session.data_dir = %(here)s/data/sessions
+
+    ## use shared db based session, fast, and allows easy management over logged in users
+    beaker.session.type = ext:database
+    beaker.session.table_name = db_session
+    # use our rc-node-1 here
+    beaker.session.sa.url = postgresql://postgres:qweqwe@rc-node-1/rhodecode
+    beaker.session.sa.pool_recycle = 3600
+    beaker.session.sa.echo = false
+
+In addition make sure both instances use the same `session.secret` so users have
+persistent sessions across nodes. Please generate other one then in this example.
+
+.. code-block:: ini
+
+    # use an unique generated long string
+    beaker.session.secret = 70e116cae2274656ba7265fd860aebbd
+
+3) Configure stored cached/archive cache to our shared NFS `rc-node-1`
 
 .. code-block:: ini
 
-    cache_dir = /shared/path/caches                 # set to shared location
-    search.location = /shared/path/search_index     # set to shared location
+    # note the `_` prefix that allows using a directory without
+    # remap and rescan checking for vcs inside it.
+    cache_dir = /home/rcdev/storage/_cache_dir/data
+    # note archive cache dir is disabled by default, however if you enable
+    # it also needs to be shared
+    #archive_cache_dir = /home/rcdev/storage/_tarball_cache_dir
+
+
+4) Change cache backends to use `Redis`_ based caches. Below full example config
+   that replaces default file-based cache to shared `Redis`_ with Distributed Lock.
+
+
+.. code-block:: ini
+
+    #####################################
+    ###         DOGPILE CACHE        ####
+    #####################################
+
+    ## `cache_perms` cache settings for permission tree, auth TTL.
+    #rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
+    #rc_cache.cache_perms.expiration_time = 300
+
+    ## alternative `cache_perms` redis backend with distributed lock
+    rc_cache.cache_perms.backend = dogpile.cache.rc.redis
+    rc_cache.cache_perms.expiration_time = 300
+    ## redis_expiration_time needs to be greater then expiration_time
+    rc_cache.cache_perms.arguments.redis_expiration_time = 7200
+    rc_cache.cache_perms.arguments.socket_timeout = 30
+    rc_cache.cache_perms.arguments.host = rc-node-1
+    rc_cache.cache_perms.arguments.password = qweqwe
+    rc_cache.cache_perms.arguments.port = 6379
+    rc_cache.cache_perms.arguments.db = 0
+    rc_cache.cache_perms.arguments.distributed_lock = true
+
+    ## `cache_repo` cache settings for FileTree, Readme, RSS FEEDS
+    #rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace
+    #rc_cache.cache_repo.expiration_time = 2592000
+
+    ## alternative `cache_repo` redis backend with distributed lock
+    rc_cache.cache_repo.backend = dogpile.cache.rc.redis
+    rc_cache.cache_repo.expiration_time = 2592000
+    ## redis_expiration_time needs to be greater then expiration_time
+    rc_cache.cache_repo.arguments.redis_expiration_time = 2678400
+    rc_cache.cache_repo.arguments.socket_timeout = 30
+    rc_cache.cache_repo.arguments.host = rc-node-1
+    rc_cache.cache_repo.arguments.password = qweqwe
+    rc_cache.cache_repo.arguments.port = 6379
+    rc_cache.cache_repo.arguments.db = 1
+    rc_cache.cache_repo.arguments.distributed_lock = true
+
+    ## cache settings for SQL queries, this needs to use memory type backend
+    rc_cache.sql_cache_short.backend = dogpile.cache.rc.memory_lru
+    rc_cache.sql_cache_short.expiration_time = 30
+
+    ## `cache_repo_longterm` cache for repo object instances, this needs to use memory
+    ## type backend as the objects kept are not pickle serializable
+    rc_cache.cache_repo_longterm.backend = dogpile.cache.rc.memory_lru
+    ## by default we use 96H, this is using invalidation on push anyway
+    rc_cache.cache_repo_longterm.expiration_time = 345600
+    ## max items in LRU cache, reduce this number to save memory, and expire last used
+    ## cached objects
+    rc_cache.cache_repo_longterm.max_size = 10000
+
+
+4) Configure `Nginx`_ as reverse proxy on `rc-node-2/3`:
+   Minimal `Nginx`_ config used:
+
 
-    ####################################
-    ###         BEAKER CACHE        ####
-    ####################################
-    beaker.cache.data_dir = /shared/path/data       # set to shared location
-    beaker.cache.lock_dir = /shared/path/lock       # set to shared location
+.. code-block:: nginx
+
+    ## rate limiter for certain pages to prevent brute force attacks
+    limit_req_zone  $binary_remote_addr  zone=req_limit:10m   rate=1r/s;
+
+    ## custom log format
+    log_format log_custom '$remote_addr - $remote_user [$time_local] '
+                          '"$request" $status $body_bytes_sent '
+                          '"$http_referer" "$http_user_agent" '
+                          '$request_time $upstream_response_time $pipe';
+
+    server {
+        listen          80;
+        server_name     rc-node-2;
+        #server_name     rc-node-3;
+
+        access_log   /var/log/nginx/rhodecode.access.log log_custom;
+        error_log    /var/log/nginx/rhodecode.error.log;
+
+        # example of proxy.conf can be found in our docs.
+        include     /etc/nginx/proxy.conf;
+
+        ## serve static files by Nginx, recommended for performance
+        location /_static/rhodecode {
+            gzip on;
+            gzip_min_length  500;
+            gzip_proxied     any;
+            gzip_comp_level 4;
+            gzip_types  text/css text/javascript text/xml text/plain text/x-component application/javascript application/json application/xml application/rss+xml font/truetype font/opentype application/vnd.ms-fontobject image/svg+xml;
+            gzip_vary on;
+            gzip_disable     "msie6";
+            #alias /home/rcdev/.rccontrol/community-1/static;
+            alias /home/rcdev/.rccontrol/enterprise-1/static;
+        }
+
+
+        location /_admin/login {
+            limit_req  zone=req_limit  burst=10  nodelay;
+            try_files $uri @rhode;
+        }
+
+        location / {
+            try_files $uri @rhode;
+        }
+
+        location @rhode {
+            # Url to running RhodeCode instance.
+            # This is shown as `- URL: <host>` in output from rccontrol status.
+            proxy_pass      http://127.0.0.1:10020;
+        }
+
+        ## custom 502 error page. Will be displayed while RhodeCode server
+        ## is turned off
+        error_page 502 /502.html;
+        location = /502.html {
+           #root  /home/rcdev/.rccontrol/community-1/static;
+           root  /home/rcdev/.rccontrol/enterprise-1/static;
+        }
+    }
+
+
+5) Optional: Full text search, in case you use `Whoosh` full text search we also need a
+   shared storage for the index. In our example our NFS is mounted at `/home/rcdev/storage`
+   which represents out storage so we can use the following:
+
+.. code-block:: ini
+
+    # note the `_` prefix that allows using a directory without
+    # remap and rescan checking for vcs inside it.
+    search.location = /home/rcdev/storage/_index_data/index
 
 
 .. note::
 
-    If you use custom caches such as `beaker.cache.auth_plugins.` it's recommended
-    to set it to the memcached/redis or database backend so it can be shared
-    across machines.
+    If you use ElasticSearch it's by default shared, and simply running ES node is
+    by default cluster compatible.
+
+
+6) Optional: If you intend to use mailing all instances need to use either a shared
+   mailing node, or each will use individual local mailagent. Simply put node-1/2/3 needs
+   to use same mailing configuration.
+
+
+
+Setup rc-node-1
+^^^^^^^^^^^^^^^
 
 
-It is recommended to create another dedicated |RCE| instance to handle
-traffic from build farms or continuous integration servers.
+Configure `Nginx`_ as Load Balancer to rc-node-2/3.
+Minimal `Nginx`_ example below:
+
+.. code-block:: nginx
+
+    ## define rc-cluster which contains a pool of our instances to connect to
+    upstream rc-cluster {
+        # rc-node-2/3 are stored in /etc/hosts with correct IP addresses
+        server rc-node-2:80;
+        server rc-node-3:80;
+    }
+
+    server {
+        listen          80;
+        server_name     rc-node-1;
+
+        location / {
+            proxy_pass http://rc-cluster;
+        }
+    }
+
 
 .. note::
 
    You should configure your load balancing accordingly. We recommend writing
    load balancing rules that will separate regular user traffic from
-   automated process traffic like continuous servers or build bots.
+   automated process traffic like continuous servers or build bots. Sticky sessions
+   are not required.
+
+
+Show which instance handles a request
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+You can easily check if load-balancing is working as expected. Visit our main node
+`rc-node-1` URL which at that point should already handle incoming requests and balance
+it across node-2/3.
+
+Add a special GET param `?showrcid=1` to show current instance handling your request.
+
+For example: visiting url `http://rc-node-1/?showrcid=1` will show, in the bottom
+of the screen` cluster instance info.
+e.g: `RhodeCode instance id: rc-node-3-rc-node-3-3246`
+which is generated from::
+
+    <NODE_HOSTNAME>-<INSTANCE_ID>-<WORKER_PID>
+
+
+Using Celery with cluster
+^^^^^^^^^^^^^^^^^^^^^^^^^
 
-.. note::
+
+If `Celery` is used we recommend setting also an instance of Enterprise/Community+VCSserver
+on the node that is running `RabbitMQ`_. Those instances will be used to executed async
+tasks on the `rc-node-1`. This is the most efficient setup. `Celery` usually
+handles tasks such as sending emails, forking repositories, importing
+repositories from external location etc. Using workers on instance that has
+the direct access to disks used by NFS as well as email server gives noticeable
+performance boost. Running local workers to the NFS storage results in faster
+execution of forking large repositories or sending lots of emails.
 
-     If Celery is used on each instance then you should run separate Celery
-     instances, but the message broker should be the same for all of them.
+Those instances need to be configured in the same way as for other nodes.
+The instance in rc-node-1 can be added to the cluser, but we don't recommend doing it.
+For best results let it be isolated to only executing `Celery` tasks in the cluster setup.
+
+
+.. _Gunicorn: http://gunicorn.org/
+.. _Whoosh: https://pypi.python.org/pypi/Whoosh/
+.. _Elasticsearch: https://www.elastic.co/..
+.. _RabbitMQ: http://www.rabbitmq.com/
+.. _Nginx: http://nginx.io
+.. _Apache: http://nginx.io
+.. _Redis: http://redis.io
+

docs/admin/tuning-user-sessions-performance.rst

@@ -1,66 +1,67 b''
 .. _user-session-ref:
 
-Increase User Session Performance
----------------------------------
+User Session Performance
+------------------------
 
 The default file-based sessions are only suitable for smaller setups, or
 instances that doesn't have a lot of users or traffic.
 They are set as default option because it's setup-free solution.
 
 The most common issue of file based sessions are file limit errors which occur
 if there are lots of session files.
 
 Therefore, in a large scale deployment, to give better performance,
 scalability, and maintainability we recommend switching from file-based
 sessions to database-based user sessions or Redis based sessions.
 
 To switch to database-based user sessions uncomment the following section in
 your :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file.
 
 
 .. code-block:: ini
 
       ## db based session, fast, and allows easy management over logged in users
       beaker.session.type = ext:database
       beaker.session.table_name = db_session
 
-      # use just one of the following accoring to the type of database
+      # use just one of the following according to the type of database
       beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode
+      # or
       beaker.session.sa.url = mysql://root:secret@127.0.0.1/rhodecode
 
       beaker.session.sa.pool_recycle = 3600
       beaker.session.sa.echo = false
 
 
 and make sure you comment out the file based sessions.
 
 .. code-block:: ini
 
       ## types are file, ext:memcached, ext:database, and memory (default).
       #beaker.session.type = file
       #beaker.session.data_dir = %(here)s/data/sessions/data
 
 
 The `table_name` will be automatically created on specified database if it isn't yet existing.
 Database specified in the `beaker.session.sa.url` can be the same that RhodeCode
 uses, or if required it can be a different one. We recommend to use the same database.
 
 
 
 To switch to reds-based user sessions uncomment the following section in
 your :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file.
 
 .. code-block:: ini
 
       ## redis sessions
       beaker.session.type = ext:redis
       beaker.session.url = localhost:6379
 
 
 and make sure you comment out the file based sessions.
 
 .. code-block:: ini
 
       ## types are file, ext:memcached, ext:database, and memory (default).
       #beaker.session.type = file
       #beaker.session.data_dir = %(here)s/data/sessions/data
  No newline at end of file

docs/admin/tuning-vcs-memory-cache.rst

@@ -1,8 +1,8 b''
 .. _adjust-vcs-mem-cache:
 
-Adjusting VCS Memory Cache
---------------------------
+VCSServer Memory Cache
+----------------------
 
 The VCS Server mamory cache can be adjusted to work best with the resources
 available to your |RCE| instance. If you find that memory resources are under
 pressure, see the :ref:`vcs-server-maintain` section for details.