merge stable

development.ini

@@ -1,519 +1,520 b''
 ###################################################################################
 ###################################################################################
 ## Kallithea config file generated with kallithea-cli                            ##
 ##                                                                               ##
 ## The %(here)s variable will generally be replaced with the parent directory of ##
 ## this file. Other use of % must be escaped as %% .                             ##
 ###################################################################################
 ###################################################################################
 
 [DEFAULT]
 
 ################################################################################
 ## Email settings                                                             ##
 ##                                                                            ##
 ## Refer to the documentation ("Email settings") for more details.            ##
 ##                                                                            ##
 ## It is recommended to use a valid sender address that passes access         ##
 ## validation and spam filtering in mail servers.                             ##
 ################################################################################
 
 ## 'From' header for application emails. You can optionally add a name.
 ## Default:
 #app_email_from = Kallithea
 ## Examples:
 #app_email_from = Kallithea <[e-mail]>
 #app_email_from = [e-mail]
 
 ## Subject prefix for application emails.
 ## A space between this prefix and the real subject is automatically added.
 ## Default:
 #email_prefix =
 ## Example:
 #email_prefix = [Kallithea]
 
 ## Recipients for error emails and fallback recipients of application mails.
 ## Multiple addresses can be specified, comma-separated.
 ## Only addresses are allowed, do not add any name part.
 ## Default:
 #email_to =
 ## Examples:
 #email_to = [e-mail]
 #email_to = [e-mail],[e-mail]
 email_to =
 
 ## 'From' header for error emails. You can optionally add a name.
 ## Default: (none)
 ## Examples:
 #error_email_from = Kallithea Errors <[e-mail]>
 #error_email_from = [e-mail]
 error_email_from =
 
 ## SMTP server settings
 ## If specifying credentials, make sure to use secure connections.
 ## Default: Send unencrypted unauthenticated mails to the specified smtp_server.
 ## For "SSL", use smtp_use_ssl = true and smtp_port = 465.
 ## For "STARTTLS", use smtp_use_tls = true and smtp_port = 587.
 smtp_server =
 smtp_username =
 smtp_password =
 smtp_port =
 smtp_use_ssl = false
 smtp_use_tls = false
 
 ## Entry point for 'gearbox serve'
 [server:main]
 #host = 127.0.0.1
 host = 0.0.0.0
 port = 5000
 
 ## Gearbox serve uses the Waitress web server ##
 use = egg:waitress#main
 ## avoid multi threading
 threads = 1
 ## allow push of repos bigger than the default of 1 GB
 max_request_body_size = 107374182400
 ## use poll instead of select, fixes fd limits, may not work on old
 ## windows systems.
 #asyncore_use_poll = True
 
 ## middleware for hosting the WSGI application under a URL prefix
 #[filter:proxy-prefix]
 #use = egg:PasteDeploy#prefix
 #prefix = /<your-prefix>
 
 [app:main]
 use = egg:kallithea
 ## enable proxy prefix middleware
 #filter-with = proxy-prefix
 
 full_stack = true
 static_files = true
 
 ## Internationalization (see setup documentation for details)
 ## By default, the languages requested by the browser are used if available, with English as default.
 ## Set i18n.enabled=false to disable automatic language choice.
 #i18n.enabled = true
 ## To Force a language, set i18n.enabled=false and specify the language in i18n.lang.
 ## Valid values are the names of subdirectories in kallithea/i18n with a LC_MESSAGES/kallithea.mo
 #i18n.lang = en
 
 cache_dir = %(here)s/data
 index_dir = %(here)s/data/index
 
 ## uncomment and set this path to use archive download cache
 archive_cache_dir = %(here)s/tarballcache
 
 ## change this to unique ID for security
 #app_instance_uuid = VERY-SECRET
 app_instance_uuid = development-not-secret
 
 ## cut off limit for large diffs (size in bytes)
 cut_off_limit = 256000
 
 ## force https in Kallithea, fixes https redirects, assumes it's always https
 force_https = false
 
 ## use Strict-Transport-Security headers
 use_htsts = false
 
 ## number of commits stats will parse on each iteration
 commit_parse_limit = 25
 
 ## Path to Python executable to be used for git hooks.
 ## This value will be written inside the git hook scripts as the text
 ## after '#!' (shebang). When empty or not defined, the value of
 ## 'sys.executable' at the time of installation of the git hooks is
 ## used, which is correct in many cases but for example not when using uwsgi.
 ## If you change this setting, you should reinstall the Git hooks via
 ## Admin > Settings > Remap and Rescan.
 #git_hook_interpreter = /srv/kallithea/venv/bin/python3
 
 ## path to git executable
 git_path = git
 
 ## git rev filter option, --all is the default filter, if you need to
 ## hide all refs in changelog switch this to --branches --tags
 #git_rev_filter = --branches --tags
 
 ## RSS feed options
 rss_cut_off_limit = 256000
 rss_items_per_page = 10
 rss_include_diff = false
 
 ## options for showing and identifying changesets
 show_sha_length = 12
 show_revision_number = false
 
 ## Canonical URL to use when creating full URLs in UI and texts.
 ## Useful when the site is available under different names or protocols.
 ## Defaults to what is provided in the WSGI environment.
 #canonical_url = https://kallithea.example.com/repos
 
 ## gist URL alias, used to create nicer urls for gist. This should be an
 ## url that does rewrites to _admin/gists/<gistid>.
 ## example: http://gist.example.com/{gistid}. Empty means use the internal
 ## Kallithea url, ie. http[s]://kallithea.example.com/_admin/gists/<gistid>
 gist_alias_url =
 
 ## default encoding used to convert from and to unicode
 ## can be also a comma separated list of encoding in case of mixed encodings
 default_encoding = utf-8
 
 ## Set Mercurial encoding, similar to setting HGENCODING before launching Kallithea
 hgencoding = utf-8
 
 ## issue tracker for Kallithea (leave blank to disable, absent for default)
 #bugtracker = https://bitbucket.org/conservancy/kallithea/issues
 
 ## issue tracking mapping for commit messages, comments, PR descriptions, ...
 ## Refer to the documentation ("Integration with issue trackers") for more details.
 
 ## regular expression to match issue references
 ## This pattern may/should contain parenthesized groups, that can
 ## be referred to in issue_server_link or issue_sub using Python backreferences
 ## (e.g. \1, \2, ...). You can also create named groups with '(?P<groupname>)'.
 ## To require mandatory whitespace before the issue pattern, use:
 ## (?:^|(?<=\s)) before the actual pattern, and for mandatory whitespace
 ## behind the issue pattern, use (?:$|(?=\s)) after the actual pattern.
 
 issue_pat = #(\d+)
 
 ## server url to the issue
 ## This pattern may/should contain backreferences to parenthesized groups in issue_pat.
 ## A backreference can be \1, \2, ... or \g<groupname> if you specified a named group
 ## called 'groupname' in issue_pat.
 ## The special token {repo} is replaced with the full repository name
 ## including repository groups, while {repo_name} is replaced with just
 ## the name of the repository.
 
 issue_server_link = https://issues.example.com/{repo}/issue/\1
 
 ## substitution pattern to use as the link text
 ## If issue_sub is empty, the text matched by issue_pat is retained verbatim
 ## for the link text. Otherwise, the link text is that of issue_sub, with any
 ## backreferences to groups in issue_pat replaced.
 
 issue_sub =
 
 ## issue_pat, issue_server_link and issue_sub can have suffixes to specify
 ## multiple patterns, to other issues server, wiki or others
 ## below an example how to create a wiki pattern
 ## wiki-some-id -> https://wiki.example.com/some-id
 
 #issue_pat_wiki = wiki-(\S+)
 #issue_server_link_wiki = https://wiki.example.com/\1
 #issue_sub_wiki = WIKI-\1
 
 ## alternative return HTTP header for failed authentication. Default HTTP
 ## response is 401 HTTPUnauthorized. Currently Mercurial clients have trouble with
 ## handling that. Set this variable to 403 to return HTTPForbidden
 auth_ret_code =
 
 ## allows to change the repository location in settings page
 allow_repo_location_change = True
 
 ## allows to setup custom hooks in settings page
 allow_custom_hooks_settings = True
 
 ## extra extensions for indexing, space separated and without the leading '.'.
 #index.extensions =
 #    gemfile
 #    lock
 
 ## extra filenames for indexing, space separated
 #index.filenames =
 #    .dockerignore
 #    .editorconfig
 #    INSTALL
 #    CHANGELOG
 
 ####################################
 ##            SSH CONFIG          ##
 ####################################
 
 ## SSH is disabled by default, until an Administrator decides to enable it.
 ssh_enabled = false
 
 ## File where users' SSH keys will be stored *if* ssh_enabled is true.
 #ssh_authorized_keys = /home/kallithea/.ssh/authorized_keys
 
 ## Path to be used in ssh_authorized_keys file to invoke kallithea-cli with ssh-serve.
 #kallithea_cli_path = /srv/kallithea/venv/bin/kallithea-cli
 
 ## Locale to be used in the ssh-serve command.
 ## This is needed because an SSH client may try to use its own locale
 ## settings, which may not be available on the server.
 ## See `locale -a` for valid values on this system.
 #ssh_locale = C.UTF-8
 
 ####################################
 ##         CELERY CONFIG          ##
 ####################################
 
 ## Note: Celery doesn't support Windows.
 use_celery = false
 
 ## Celery config settings from https://docs.celeryproject.org/en/4.4.0/userguide/configuration.html prefixed with 'celery.'.
 
 ## Example: use the message queue on the local virtual host 'kallitheavhost' as the RabbitMQ user 'kallithea':
 celery.broker_url = amqp://kallithea:thepassword@localhost:5672/kallitheavhost
 
 celery.result.backend = db+sqlite:///celery-results.db
 
 #celery.amqp.task.result.expires = 18000
 
 celery.worker_concurrency = 2
 celery.worker_max_tasks_per_child = 1
 
 ## If true, tasks will never be sent to the queue, but executed locally instead.
 celery.task_always_eager = false
 
 ####################################
 ##          BEAKER CACHE          ##
 ####################################
 
 beaker.cache.data_dir = %(here)s/data/cache/data
 beaker.cache.lock_dir = %(here)s/data/cache/lock
 
 beaker.cache.regions = long_term,long_term_file
 
 beaker.cache.long_term.type = memory
 beaker.cache.long_term.expire = 36000
 beaker.cache.long_term.key_length = 256
 
 beaker.cache.long_term_file.type = file
 beaker.cache.long_term_file.expire = 604800
 beaker.cache.long_term_file.key_length = 256
 
 ####################################
 ##        BEAKER SESSION          ##
 ####################################
 
 ## Name of session cookie. Should be unique for a given host and path, even when running
 ## on different ports. Otherwise, cookie sessions will be shared and messed up.
 session.key = kallithea
 ## Sessions should always only be accessible by the browser, not directly by JavaScript.
 session.httponly = true
 ## Session lifetime. 2592000 seconds is 30 days.
 session.timeout = 2592000
 
 ## Server secret used with HMAC to ensure integrity of cookies.
 #session.secret = VERY-SECRET
 session.secret = development-not-secret
 ## Further, encrypt the data with AES.
 #session.encrypt_key = <key_for_encryption>
 #session.validate_key = <validation_key>
 
 ## Type of storage used for the session, current types are
 ## dbm, file, memcached, database, and memory.
 
 ## File system storage of session data. (default)
 #session.type = file
 
 ## Cookie only, store all session data inside the cookie. Requires secure secrets.
 #session.type = cookie
 
 ## Database storage of session data.
 #session.type = ext:database
 #session.sa.url = postgresql://postgres:qwe@localhost/kallithea
 #session.table_name = db_session
 
 ####################################
 ##        ERROR HANDLING          ##
 ####################################
 
 ## Show a nice error page for application HTTP errors and exceptions (default true)
 #errorpage.enabled = true
 
 ## Enable Backlash client-side interactive debugger (default false)
 ## WARNING: *THIS MUST BE false IN PRODUCTION ENVIRONMENTS!!!*
 ## This debug mode will allow all visitors to execute malicious code.
 #debug = false
 debug = true
 
 ## Enable Backlash server-side error reporting (unless debug mode handles it client-side) (default true)
 #trace_errors.enable = true
 ## Errors will be reported by mail if trace_errors.error_email is set.
 
 ## Propagate email settings to ErrorReporter of TurboGears2
 ## You do not normally need to change these lines
 get trace_errors.smtp_server = smtp_server
 get trace_errors.smtp_port = smtp_port
 get trace_errors.from_address = error_email_from
 get trace_errors.error_email = email_to
 get trace_errors.smtp_username = smtp_username
 get trace_errors.smtp_password = smtp_password
 get trace_errors.smtp_use_tls = smtp_use_tls
 
 
 ##################################
 ##        LOGVIEW CONFIG        ##
 ##################################
 
 logview.sqlalchemy = #faa
 logview.pylons.templating = #bfb
 logview.pylons.util = #eee
 
 #########################
 ##      DB CONFIG      ##
 #########################
 
 sqlalchemy.url = sqlite:///%(here)s/kallithea.db?timeout=60
 #sqlalchemy.url = postgresql://user:pass@localhost/kallithea
 #sqlalchemy.url = mysql://user:pass@localhost/kallithea?charset=utf8
+## Note: the mysql:// prefix should also be used for MariaDB
 
 sqlalchemy.pool_recycle = 3600
 
 ################################
 ##   ALEMBIC CONFIGURATION    ##
 ################################
 
 [alembic]
 script_location = kallithea:alembic
 
 ################################
 ##   LOGGING CONFIGURATION    ##
 ################################
 
 [loggers]
 keys = root, routes, kallithea, sqlalchemy, tg, gearbox, beaker, templates, whoosh_indexer, werkzeug, backlash
 
 [handlers]
 keys = console, console_color, console_color_sql, null
 
 [formatters]
 keys = generic, color_formatter, color_formatter_sql
 
 #############
 ## LOGGERS ##
 #############
 
 [logger_root]
 level = NOTSET
 #handlers = console
 ## For coloring based on log level:
 handlers = console_color
 
 [logger_routes]
 #level = WARN
 level = DEBUG
 handlers =
 qualname = routes.middleware
 ## "level = DEBUG" logs the route matched and routing variables.
 
 [logger_beaker]
 #level = WARN
 level = DEBUG
 handlers =
 qualname = beaker.container
 
 [logger_templates]
 #level = WARN
 level = INFO
 handlers =
 qualname = pylons.templating
 
 [logger_kallithea]
 #level = WARN
 level = DEBUG
 handlers =
 qualname = kallithea
 
 [logger_tg]
 #level = WARN
 level = DEBUG
 handlers =
 qualname = tg
 
 [logger_gearbox]
 #level = WARN
 level = DEBUG
 handlers =
 qualname = gearbox
 
 [logger_sqlalchemy]
 level = WARN
 handlers =
 qualname = sqlalchemy.engine
 ## For coloring based on log level and pretty printing of SQL:
 #level = INFO
 #handlers = console_color_sql
 #propagate = 0
 
 [logger_whoosh_indexer]
 #level = WARN
 level = DEBUG
 handlers =
 qualname = whoosh_indexer
 
 [logger_werkzeug]
 level = WARN
 handlers =
 qualname = werkzeug
 
 [logger_backlash]
 level = WARN
 handlers =
 qualname = backlash
 
 ##############
 ## HANDLERS ##
 ##############
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
 formatter = generic
 
 [handler_console_color]
 ## ANSI color coding based on log level
 class = StreamHandler
 args = (sys.stderr,)
 formatter = color_formatter
 
 [handler_console_color_sql]
 ## ANSI color coding and pretty printing of SQL statements
 class = StreamHandler
 args = (sys.stderr,)
 formatter = color_formatter_sql
 
 [handler_null]
 class = NullHandler
 args = ()
 
 ################
 ## FORMATTERS ##
 ################
 
 [formatter_generic]
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class = kallithea.lib.colored_formatter.ColorFormatter
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter_sql]
 class = kallithea.lib.colored_formatter.ColorFormatterSql
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 #################
 ## SSH LOGGING ##
 #################
 
 ## The default loggers use 'handler_console' that uses StreamHandler with
 ## destination 'sys.stderr'. In the context of the SSH server process, these log
 ## messages would be sent to the client, which is normally not what you want.
 ## By default, when running ssh-serve, just use NullHandler and disable logging
 ## completely. For other logging options, see:
 ## https://docs.python.org/2/library/logging.handlers.html
 
 [ssh_serve:logger_root]
 level = CRITICAL
 handlers = null
 
 ## Note: If logging is configured with other handlers, they might need similar
 ## muting for ssh-serve too.

docs/overview.rst

@@ -1,216 +1,232 b''
 .. _overview:
 
 =====================
 Installation overview
 =====================
 
 Some overview and some details that can help understanding the options when
 installing Kallithea.
 
 1. **Prepare environment and external dependencies.**
     Kallithea needs:
 
     * A filesystem where the Mercurial and Git repositories can be stored.
     * A database where meta data can be stored.
     * A Python environment where the Kallithea application and its dependencies
       can be installed.
     * A web server that can host the Kallithea web application using the WSGI
       API.
 
 2. **Install Kallithea software.**
     This makes the ``kallithea-cli`` command line tool available.
 
 3. **Create low level configuration file.**
     Use ``kallithea-cli config-create`` to create a ``.ini`` file with database
-    connection info, mail server information, some web server configuration,
-    etc.
+    connection info, mail server information, configuration for the specified
+    web server, etc.
 
 4. **Populate the database.**
     Use ``kallithea-cli db-create`` with the ``.ini`` file to create the
     database schema and insert the most basic information: the location of the
     repository store and an initial local admin user.
 
 5. **Configure the web server.**
     The web server must invoke the WSGI entrypoint for the Kallithea software
     using the ``.ini`` file (and thus the database). This makes the web
     application available so the local admin user can log in and tweak the
     configuration further.
 
 6. **Configure users.**
     The initial admin user can create additional local users, or configure how
     users can be created and authenticated from other user directories.
 
 See the subsequent sections, the separate OS-specific instructions, and
 :ref:`setup` for details on these steps.
 
 
 Python environment
 ------------------
 
 **Kallithea** is written entirely in Python_ and requires Python version
 3.6 or higher.
 
 Given a Python installation, there are different ways of providing the
 environment for running Python applications. Each of them pretty much
 corresponds to a ``site-packages`` directory somewhere where packages can be
 installed.
 
 Kallithea itself can be run from source or be installed, but even when running
 from source, there are some dependencies that must be installed in the Python
 environment used for running Kallithea.
 
 - Packages *could* be installed in Python's ``site-packages`` directory ... but
   that would require running pip_ as root and it would be hard to uninstall or
   upgrade and is probably not a good idea unless using a package manager.
 
 - Packages could also be installed in ``~/.local`` ... but that is probably
   only a good idea if using a dedicated user per application or instance.
 
 - Finally, it can be installed in a virtualenv. That is a very lightweight
   "container" where each Kallithea instance can get its own dedicated and
   self-contained virtual environment.
 
 We recommend using virtualenv for installing Kallithea.
 
 
 Locale environment
 ------------------
 
 In order to ensure a correct functioning of Kallithea with respect to non-ASCII
 characters in user names, file paths, commit messages, etc., it is very
 important that Kallithea is run with a correct `locale` configuration.
 
 On Unix, environment variables like ``LANG`` or ``LC_ALL`` can specify a language (like
 ``en_US``) and encoding (like ``UTF-8``) to use for code points outside the ASCII
 range. The flexibility of supporting multiple encodings of Unicode has the flip
 side of having to specify which encoding to use - especially for Mercurial.
 
 It depends on the OS distribution and system configuration which locales are
 available. For example, some Docker containers based on Debian default to only
 supporting the ``C`` language, while other Linux environments have ``en_US`` but not
 ``C``. The ``locale -a`` command will show which values are available on the
 current system. Regardless of the actual language, you should normally choose a
 locale that has the ``UTF-8`` encoding (note that spellings ``utf8``, ``utf-8``,
 ``UTF8``, ``UTF-8`` are all referring to the same thing)
 
 For technical reasons, the locale configuration **must** be provided in the
 environment in which Kallithea runs - it cannot be specified in the ``.ini`` file.
 How to practically do this depends on the web server that is used and the way it
 is started. For example, gearbox is often started by a normal user, either
 manually or via a script. In this case, the required locale environment
 variables can be provided directly in that user's environment or in the script.
 However, web servers like Apache are often started at boot via an init script or
 service file. Modifying the environment for this case would thus require
 root/administrator privileges. Moreover, that environment would dictate the
 settings for all web services running under that web server, Kallithea being
 just one of them. Specifically in the case of Apache with ``mod_wsgi``, the
 locale can be set for a specific service in its ``WSGIDaemonProcess`` directive,
 using the ``lang`` parameter.
 
 
 Installation methods
 --------------------
 
 Kallithea must be installed on a server. Kallithea is installed in a Python
 environment so it can use packages that are installed there and make itself
 available for other packages.
 
 Two different cases will pretty much cover the options for how it can be
 installed.
 
 - The Kallithea source repository can be cloned and used -- it is kept stable and
   can be used in production. The Kallithea maintainers use the development
   branch in production. The advantage of installation from source and regularly
   updating it is that you take advantage of the most recent improvements. Using
   it directly from a DVCS also means that it is easy to track local customizations.
 
   Running ``pip install -e .`` in the source will use pip to install the
   necessary dependencies in the Python environment and create a
   ``.../site-packages/Kallithea.egg-link`` file there that points at the Kallithea
   source.
 
 - Kallithea can also be installed from ready-made packages using a package manager.
   The official released versions are available on PyPI_ and can be downloaded and
   installed with all dependencies using ``pip install kallithea``.
 
   With this method, Kallithea is installed in the Python environment as any
   other package, usually as a ``.../site-packages/Kallithea-X-py3.8.egg/``
   directory with Python files and everything else that is needed.
 
   (``pip install kallithea`` from a source tree will do pretty much the same
   but build the Kallithea package itself locally instead of downloading it.)
 
 .. note::
-   Kallithea includes front-end code that needs to be processed first.
-   The tool npm_ is used to download external dependencies and orchestrate the
-   processing. The ``npm`` binary must thus be available.
+   Kallithea includes front-end code that needs to be processed to prepare
+   static files that can be served at run time and used on the client side. The
+   tool npm_ is used to download external dependencies and orchestrate the
+   processing. The ``npm`` binary must thus be available at install time but is
+   not used at run time.
 
 
 Web server
 ----------
 
 Kallithea is (primarily) a WSGI_ application that must be run from a web
 server that serves WSGI applications over HTTP.
 
 Kallithea itself is not serving HTTP (or HTTPS); that is the web server's
 responsibility. Kallithea does however need to know its own user facing URL
 (protocol, address, port and path) for each HTTP request. Kallithea will
 usually use its own HTML/cookie based authentication but can also be configured
 to use web server authentication.
 
 There are several web server options:
 
 - Kallithea uses the Gearbox_ tool as command line interface. Gearbox provides
   ``gearbox serve`` as a convenient way to launch a Python WSGI / web server
   from the command line. That is perfect for development and evaluation.
   Actual use in production might have different requirements and need extra
   work to make it manageable as a scalable system service.
 
-  Gearbox comes with its own built-in web server but Kallithea defaults to use
-  Waitress_. Gunicorn_ is also an option. These web servers have different
-  limited feature sets.
+  Gearbox comes with its own built-in web server for development but Kallithea
+  defaults to using Waitress_. Gunicorn_ and Gevent_ are also options. These
+  web servers have different limited feature sets.
 
-  The web server used by ``gearbox`` is configured in the ``.ini`` file passed
-  to it. The entry point for the WSGI application is configured
-  in ``setup.py`` as ``kallithea.config.application:make_app``.
+  The web server used by ``gearbox serve`` is configured in the ``.ini`` file.
+  Create it with ``config-create`` using for example ``http_server=waitress``
+  to get a configuration starting point for your choice of web server.
+
+  (Gearbox will do like ``paste`` and use the WSGI application entry point
+  ``kallithea.config.middleware:make_app`` as specified in ``setup.py``.)
 
 - `Apache httpd`_ can serve WSGI applications directly using mod_wsgi_ and a
   simple Python file with the necessary configuration. This is a good option if
   Apache is an option.
 
-- uWSGI_ is also a full web server with built-in WSGI module.
+- uWSGI_ is also a full web server with built-in WSGI module. Use
+  ``config-create`` with ``http_server=uwsgi`` to get a ``.ini`` file with
+  uWSGI configuration.
 
 - IIS_ can also server WSGI applications directly using isapi-wsgi_.
 
 - A `reverse HTTP proxy <https://en.wikipedia.org/wiki/Reverse_proxy>`_
   can be put in front of another web server which has WSGI support.
   Such a layered setup can be complex but might in some cases be the right
   option, for example to standardize on one internet-facing web server, to add
   encryption or special authentication or for other security reasons, to
   provide caching of static files, or to provide load balancing or fail-over.
   Nginx_, Varnish_ and HAProxy_ are often used for this purpose, often in front
   of a ``gearbox serve`` that somehow is wrapped as a service.
 
 The best option depends on what you are familiar with and the requirements for
 performance and stability. Also, keep in mind that Kallithea mainly is serving
 dynamically generated pages from a relatively slow Python process. Kallithea is
 also often used inside organizations with a limited amount of users and thus no
 continuous hammering from the internet.
 
+.. note::
+   Kallithea, the libraries it uses, and Python itself do in several places use
+   simple caching in memory. Caches and memory are not always released in a way
+   that is suitable for long-running processes. They might appear to be leaking
+   memory. The worker processes should thus regularly be restarted - for
+   example after 1000 requests and/or one hour. This can usually be done by the
+   web server or the tool used for running it as a system service.
+
 
 .. _Python: http://www.python.org/
 .. _Gunicorn: http://gunicorn.org/
+.. _Gevent: http://www.gevent.org/
 .. _Waitress: http://waitress.readthedocs.org/en/latest/
 .. _Gearbox: http://turbogears.readthedocs.io/en/latest/turbogears/gearbox.html
 .. _PyPI: https://pypi.python.org/pypi
 .. _Apache httpd: http://httpd.apache.org/
 .. _mod_wsgi: https://code.google.com/p/modwsgi/
 .. _isapi-wsgi: https://github.com/hexdump42/isapi-wsgi
 .. _uWSGI: https://uwsgi-docs.readthedocs.org/en/latest/
 .. _nginx: http://nginx.org/en/
 .. _iis: http://en.wikipedia.org/wiki/Internet_Information_Services
 .. _pip: http://en.wikipedia.org/wiki/Pip_%28package_manager%29
 .. _WSGI: http://en.wikipedia.org/wiki/Web_Server_Gateway_Interface
 .. _HAProxy: http://www.haproxy.org/
 .. _Varnish: https://www.varnish-cache.org/
 .. _npm: https://www.npmjs.com/

docs/setup.rst

@@ -1,646 +1,648 b''
 .. _setup:
 
 =====
 Setup
 =====
 
 
 Setting up Kallithea
 --------------------
 
 First, you will need to create a Kallithea configuration file. Run the
 following command to do so::
 
     kallithea-cli config-create my.ini
 
 This will create the file ``my.ini`` in the current directory. This
 configuration file contains the various settings for Kallithea, e.g.
 proxy port, email settings, usage of static files, cache, Celery
 settings, and logging. Extra settings can be specified like::
 
     kallithea-cli config-create my.ini host=8.8.8.8 "[handler_console]" formatter=color_formatter
 
 Next, you need to create the databases used by Kallithea. It is recommended to
 use PostgreSQL or SQLite (default). If you choose a database other than the
 default, ensure you properly adjust the database URL in your ``my.ini``
 configuration file to use this other database. Kallithea currently supports
-PostgreSQL, SQLite and MySQL databases. Create the database by running
+PostgreSQL, SQLite and MariaDB/MySQL databases. Create the database by running
 the following command::
 
     kallithea-cli db-create -c my.ini
 
 This will prompt you for a "root" path. This "root" path is the location where
 Kallithea will store all of its repositories on the current machine. After
 entering this "root" path ``db-create`` will also prompt you for a username
 and password for the initial admin account which ``db-create`` sets
 up for you.
 
 The ``db-create`` values can also be given on the command line.
 Example::
 
     kallithea-cli db-create -c my.ini --user=nn --password=secret --email=[e-mail] --repos=/srv/repos
 
 The ``db-create`` command will create all needed tables and an
 admin account. When choosing a root path you can either use a new
 empty location, or a location which already contains existing
 repositories. If you choose a location which contains existing
 repositories Kallithea will add all of the repositories at the chosen
 location to its database.  (Note: make sure you specify the correct
 path to the root).
 
 .. note:: the given path for Mercurial_ repositories **must** be write
           accessible for the application. It's very important since
           the Kallithea web interface will work without write access,
           but when trying to do a push it will fail with permission
           denied errors unless it has write access.
 
-Finally, prepare the front-end by running::
+Finally, the front-end files must be prepared. This requires ``npm`` version 6
+or later, which needs ``node.js`` (version 12 or later). Prepare the front-end
+by running::
 
     kallithea-cli front-end-build
 
 You are now ready to use Kallithea. To run it simply execute::
 
     gearbox serve -c my.ini
 
 - This command runs the Kallithea server. The web app should be available at
   http://127.0.0.1:5000. The IP address and port is configurable via the
   configuration file created in the previous step.
 - Log in to Kallithea using the admin account created when running ``db-create``.
 - The default permissions on each repository is read, and the owner is admin.
   Remember to update these if needed.
 - In the admin panel you can toggle LDAP, anonymous, and permissions
   settings, as well as edit more advanced options on users and
   repositories.
 
 
 Internationalization (i18n support)
 -----------------------------------
 
 The Kallithea web interface is automatically displayed in the user's preferred
 language, as indicated by the browser. Thus, different users may see the
 application in different languages. If the requested language is not available
 (because the translation file for that language does not yet exist or is
 incomplete), English is used.
 
 If you want to disable automatic language detection and instead configure a
 fixed language regardless of user preference, set ``i18n.enabled = false`` and
 specify another language by setting ``i18n.lang`` in the Kallithea
 configuration file.
 
 
 Using Kallithea with SSH
 ------------------------
 
 Kallithea supports repository access via SSH key based authentication.
 This means:
 
 - repository URLs like ``ssh://[e-mail]/name/of/repository``
 
 - all network traffic for both read and write happens over the SSH protocol on
   port 22, without using HTTP/HTTPS nor the Kallithea WSGI application
 
 - encryption and authentication protocols are managed by the system's ``sshd``
   process, with all users using the same Kallithea system user (e.g.
   ``kallithea``) when connecting to the SSH server, but with users' public keys
   in the Kallithea system user's `.ssh/authorized_keys` file granting each user
   sandboxed access to the repositories.
 
 - users and admins can manage SSH public keys in the web UI
 
 - in their SSH client configuration, users can configure how the client should
   control access to their SSH key - without passphrase, with passphrase, and
   optionally with passphrase caching in the local shell session (``ssh-agent``).
   This is standard SSH functionality, not something Kallithea provides or
   interferes with.
 
 - network communication between client and server happens in a bidirectional
   stateful stream, and will in some cases be faster than HTTP/HTTPS with several
   stateless round-trips.
 
 .. note:: At this moment, repository access via SSH has been tested on Unix
     only. Windows users that care about SSH are invited to test it and report
     problems, ideally contributing patches that solve these problems.
 
 Users and admins can upload SSH public keys (e.g. ``.ssh/id_rsa.pub``) through
 the web interface. The server's ``.ssh/authorized_keys`` file is automatically
 maintained with an entry for each SSH key. Each entry will tell ``sshd`` to run
 ``kallithea-cli`` with the ``ssh-serve`` sub-command and the right Kallithea user ID
 when encountering the corresponding SSH key.
 
 To enable SSH repository access, Kallithea must be configured with the path to
 the ``.ssh/authorized_keys`` file for the Kallithea user, and the path to the
 ``kallithea-cli`` command. Put something like this in the ``.ini`` file::
 
     ssh_enabled = true
     ssh_authorized_keys = /home/kallithea/.ssh/authorized_keys
     kallithea_cli_path = /srv/kallithea/venv/bin/kallithea-cli
 
 The SSH service must be running, and the Kallithea user account must be active
 (not necessarily with password access, but public key access must be enabled),
 all file permissions must be set as sshd wants it, and ``authorized_keys`` must
 be writeable by the Kallithea user.
 
 .. note:: The ``authorized_keys`` file will be rewritten from scratch on
     each update. If it already exists with other data, Kallithea will not
     overwrite the existing ``authorized_keys``, and the server process will
     instead throw an exception. The system administrator thus cannot ssh
     directly to the Kallithea user but must use su/sudo from another account.
 
     If ``/home/kallithea/.ssh/`` (the directory of the path specified in the
     ``ssh_authorized_keys`` setting of the ``.ini`` file) does not exist as a
     directory, Kallithea will attempt to create it. If that path exists but is
     *not* a directory, or is not readable-writable-executable by the server
     process, the server process will raise an exception each time it attempts to
     write the ``authorized_keys`` file.
 
 .. note:: It is possible to configure the SSH server to look for authorized
    keys in multiple files, for example reserving ``ssh/authorized_keys`` to be
    used for normal SSH and with Kallithea using
    ``.ssh/authorized_keys_kallithea``. In ``/etc/ssh/sshd_config`` set
    ``AuthorizedKeysFile .ssh/authorized_keys .ssh/authorized_keys_kallithea``
    and restart sshd, and in ``my.ini`` set ``ssh_authorized_keys =
    /home/kallithea/.ssh/authorized_keys_kallithea``. Note that this new
    location will apply to all system users, and that multiple entries for the
    same SSH key will shadow each other.
 
 .. warning:: The handling of SSH access is steered directly by the command
     specified in the ``authorized_keys`` file. There is no interaction with the
     web UI.  Once SSH access is correctly configured and enabled, it will work
     regardless of whether the Kallithea web process is actually running. Hence,
     if you want to perform repository or server maintenance and want to fully
     disable all access to the repositories, disable SSH access by setting
     ``ssh_enabled = false`` in the correct ``.ini`` file (i.e. the ``.ini`` file
     specified in the ``authorized_keys`` file.)
 
 The ``authorized_keys`` file can be updated manually with ``kallithea-cli
 ssh-update-authorized-keys -c my.ini``. This command is not needed in normal
 operation but is for example useful after changing SSH-related settings in the
 ``.ini`` file or renaming that file. (The path to the ``.ini`` file is used in
 the generated ``authorized_keys`` file).
 
 
 Setting up Whoosh full text search
 ----------------------------------
 
 Kallithea provides full text search of repositories using `Whoosh`__.
 
 .. __: https://whoosh.readthedocs.io/en/latest/
 
 For an incremental index build, run::
 
     kallithea-cli index-create -c my.ini
 
 For a full index rebuild, run::
 
     kallithea-cli index-create -c my.ini --full
 
 The ``--repo-location`` option allows the location of the repositories to be overridden;
 usually, the location is retrieved from the Kallithea database.
 
 The ``--index-only`` option can be used to limit the indexed repositories to a comma-separated list::
 
     kallithea-cli index-create -c my.ini --index-only=vcs,kallithea
 
 To keep your index up-to-date it is necessary to do periodic index builds;
 for this, it is recommended to use a crontab entry. Example::
 
     0  3  *  *  *  /path/to/virtualenv/bin/kallithea-cli index-create -c /path/to/kallithea/my.ini
 
 When using incremental mode (the default), Whoosh will check the last
 modification date of each file and add it to be reindexed if a newer file is
 available. The indexing daemon checks for any removed files and removes them
 from index.
 
 If you want to rebuild the index from scratch, you can use the ``-f`` flag as above,
 or in the admin panel you can check the "build from scratch" checkbox.
 
 
 Integration with issue trackers
 -------------------------------
 
 Kallithea provides a simple integration with issue trackers. It's possible
 to define a regular expression that will match an issue ID in commit messages,
 and have that replaced with a URL to the issue.
 
 This is achieved with following three variables in the ini file::
 
     issue_pat = #(\d+)
     issue_server_link = https://issues.example.com/{repo}/issue/\1
     issue_sub =
 
 ``issue_pat`` is the regular expression describing which strings in
 commit messages will be treated as issue references. The expression can/should
 have one or more parenthesized groups that can later be referred to in
 ``issue_server_link`` and ``issue_sub`` (see below). If you prefer, named groups
 can be used instead of simple parenthesized groups.
 
 If the pattern should only match if it is preceded by whitespace, add the
 following string before the actual pattern: ``(?:^|(?<=\s))``.
 If the pattern should only match if it is followed by whitespace, add the
 following string after the actual pattern: ``(?:$|(?=\s))``.
 These expressions use lookbehind and lookahead assertions of the Python regular
 expression module to avoid the whitespace to be part of the actual pattern,
 otherwise the link text will also contain that whitespace.
 
 Matched issue references are replaced with the link specified in
 ``issue_server_link``, in which any backreferences are resolved. Backreferences
 can be ``\1``, ``\2``, ... or for named groups ``\g<groupname>``.
 The special token ``{repo}`` is replaced with the full repository path
 (including repository groups), while token ``{repo_name}`` is replaced with the
 repository name (without repository groups).
 
 The link text is determined by ``issue_sub``, which can be a string containing
 backreferences to the groups specified in ``issue_pat``. If ``issue_sub`` is
 empty, then the text matched by ``issue_pat`` is used verbatim.
 
 The example settings shown above match issues in the format ``#<number>``.
 This will cause the text ``#300`` to be transformed into a link:
 
 .. code-block:: html
 
   <a href="https://issues.example.com/example_repo/issue/300">#300</a>
 
 The following example transforms a text starting with either of 'pullrequest',
 'pull request' or 'PR', followed by an optional space, then a pound character
 (#) and one or more digits, into a link with the text 'PR #' followed by the
 digits::
 
     issue_pat = (pullrequest|pull request|PR) ?#(\d+)
     issue_server_link = https://issues.example.com/\2
     issue_sub = PR #\2
 
 The following example demonstrates how to require whitespace before the issue
 reference in order for it to be recognized, such that the text ``issue#123`` will
 not cause a match, but ``issue #123`` will::
 
     issue_pat = (?:^|(?<=\s))#(\d+)
     issue_server_link = https://issues.example.com/\1
     issue_sub =
 
 If needed, more than one pattern can be specified by appending a unique suffix to
 the variables. For example, also demonstrating the use of named groups::
 
     issue_pat_wiki = wiki-(?P<pagename>\S+)
     issue_server_link_wiki = https://wiki.example.com/\g<pagename>
     issue_sub_wiki = WIKI-\g<pagename>
 
 With these settings, wiki pages can be referenced as wiki-some-id, and every
 such reference will be transformed into:
 
 .. code-block:: html
 
   <a href="https://wiki.example.com/some-id">WIKI-some-id</a>
 
 Refer to the `Python regular expression documentation`_ for more details about
 the supported syntax in ``issue_pat``, ``issue_server_link`` and ``issue_sub``.
 
 
 Hook management
 ---------------
 
 Hooks can be managed in similar way to that used in ``.hgrc`` files.
 To manage hooks, choose *Admin > Settings > Hooks*.
 
 The built-in hooks cannot be modified, though they can be enabled or disabled in the *VCS* section.
 
 To add another custom hook simply fill in the first textbox with
 ``<name>.<hook_type>`` and the second with the hook path. Example hooks
 can be found in ``kallithea.lib.hooks``.
 
 
 Changing default encoding
 -------------------------
 
 By default, Kallithea uses UTF-8 encoding.
 This is configurable as ``default_encoding`` in the .ini file.
 This affects many parts in Kallithea including user names, filenames, and
 encoding of commit messages. In addition Kallithea can detect if the ``chardet``
 library is installed. If ``chardet`` is detected Kallithea will fallback to it
 when there are encode/decode errors.
 
 The Mercurial encoding is configurable as ``hgencoding``. It is similar to
 setting the ``HGENCODING`` environment variable, but will override it.
 
 
 Celery configuration
 --------------------
 
 Kallithea can use the distributed task queue system Celery_ to run tasks like
 cloning repositories or sending emails.
 
 Kallithea will in most setups work perfectly fine out of the box (without
 Celery), executing all tasks in the web server process. Some tasks can however
 take some time to run and it can be better to run such tasks asynchronously in
 a separate process so the web server can focus on serving web requests.
 
 For installation and configuration of Celery, see the `Celery documentation`_.
 Note that Celery requires a message broker service like RabbitMQ_ (recommended)
 or Redis_.
 
 The use of Celery is configured in the Kallithea ini configuration file.
 To enable it, simply set::
 
   use_celery = true
 
 and add or change the ``celery.*`` configuration variables.
 
 Configuration settings are prefixed with 'celery.', so for example setting
 `broker_url` in Celery means setting `celery.broker_url` in the configuration
 file.
 
 To start the Celery process, run::
 
   kallithea-cli celery-run -c my.ini
 
 Extra options to the Celery worker can be passed after ``--`` - see ``-- -h``
 for more info.
 
 .. note::
    Make sure you run this command from the same virtualenv, and with the same
    user that Kallithea runs.
 
 
 HTTPS support
 -------------
 
 Kallithea will by default generate URLs based on the WSGI environment.
 
 Alternatively, you can use some special configuration settings to control
 directly which scheme/protocol Kallithea will use when generating URLs:
 
 - With ``https_fixup = true``, the scheme will be taken from the
   ``X-Url-Scheme``, ``X-Forwarded-Scheme`` or ``X-Forwarded-Proto`` HTTP header
   (default ``http``).
 - With ``force_https = true`` the default will be ``https``.
 - With ``use_htsts = true``, Kallithea will set ``Strict-Transport-Security`` when using https.
 
 .. _nginx_virtual_host:
 
 
 Nginx virtual host example
 --------------------------
 
 Sample config for Nginx using proxy:
 
 .. code-block:: nginx
 
     upstream kallithea {
         server 127.0.0.1:5000;
         # add more instances for load balancing
         #server 127.0.0.1:5001;
         #server 127.0.0.1:5002;
     }
 
     ## gist alias
     server {
        listen          443;
        server_name     gist.example.com;
        access_log      /var/log/nginx/gist.access.log;
        error_log       /var/log/nginx/gist.error.log;
 
        ssl on;
        ssl_certificate     gist.your.kallithea.server.crt;
        ssl_certificate_key gist.your.kallithea.server.key;
 
        ssl_session_timeout 5m;
 
        ssl_protocols SSLv3 TLSv1;
        ssl_ciphers DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:EDH-RSA-DES-CBC3-SHA:AES256-SHA:DES-CBC3-SHA:AES128-SHA:RC4-SHA:RC4-MD5;
        ssl_prefer_server_ciphers on;
 
        rewrite ^/(.+)$ https://kallithea.example.com/_admin/gists/$1;
        rewrite (.*)    https://kallithea.example.com/_admin/gists;
     }
 
     server {
        listen          443;
        server_name     kallithea.example.com
        access_log      /var/log/nginx/kallithea.access.log;
        error_log       /var/log/nginx/kallithea.error.log;
 
        ssl on;
        ssl_certificate     your.kallithea.server.crt;
        ssl_certificate_key your.kallithea.server.key;
 
        ssl_session_timeout 5m;
 
        ssl_protocols SSLv3 TLSv1;
        ssl_ciphers DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:EDH-RSA-DES-CBC3-SHA:AES256-SHA:DES-CBC3-SHA:AES128-SHA:RC4-SHA:RC4-MD5;
        ssl_prefer_server_ciphers on;
 
        ## uncomment root directive if you want to serve static files by nginx
        ## requires static_files = false in .ini file
        #root /srv/kallithea/kallithea/kallithea/public;
        include         /etc/nginx/proxy.conf;
        location / {
             try_files $uri @kallithea;
        }
 
        location @kallithea {
             proxy_pass      http://127.0.0.1:5000;
        }
 
     }
 
 Here's the proxy.conf. It's tuned so it will not timeout on long
 pushes or large pushes::
 
     proxy_redirect              off;
     proxy_set_header            Host $host;
     ## needed for container auth
     #proxy_set_header            REMOTE_USER $remote_user;
     #proxy_set_header            X-Forwarded-User $remote_user;
     proxy_set_header            X-Url-Scheme $scheme;
     proxy_set_header            X-Host $http_host;
     proxy_set_header            X-Real-IP $remote_addr;
     proxy_set_header            X-Forwarded-For $proxy_add_x_forwarded_for;
     proxy_set_header            Proxy-host $proxy_host;
     proxy_buffering             off;
     proxy_connect_timeout       7200;
     proxy_send_timeout          7200;
     proxy_read_timeout          7200;
     proxy_buffers               8 32k;
     client_max_body_size        1024m;
     client_body_buffer_size     128k;
     large_client_header_buffers 8 64k;
 
 .. _apache_virtual_host_reverse_proxy:
 
 
 Apache virtual host reverse proxy example
 -----------------------------------------
 
 Here is a sample configuration file for Apache using proxy:
 
 .. code-block:: apache
 
     <VirtualHost *:80>
             ServerName kallithea.example.com
 
             <Proxy *>
               # For Apache 2.4 and later:
               Require all granted
 
               # For Apache 2.2 and earlier, instead use:
               # Order allow,deny
               # Allow from all
             </Proxy>
 
             #important !
             #Directive to properly generate url (clone url) for Kallithea
             ProxyPreserveHost On
 
             #kallithea instance
             ProxyPass / http://127.0.0.1:5000/
             ProxyPassReverse / http://127.0.0.1:5000/
 
             #to enable https use line below
             #SetEnvIf X-Url-Scheme https HTTPS=1
     </VirtualHost>
 
 Additional tutorial
 http://pylonsbook.com/en/1.1/deployment.html#using-apache-to-proxy-requests-to-pylons
 
 .. _apache_subdirectory:
 
 
 Apache as subdirectory
 ----------------------
 
 Apache subdirectory part:
 
 .. code-block:: apache
 
     <Location /PREFIX >
       ProxyPass http://127.0.0.1:5000/PREFIX
       ProxyPassReverse http://127.0.0.1:5000/PREFIX
       SetEnvIf X-Url-Scheme https HTTPS=1
     </Location>
 
 Besides the regular apache setup you will need to add the following line
 into ``[app:main]`` section of your .ini file::
 
     filter-with = proxy-prefix
 
 Add the following at the end of the .ini file::
 
     [filter:proxy-prefix]
     use = egg:PasteDeploy#prefix
     prefix = /PREFIX
 
 then change ``PREFIX`` into your chosen prefix
 
 .. _apache_mod_wsgi:
 
 
 Apache with mod_wsgi
 --------------------
 
 Alternatively, Kallithea can be set up with Apache under mod_wsgi. For
 that, you'll need to:
 
 - Install mod_wsgi. If using a Debian-based distro, you can install
   the package libapache2-mod-wsgi::
 
     aptitude install libapache2-mod-wsgi
 
 - Enable mod_wsgi::
 
     a2enmod wsgi
 
 - Add global Apache configuration to tell mod_wsgi that Python only will be
   used in the WSGI processes and shouldn't be initialized in the Apache
   processes::
 
     WSGIRestrictEmbedded On
 
 - Create a WSGI dispatch script, like the one below. Make sure you
   check that the paths correctly point to where you installed Kallithea
   and its Python Virtual Environment.
 
   .. code-block:: python
 
       import os
       os.environ['PYTHON_EGG_CACHE'] = '/srv/kallithea/.egg-cache'
 
       # sometimes it's needed to set the current dir
       os.chdir('/srv/kallithea/')
 
       import site
       site.addsitedir("/srv/kallithea/venv/lib/python3.7/site-packages")
 
       ini = '/srv/kallithea/my.ini'
       from logging.config import fileConfig
       fileConfig(ini, {'__file__': ini, 'here': '/srv/kallithea'})
       from paste.deploy import loadapp
       application = loadapp('config:' + ini)
 
   Or using proper virtualenv activation:
 
   .. code-block:: python
 
       activate_this = '/srv/kallithea/venv/bin/activate_this.py'
       execfile(activate_this, dict(__file__=activate_this))
 
       import os
       os.environ['HOME'] = '/srv/kallithea'
 
       ini = '/srv/kallithea/kallithea.ini'
       from logging.config import fileConfig
       fileConfig(ini, {'__file__': ini, 'here': '/srv/kallithea'})
       from paste.deploy import loadapp
       application = loadapp('config:' + ini)
 
 - Add the necessary ``WSGI*`` directives to the Apache Virtual Host configuration
   file, like in the example below. Notice that the WSGI dispatch script created
   above is referred to with the ``WSGIScriptAlias`` directive.
   The default locale settings Apache provides for web services are often not
   adequate, with `C` as the default language and `ASCII` as the encoding.
   Instead, use the ``lang`` parameter of ``WSGIDaemonProcess`` to specify a
   suitable locale. See also the :ref:`overview` section and the
   `WSGIDaemonProcess documentation`_.
 
   Apache will by default run as a special Apache user, on Linux systems
   usually ``www-data`` or ``apache``. If you need to have the repositories
   directory owned by a different user, use the user and group options to
   WSGIDaemonProcess to set the name of the user and group.
 
   Once again, check that all paths are correctly specified.
 
   .. code-block:: apache
 
       WSGIDaemonProcess kallithea processes=5 threads=1 maximum-requests=100 \
           python-home=/srv/kallithea/venv lang=C.UTF-8
       WSGIProcessGroup kallithea
       WSGIScriptAlias / /srv/kallithea/dispatch.wsgi
       WSGIPassAuthorization On
 
   Or if using a dispatcher WSGI script with proper virtualenv activation:
 
   .. code-block:: apache
 
       WSGIDaemonProcess kallithea processes=5 threads=1 maximum-requests=100 lang=en_US.utf8
       WSGIProcessGroup kallithea
       WSGIScriptAlias / /srv/kallithea/dispatch.wsgi
       WSGIPassAuthorization On
 
 
 Other configuration files
 -------------------------
 
 A number of `example init.d scripts`__ can be found in
 the ``init.d`` directory of the Kallithea source.
 
 .. __: https://kallithea-scm.org/repos/kallithea/files/tip/init.d/ .
 
 
 .. _python: http://www.python.org/
 .. _Python regular expression documentation: https://docs.python.org/2/library/re.html
 .. _Mercurial: https://www.mercurial-scm.org/
 .. _Celery: http://celeryproject.org/
 .. _Celery documentation: http://docs.celeryproject.org/en/latest/getting-started/index.html
 .. _RabbitMQ: http://www.rabbitmq.com/
 .. _Redis: http://redis.io/
 .. _mercurial-server: http://www.lshift.net/mercurial-server.html
 .. _PublishingRepositories: https://www.mercurial-scm.org/wiki/PublishingRepositories
 .. _WSGIDaemonProcess documentation: https://modwsgi.readthedocs.io/en/develop/configuration-directives/WSGIDaemonProcess.html

docs/upgrade.rst

@@ -1,243 +1,243 b''
 .. _upgrade:
 
 ===================
 Upgrading Kallithea
 ===================
 
 This describes the process for upgrading Kallithea, independently of the
 Kallithea installation method.
 
 .. note::
     If you are upgrading from a RhodeCode installation, you must first
     install Kallithea 0.3.2 and follow the instructions in the 0.3.2
     README to perform a one-time conversion of the database from
     RhodeCode to Kallithea, before upgrading to the latest version
     of Kallithea.
 
 
 1. Stop the Kallithea web application
 -------------------------------------
 
 This step depends entirely on the web server software used to serve
 Kallithea, but in any case, Kallithea should not be running during
 the upgrade.
 
 .. note::
     If you're using Celery, make sure you stop all instances during the
     upgrade.
 
 
 2. Create a backup of both database and configuration
 -----------------------------------------------------
 
 You are of course strongly recommended to make backups regularly, but it
 is *especially* important to make a full database and configuration
 backup before performing a Kallithea upgrade.
 
 Back up your configuration
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Make a copy of your Kallithea configuration (``.ini``) file.
 
 If you are using :ref:`rcextensions <customization>`, you should also
 make a copy of the entire ``rcextensions`` directory.
 
 Back up your database
 ^^^^^^^^^^^^^^^^^^^^^
 
 If using SQLite, simply make a copy of the Kallithea database (``.db``)
 file.
 
 If using PostgreSQL, please consult the documentation for the ``pg_dump``
 utility.
 
-If using MySQL, please consult the documentation for the ``mysqldump``
+If using MariaDB/MySQL, please consult the documentation for the ``mysqldump``
 utility.
 
 Look for ``sqlalchemy.url`` in your configuration file to determine
 database type, settings, location, etc. If you were running Kallithea 0.3.x or
 older, this was ``sqlalchemy.db1.url``.
 
 
 3. Activate or recreate the Kallithea virtual environment (if any)
 ------------------------------------------------------------------
 
 .. note::
     If you did not install Kallithea in a virtual environment, skip this step.
 
 For major upgrades, e.g. from 0.3.x to 0.4.x, it is recommended to create a new
 virtual environment, rather than reusing the old. For minor upgrades, e.g.
 within the 0.4.x range, this is not really necessary (but equally fine).
 
 To create a new virtual environment, please refer to the appropriate
 installation page for details. After creating and activating the new virtual
 environment, proceed with the rest of the upgrade process starting from the next
 section.
 
 To reuse the same virtual environment, first activate it, then verify that you
 are using the correct environment by running::
 
     pip freeze
 
 This will list all packages installed in the current environment. If
 Kallithea isn't listed, deactivate the environment and then activate the correct
 one, or recreate a new environment. See the appropriate installation page for
 details.
 
 
 4. Install new version of Kallithea
 -----------------------------------
 
 Please refer to the instructions for the installation method you
 originally used to install Kallithea.
 
 If you originally installed using pip, it is as simple as::
 
     pip install --upgrade kallithea
 
 If you originally installed from version control, assuming you did not make
 private changes (in which case you should adapt the instructions accordingly)::
 
     cd my-kallithea-clone
     hg parent   # make a note of the original revision
     hg pull
     hg update
     hg parent   # make a note of the new revision
     pip install --upgrade -e .
 
 .. _upgrade_config:
 
 
 5. Upgrade your configuration
 -----------------------------
 
 Run the following command to create a new configuration (``.ini``) file::
 
     kallithea-cli config-create new.ini
 
 Then compare it with your old config file and copy over the required
 configuration values from the old to the new file.
 
 .. note::
     Please always make sure your ``.ini`` files are up to date. Errors
     can often be caused by missing parameters added in new versions.
 
 .. _upgrade_db:
 
 
 6. Upgrade your database
 ------------------------
 
 .. note::
     If you are *downgrading* Kallithea, you should perform the database
     migration step *before* installing the older version. (That is,
     always perform migrations using the most recent of the two versions
     you're migrating between.)
 
 First, run the following command to see your current database version::
 
     alembic -c new.ini current
 
 Typical output will be something like "9358dc3d6828 (head)", which is
 the current Alembic database "revision ID". Write down the entire output
 for troubleshooting purposes.
 
 The output will be empty if you're upgrading from Kallithea 0.3.x or
 older. That's expected. If you get an error that the config file was not
 found or has no ``[alembic]`` section, see the next section.
 
 Next, if you are performing an *upgrade*: Run the following command to
 upgrade your database to the current Kallithea version::
 
     alembic -c new.ini upgrade head
 
 If you are performing a *downgrade*: Run the following command to
 downgrade your database to the given version::
 
     alembic -c new.ini downgrade 0.4
 
 Alembic will show the necessary migrations (if any) as it executes them.
 If no "ERROR" is displayed, the command was successful.
 
 Should an error occur, the database may be "stranded" half-way
 through the migration, and you should restore it from backup.
 
 Enabling old Kallithea config files for Alembic use
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Kallithea configuration files created before the introduction of Alembic
 (i.e. predating Kallithea 0.4) need to be updated for use with Alembic.
 Without this, Alembic will fail with an error like this::
 
     FAILED: No config file 'my.ini' found, or file has no '[alembic]' section
 
 .. note::
     If you followed this upgrade guide correctly, you will have created a
     new configuration file in section :ref:`Upgrading your configuration
     <upgrade_config>`. When calling Alembic, make
     sure to use this new config file. In this case, you should not get any
     errors and the below manual steps should not be needed.
 
 If Alembic complains specifically about a missing ``alembic.ini``, it is
 likely because you did not specify a config file using the ``-c`` option.
 On the other hand, if the mentioned config file actually exists, you
 need to append the following lines to it::
 
     [alembic]
     script_location = kallithea:alembic
 
 Your config file should now work with Alembic.
 
 
 7. Prepare the front-end
 ------------------------
 
 Starting with Kallithea 0.4, external front-end dependencies are no longer
 shipped but need to be downloaded and/or generated at installation time. Run the
 following command::
 
     kallithea-cli front-end-build
 
 
 8. Rebuild the Whoosh full-text index
 -------------------------------------
 
 It is recommended that you rebuild the Whoosh index after upgrading since
 new Whoosh versions can introduce incompatible index changes.
 
 
 9. Start the Kallithea web application
 --------------------------------------
 
 This step once again depends entirely on the web server software used to
 serve Kallithea.
 
 If you were running Kallithea 0.3.x or older and were using ``paster serve
 my.ini`` before, then the corresponding command in Kallithea 0.4 and later is::
 
     gearbox serve -c new.ini
 
 Before starting the new version of Kallithea, you may find it helpful to
 clear out your log file so that new errors are readily apparent.
 
 .. note::
     If you're using Celery, make sure you restart all instances of it after
     upgrade.
 
 
 10. Update Git repository hooks
 -------------------------------
 
 It is possible that an upgrade involves changes to the Git hooks installed by
 Kallithea. As these hooks are created inside the repositories on the server
 filesystem, they are not updated automatically when upgrading Kallithea itself.
 
 To update the hooks of your Git repositories:
 
 * Go to *Admin > Settings > Remap and Rescan*
 * Select the checkbox *Install Git hooks*
 * Click the button *Rescan repositories*
 
 .. note::
     Kallithea does not use hooks on Mercurial repositories. This step is thus
     not necessary if you only have Mercurial repositories.

docs/usage/performance.rst

@@ -1,127 +1,127 b''
 .. _performance:
 
 ================================
 Optimizing Kallithea performance
 ================================
 
 When serving a large amount of big repositories, Kallithea can start performing
 slower than expected. Because of the demanding nature of handling large amounts
 of data from version control systems, here are some tips on how to get the best
 performance.
 
 
 Fast storage
 ------------
 
 Kallithea is often I/O bound, and hence a fast disk (SSD/SAN) and plenty of RAM
 is usually more important than a fast CPU.
 
 
 Caching
 -------
 
 Tweak beaker cache settings in the ini file. The actual effect of that is
 questionable.
 
 .. note::
 
     Beaker has no upper bound on cache size and will never drop any caches. For
     memory cache, the only option is to regularly restart the worker process.
     For file cache, it must be cleaned manually, as described in the `Beaker
     documentation <https://beaker.readthedocs.io/en/latest/sessions.html#removing-expired-old-sessions>`_::
 
         find data/cache -type f -mtime +30 -print -exec rm {} \;
 
 
 Database
 --------
 
 SQLite is a good option when having a small load on the system. But due to
 locking issues with SQLite, it is not recommended to use it for larger
 deployments.
 
-Switching to MySQL or PostgreSQL will result in an immediate performance
+Switching to PostgreSQL or MariaDB/MySQL will result in an immediate performance
 increase. A tool like SQLAlchemyGrate_ can be used for migrating to another
 database platform.
 
 
 Horizontal scaling
 ------------------
 
 Scaling horizontally means running several Kallithea instances and let them
 share the load. That can give huge performance benefits when dealing with large
 amounts of traffic (many users, CI servers, etc.). Kallithea can be scaled
 horizontally on one (recommended) or multiple machines.
 
 It is generally possible to run WSGI applications multithreaded, so that
 several HTTP requests are served from the same Python process at once. That can
 in principle give better utilization of internal caches and less process
 overhead.
 
 One danger of running multithreaded is that program execution becomes much more
 complex; programs must be written to consider all combinations of events and
 problems might depend on timing and be impossible to reproduce.
 
 Kallithea can't promise to be thread-safe, just like the embedded Mercurial
 backend doesn't make any strong promises when used as Kallithea uses it.
 Instead, we recommend scaling by using multiple server processes.
 
 Web servers with multiple worker processes (such as ``mod_wsgi`` with the
 ``WSGIDaemonProcess`` ``processes`` parameter) will work out of the box.
 
 In order to scale horizontally on multiple machines, you need to do the
 following:
 
     - Each instance's ``data`` storage needs to be configured to be stored on a
       shared disk storage, preferably together with repositories. This ``data``
       dir contains template caches, sessions, whoosh index and is used for
       task locking (so it is safe across multiple instances). Set the
       ``cache_dir``, ``index_dir``, ``beaker.cache.data_dir``, ``beaker.cache.lock_dir``
       variables in each .ini file to a shared location across Kallithea instances
     - If using several Celery instances,
       the message broker should be common to all of them (e.g.,  one
       shared RabbitMQ server)
     - Load balance using round robin or IP hash, recommended is writing LB rules
       that will separate regular user traffic from automated processes like CI
       servers or build bots.
 
 
 Serve static files directly from the web server
 -----------------------------------------------
 
 With the default ``static_files`` ini setting, the Kallithea WSGI application
 will take care of serving the static files from ``kallithea/public/`` at the
 root of the application URL.
 
 The actual serving of the static files is very fast and unlikely to be a
 problem in a Kallithea setup - the responses generated by Kallithea from
 database and repository content will take significantly more time and
 resources.
 
 To serve static files from the web server, use something like this Apache config
 snippet::
 
         Alias /images/ /srv/kallithea/kallithea/kallithea/public/images/
         Alias /css/ /srv/kallithea/kallithea/kallithea/public/css/
         Alias /js/ /srv/kallithea/kallithea/kallithea/public/js/
         Alias /codemirror/ /srv/kallithea/kallithea/kallithea/public/codemirror/
         Alias /fontello/ /srv/kallithea/kallithea/kallithea/public/fontello/
 
 Then disable serving of static files in the ``.ini`` ``app:main`` section::
 
         static_files = false
 
 If using Kallithea installed as a package, you should be able to find the files
 under ``site-packages/kallithea``, either in your Python installation or in your
 virtualenv. When upgrading, make sure to update the web server configuration
 too if necessary.
 
 It might also be possible to improve performance by configuring the web server
 to compress responses (served from static files or generated by Kallithea) when
 serving them. That might also imply buffering of responses - that is more
 likely to be a problem; large responses (clones or pulls) will have to be fully
 processed and spooled to disk or memory before the client will see any
 response. See the documentation for your web server.
 
 
 .. _SQLAlchemyGrate: https://github.com/shazow/sqlalchemygrate

kallithea/lib/paster_commands/template.ini.mako

@@ -1,617 +1,618 b''
 ## -*- coding: utf-8 -*-
 <%text>##</%text>#################################################################################
 <%text>##</%text>#################################################################################
 <%text>##</%text> Kallithea config file generated with kallithea-cli ${'%-27s' % version       }##
 <%text>##</%text>                                                                               ##
 <%text>##</%text> The %(here)s variable will generally be replaced with the parent directory of ##
 <%text>##</%text> this file. Other use of % must be escaped as %% .                             ##
 <%text>##</%text>#################################################################################
 <%text>##</%text>#################################################################################
 
 [DEFAULT]
 
 <%text>##</%text>##############################################################################
 <%text>##</%text> Email settings                                                             ##
 <%text>##</%text>                                                                            ##
 <%text>##</%text> Refer to the documentation ("Email settings") for more details.            ##
 <%text>##</%text>                                                                            ##
 <%text>##</%text> It is recommended to use a valid sender address that passes access         ##
 <%text>##</%text> validation and spam filtering in mail servers.                             ##
 <%text>##</%text>##############################################################################
 
 <%text>##</%text> 'From' header for application emails. You can optionally add a name.
 <%text>##</%text> Default:
 #app_email_from = Kallithea
 <%text>##</%text> Examples:
 #app_email_from = Kallithea <[e-mail]>
 #app_email_from = [e-mail]
 
 <%text>##</%text> Subject prefix for application emails.
 <%text>##</%text> A space between this prefix and the real subject is automatically added.
 <%text>##</%text> Default:
 #email_prefix =
 <%text>##</%text> Example:
 #email_prefix = [Kallithea]
 
 <%text>##</%text> Recipients for error emails and fallback recipients of application mails.
 <%text>##</%text> Multiple addresses can be specified, comma-separated.
 <%text>##</%text> Only addresses are allowed, do not add any name part.
 <%text>##</%text> Default:
 #email_to =
 <%text>##</%text> Examples:
 #email_to = [e-mail]
 #email_to = [e-mail],[e-mail]
 email_to =
 
 <%text>##</%text> 'From' header for error emails. You can optionally add a name.
 <%text>##</%text> Default: (none)
 <%text>##</%text> Examples:
 #error_email_from = Kallithea Errors <[e-mail]>
 #error_email_from = [e-mail]
 error_email_from =
 
 <%text>##</%text> SMTP server settings
 <%text>##</%text> If specifying credentials, make sure to use secure connections.
 <%text>##</%text> Default: Send unencrypted unauthenticated mails to the specified smtp_server.
 <%text>##</%text> For "SSL", use smtp_use_ssl = true and smtp_port = 465.
 <%text>##</%text> For "STARTTLS", use smtp_use_tls = true and smtp_port = 587.
 smtp_server =
 smtp_username =
 smtp_password =
 smtp_port =
 smtp_use_ssl = false
 smtp_use_tls = false
 
 %if http_server != 'uwsgi':
 <%text>##</%text> Entry point for 'gearbox serve'
 [server:main]
 host = ${host}
 port = ${port}
 
 %if http_server == 'gearbox':
 <%text>##</%text> Gearbox serve uses the built-in development web server ##
 use = egg:gearbox#wsgiref
 <%text>##</%text> nr of worker threads to spawn
 threadpool_workers = 1
 <%text>##</%text> max request before thread respawn
 threadpool_max_requests = 100
 <%text>##</%text> option to use threads of process
 use_threadpool = true
 
 %elif http_server == 'gevent':
 <%text>##</%text> Gearbox serve uses the gevent web server ##
 use = egg:gearbox#gevent
 
 %elif http_server == 'waitress':
 <%text>##</%text> Gearbox serve uses the Waitress web server ##
 use = egg:waitress#main
 <%text>##</%text> avoid multi threading
 threads = 1
 <%text>##</%text> allow push of repos bigger than the default of 1 GB
 max_request_body_size = 107374182400
 <%text>##</%text> use poll instead of select, fixes fd limits, may not work on old
 <%text>##</%text> windows systems.
 #asyncore_use_poll = True
 
 %elif http_server == 'gunicorn':
 <%text>##</%text> Gearbox serve uses the Gunicorn web server ##
 use = egg:gunicorn#main
 <%text>##</%text> number of process workers. You must set `instance_id = *` when this option
 <%text>##</%text> is set to more than one worker
 workers = 4
 <%text>##</%text> process name
 proc_name = kallithea
 <%text>##</%text> type of worker class, one of sync, eventlet, gevent, tornado
 <%text>##</%text> recommended for bigger setup is using of of other than sync one
 worker_class = sync
 max_requests = 1000
 <%text>##</%text> amount of time a worker can handle request before it gets killed and
 <%text>##</%text> restarted
 timeout = 3600
 
 %endif
 %else:
 <%text>##</%text> UWSGI ##
 [uwsgi]
 <%text>##</%text> Note: this section is parsed by the uWSGI .ini parser when run as:
 <%text>##</%text> uwsgi --venv /srv/kallithea/venv --ini-paste-logged my.ini
 <%text>##</%text> Note: in uWSGI 2.0.18 or older, pastescript needs to be installed to
 <%text>##</%text> get correct application logging. In later versions this is not necessary.
 <%text>##</%text> pip install pastescript
 
 <%text>##</%text> HTTP Basics:
 http-socket = ${host}:${port}
 buffer-size = 65535                    ; Mercurial will use huge GET headers for discovery
 
 <%text>##</%text> Scaling:
 master = true                          ; Use separate master and worker processes
 auto-procname = true                   ; Name worker processes accordingly
 lazy = true                            ; App *must* be loaded in workers - db connections can't be shared
 workers = 4                            ; On demand scaling up to this many worker processes
 cheaper = 1                            ; Initial and on demand scaling down to this many worker processes
 max-requests = 1000                    ; Graceful reload of worker processes to avoid leaks
 
 <%text>##</%text> Tweak defaults:
 strict = true                          ; Fail on unknown config directives
 enable-threads = true                  ; Enable Python threads (not threaded workers)
 vacuum = true                          ; Delete sockets during shutdown
 single-interpreter = true
 die-on-term = true                     ; Shutdown when receiving SIGTERM (default is respawn)
 need-app = true                        ; Exit early if no app can be loaded.
 reload-on-exception = true             ; Don't assume that the application worker can process more requests after a severe error
 
 %endif
 <%text>##</%text> middleware for hosting the WSGI application under a URL prefix
 #[filter:proxy-prefix]
 #use = egg:PasteDeploy#prefix
 #prefix = /<your-prefix>
 
 [app:main]
 use = egg:kallithea
 <%text>##</%text> enable proxy prefix middleware
 #filter-with = proxy-prefix
 
 full_stack = true
 static_files = true
 
 <%text>##</%text> Internationalization (see setup documentation for details)
 <%text>##</%text> By default, the languages requested by the browser are used if available, with English as default.
 <%text>##</%text> Set i18n.enabled=false to disable automatic language choice.
 #i18n.enabled = true
 <%text>##</%text> To Force a language, set i18n.enabled=false and specify the language in i18n.lang.
 <%text>##</%text> Valid values are the names of subdirectories in kallithea/i18n with a LC_MESSAGES/kallithea.mo
 #i18n.lang = en
 
 cache_dir = %(here)s/data
 index_dir = %(here)s/data/index
 
 <%text>##</%text> uncomment and set this path to use archive download cache
 archive_cache_dir = %(here)s/tarballcache
 
 <%text>##</%text> change this to unique ID for security
 app_instance_uuid = ${uuid()}
 
 <%text>##</%text> cut off limit for large diffs (size in bytes)
 cut_off_limit = 256000
 
 <%text>##</%text> force https in Kallithea, fixes https redirects, assumes it's always https
 force_https = false
 
 <%text>##</%text> use Strict-Transport-Security headers
 use_htsts = false
 
 <%text>##</%text> number of commits stats will parse on each iteration
 commit_parse_limit = 25
 
 <%text>##</%text> Path to Python executable to be used for git hooks.
 <%text>##</%text> This value will be written inside the git hook scripts as the text
 <%text>##</%text> after '#!' (shebang). When empty or not defined, the value of
 <%text>##</%text> 'sys.executable' at the time of installation of the git hooks is
 <%text>##</%text> used, which is correct in many cases but for example not when using uwsgi.
 <%text>##</%text> If you change this setting, you should reinstall the Git hooks via
 <%text>##</%text> Admin > Settings > Remap and Rescan.
 #git_hook_interpreter = /srv/kallithea/venv/bin/python3
 %if git_hook_interpreter:
 git_hook_interpreter = ${git_hook_interpreter}
 %endif
 
 <%text>##</%text> path to git executable
 git_path = git
 
 <%text>##</%text> git rev filter option, --all is the default filter, if you need to
 <%text>##</%text> hide all refs in changelog switch this to --branches --tags
 #git_rev_filter = --branches --tags
 
 <%text>##</%text> RSS feed options
 rss_cut_off_limit = 256000
 rss_items_per_page = 10
 rss_include_diff = false
 
 <%text>##</%text> options for showing and identifying changesets
 show_sha_length = 12
 show_revision_number = false
 
 <%text>##</%text> Canonical URL to use when creating full URLs in UI and texts.
 <%text>##</%text> Useful when the site is available under different names or protocols.
 <%text>##</%text> Defaults to what is provided in the WSGI environment.
 #canonical_url = https://kallithea.example.com/repos
 
 <%text>##</%text> gist URL alias, used to create nicer urls for gist. This should be an
 <%text>##</%text> url that does rewrites to _admin/gists/<gistid>.
 <%text>##</%text> example: http://gist.example.com/{gistid}. Empty means use the internal
 <%text>##</%text> Kallithea url, ie. http[s]://kallithea.example.com/_admin/gists/<gistid>
 gist_alias_url =
 
 <%text>##</%text> default encoding used to convert from and to unicode
 <%text>##</%text> can be also a comma separated list of encoding in case of mixed encodings
 default_encoding = utf-8
 
 <%text>##</%text> Set Mercurial encoding, similar to setting HGENCODING before launching Kallithea
 hgencoding = utf-8
 
 <%text>##</%text> issue tracker for Kallithea (leave blank to disable, absent for default)
 #bugtracker = https://bitbucket.org/conservancy/kallithea/issues
 
 <%text>##</%text> issue tracking mapping for commit messages, comments, PR descriptions, ...
 <%text>##</%text> Refer to the documentation ("Integration with issue trackers") for more details.
 
 <%text>##</%text> regular expression to match issue references
 <%text>##</%text> This pattern may/should contain parenthesized groups, that can
 <%text>##</%text> be referred to in issue_server_link or issue_sub using Python backreferences
 <%text>##</%text> (e.g. \1, \2, ...). You can also create named groups with '(?P<groupname>)'.
 <%text>##</%text> To require mandatory whitespace before the issue pattern, use:
 <%text>##</%text> (?:^|(?<=\s)) before the actual pattern, and for mandatory whitespace
 <%text>##</%text> behind the issue pattern, use (?:$|(?=\s)) after the actual pattern.
 
 issue_pat = #(\d+)
 
 <%text>##</%text> server url to the issue
 <%text>##</%text> This pattern may/should contain backreferences to parenthesized groups in issue_pat.
 <%text>##</%text> A backreference can be \1, \2, ... or \g<groupname> if you specified a named group
 <%text>##</%text> called 'groupname' in issue_pat.
 <%text>##</%text> The special token {repo} is replaced with the full repository name
 <%text>##</%text> including repository groups, while {repo_name} is replaced with just
 <%text>##</%text> the name of the repository.
 
 issue_server_link = https://issues.example.com/{repo}/issue/\1
 
 <%text>##</%text> substitution pattern to use as the link text
 <%text>##</%text> If issue_sub is empty, the text matched by issue_pat is retained verbatim
 <%text>##</%text> for the link text. Otherwise, the link text is that of issue_sub, with any
 <%text>##</%text> backreferences to groups in issue_pat replaced.
 
 issue_sub =
 
 <%text>##</%text> issue_pat, issue_server_link and issue_sub can have suffixes to specify
 <%text>##</%text> multiple patterns, to other issues server, wiki or others
 <%text>##</%text> below an example how to create a wiki pattern
 <%text>##</%text> wiki-some-id -> https://wiki.example.com/some-id
 
 #issue_pat_wiki = wiki-(\S+)
 #issue_server_link_wiki = https://wiki.example.com/\1
 #issue_sub_wiki = WIKI-\1
 
 <%text>##</%text> alternative return HTTP header for failed authentication. Default HTTP
 <%text>##</%text> response is 401 HTTPUnauthorized. Currently Mercurial clients have trouble with
 <%text>##</%text> handling that. Set this variable to 403 to return HTTPForbidden
 auth_ret_code =
 
 <%text>##</%text> allows to change the repository location in settings page
 allow_repo_location_change = True
 
 <%text>##</%text> allows to setup custom hooks in settings page
 allow_custom_hooks_settings = True
 
 <%text>##</%text> extra extensions for indexing, space separated and without the leading '.'.
 #index.extensions =
 #    gemfile
 #    lock
 
 <%text>##</%text> extra filenames for indexing, space separated
 #index.filenames =
 #    .dockerignore
 #    .editorconfig
 #    INSTALL
 #    CHANGELOG
 
 <%text>##</%text>##################################
 <%text>##</%text>            SSH CONFIG          ##
 <%text>##</%text>##################################
 
 <%text>##</%text> SSH is disabled by default, until an Administrator decides to enable it.
 ssh_enabled = false
 
 <%text>##</%text> File where users' SSH keys will be stored *if* ssh_enabled is true.
 #ssh_authorized_keys = /home/kallithea/.ssh/authorized_keys
 %if user_home_path:
 ssh_authorized_keys = ${user_home_path}/.ssh/authorized_keys
 %endif
 
 <%text>##</%text> Path to be used in ssh_authorized_keys file to invoke kallithea-cli with ssh-serve.
 #kallithea_cli_path = /srv/kallithea/venv/bin/kallithea-cli
 %if kallithea_cli_path:
 kallithea_cli_path = ${kallithea_cli_path}
 %endif
 
 <%text>##</%text> Locale to be used in the ssh-serve command.
 <%text>##</%text> This is needed because an SSH client may try to use its own locale
 <%text>##</%text> settings, which may not be available on the server.
 <%text>##</%text> See `locale -a` for valid values on this system.
 #ssh_locale = C.UTF-8
 %if ssh_locale:
 ssh_locale = ${ssh_locale}
 %endif
 
 <%text>##</%text>##################################
 <%text>##</%text>         CELERY CONFIG          ##
 <%text>##</%text>##################################
 
 <%text>##</%text> Note: Celery doesn't support Windows.
 use_celery = false
 
 <%text>##</%text> Celery config settings from https://docs.celeryproject.org/en/4.4.0/userguide/configuration.html prefixed with 'celery.'.
 
 <%text>##</%text> Example: use the message queue on the local virtual host 'kallitheavhost' as the RabbitMQ user 'kallithea':
 celery.broker_url = amqp://kallithea:thepassword@localhost:5672/kallitheavhost
 
 celery.result.backend = db+sqlite:///celery-results.db
 
 #celery.amqp.task.result.expires = 18000
 
 celery.worker_concurrency = 2
 celery.worker_max_tasks_per_child = 1
 
 <%text>##</%text> If true, tasks will never be sent to the queue, but executed locally instead.
 celery.task_always_eager = false
 
 <%text>##</%text>##################################
 <%text>##</%text>          BEAKER CACHE          ##
 <%text>##</%text>##################################
 
 beaker.cache.data_dir = %(here)s/data/cache/data
 beaker.cache.lock_dir = %(here)s/data/cache/lock
 
 beaker.cache.regions = long_term,long_term_file
 
 beaker.cache.long_term.type = memory
 beaker.cache.long_term.expire = 36000
 beaker.cache.long_term.key_length = 256
 
 beaker.cache.long_term_file.type = file
 beaker.cache.long_term_file.expire = 604800
 beaker.cache.long_term_file.key_length = 256
 
 <%text>##</%text>##################################
 <%text>##</%text>        BEAKER SESSION          ##
 <%text>##</%text>##################################
 
 <%text>##</%text> Name of session cookie. Should be unique for a given host and path, even when running
 <%text>##</%text> on different ports. Otherwise, cookie sessions will be shared and messed up.
 session.key = kallithea
 <%text>##</%text> Sessions should always only be accessible by the browser, not directly by JavaScript.
 session.httponly = true
 <%text>##</%text> Session lifetime. 2592000 seconds is 30 days.
 session.timeout = 2592000
 
 <%text>##</%text> Server secret used with HMAC to ensure integrity of cookies.
 session.secret = ${uuid()}
 <%text>##</%text> Further, encrypt the data with AES.
 #session.encrypt_key = <key_for_encryption>
 #session.validate_key = <validation_key>
 
 <%text>##</%text> Type of storage used for the session, current types are
 <%text>##</%text> dbm, file, memcached, database, and memory.
 
 <%text>##</%text> File system storage of session data. (default)
 #session.type = file
 
 <%text>##</%text> Cookie only, store all session data inside the cookie. Requires secure secrets.
 #session.type = cookie
 
 <%text>##</%text> Database storage of session data.
 #session.type = ext:database
 #session.sa.url = postgresql://postgres:qwe@localhost/kallithea
 #session.table_name = db_session
 
 <%text>##</%text>##################################
 <%text>##</%text>        ERROR HANDLING          ##
 <%text>##</%text>##################################
 
 <%text>##</%text> Show a nice error page for application HTTP errors and exceptions (default true)
 #errorpage.enabled = true
 
 <%text>##</%text> Enable Backlash client-side interactive debugger (default false)
 <%text>##</%text> WARNING: *THIS MUST BE false IN PRODUCTION ENVIRONMENTS!!!*
 <%text>##</%text> This debug mode will allow all visitors to execute malicious code.
 #debug = false
 
 <%text>##</%text> Enable Backlash server-side error reporting (unless debug mode handles it client-side) (default true)
 #trace_errors.enable = true
 <%text>##</%text> Errors will be reported by mail if trace_errors.error_email is set.
 
 <%text>##</%text> Propagate email settings to ErrorReporter of TurboGears2
 <%text>##</%text> You do not normally need to change these lines
 get trace_errors.smtp_server = smtp_server
 get trace_errors.smtp_port = smtp_port
 get trace_errors.from_address = error_email_from
 get trace_errors.error_email = email_to
 get trace_errors.smtp_username = smtp_username
 get trace_errors.smtp_password = smtp_password
 get trace_errors.smtp_use_tls = smtp_use_tls
 
 %if error_aggregation_service == 'sentry':
 <%text>##</%text>##############
 <%text>##</%text>  [sentry]  ##
 <%text>##</%text>##############
 
 <%text>##</%text> sentry is a alternative open source error aggregator
 <%text>##</%text> you must install python packages `sentry` and `raven` to enable
 
 sentry.dsn = YOUR_DNS
 sentry.servers =
 sentry.name =
 sentry.key =
 sentry.public_key =
 sentry.secret_key =
 sentry.project =
 sentry.site =
 sentry.include_paths =
 sentry.exclude_paths =
 
 %endif
 
 <%text>##</%text>################################
 <%text>##</%text>        LOGVIEW CONFIG        ##
 <%text>##</%text>################################
 
 logview.sqlalchemy = #faa
 logview.pylons.templating = #bfb
 logview.pylons.util = #eee
 
 <%text>##</%text>#######################
 <%text>##</%text>      DB CONFIG      ##
 <%text>##</%text>#######################
 
 %if database_engine == 'sqlite':
 sqlalchemy.url = sqlite:///%(here)s/kallithea.db?timeout=60
 %else:
 #sqlalchemy.url = sqlite:///%(here)s/kallithea.db?timeout=60
 %endif
 %if database_engine == 'postgres':
 sqlalchemy.url = postgresql://user:pass@localhost/kallithea
 %else:
 #sqlalchemy.url = postgresql://user:pass@localhost/kallithea
 %endif
 %if database_engine == 'mysql':
 sqlalchemy.url = mysql://user:pass@localhost/kallithea?charset=utf8
 %else:
 #sqlalchemy.url = mysql://user:pass@localhost/kallithea?charset=utf8
 %endif
+<%text>##</%text> Note: the mysql:// prefix should also be used for MariaDB
 
 sqlalchemy.pool_recycle = 3600
 
 <%text>##</%text>##############################
 <%text>##</%text>   ALEMBIC CONFIGURATION    ##
 <%text>##</%text>##############################
 
 [alembic]
 script_location = kallithea:alembic
 
 <%text>##</%text>##############################
 <%text>##</%text>   LOGGING CONFIGURATION    ##
 <%text>##</%text>##############################
 
 [loggers]
 keys = root, routes, kallithea, sqlalchemy, tg, gearbox, beaker, templates, whoosh_indexer, werkzeug, backlash
 
 [handlers]
 keys = console, console_color, console_color_sql, null
 
 [formatters]
 keys = generic, color_formatter, color_formatter_sql
 
 <%text>##</%text>###########
 <%text>##</%text> LOGGERS ##
 <%text>##</%text>###########
 
 [logger_root]
 level = NOTSET
 handlers = console
 <%text>##</%text> For coloring based on log level:
 #handlers = console_color
 
 [logger_routes]
 level = WARN
 handlers =
 qualname = routes.middleware
 <%text>##</%text> "level = DEBUG" logs the route matched and routing variables.
 
 [logger_beaker]
 level = WARN
 handlers =
 qualname = beaker.container
 
 [logger_templates]
 level = WARN
 handlers =
 qualname = pylons.templating
 
 [logger_kallithea]
 level = WARN
 handlers =
 qualname = kallithea
 
 [logger_tg]
 level = WARN
 handlers =
 qualname = tg
 
 [logger_gearbox]
 level = WARN
 handlers =
 qualname = gearbox
 
 [logger_sqlalchemy]
 level = WARN
 handlers =
 qualname = sqlalchemy.engine
 <%text>##</%text> For coloring based on log level and pretty printing of SQL:
 #level = INFO
 #handlers = console_color_sql
 #propagate = 0
 
 [logger_whoosh_indexer]
 level = WARN
 handlers =
 qualname = whoosh_indexer
 
 [logger_werkzeug]
 level = WARN
 handlers =
 qualname = werkzeug
 
 [logger_backlash]
 level = WARN
 handlers =
 qualname = backlash
 
 <%text>##</%text>############
 <%text>##</%text> HANDLERS ##
 <%text>##</%text>############
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
 formatter = generic
 
 [handler_console_color]
 <%text>##</%text> ANSI color coding based on log level
 class = StreamHandler
 args = (sys.stderr,)
 formatter = color_formatter
 
 [handler_console_color_sql]
 <%text>##</%text> ANSI color coding and pretty printing of SQL statements
 class = StreamHandler
 args = (sys.stderr,)
 formatter = color_formatter_sql
 
 [handler_null]
 class = NullHandler
 args = ()
 
 <%text>##</%text>##############
 <%text>##</%text> FORMATTERS ##
 <%text>##</%text>##############
 
 [formatter_generic]
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class = kallithea.lib.colored_formatter.ColorFormatter
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter_sql]
 class = kallithea.lib.colored_formatter.ColorFormatterSql
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 <%text>##</%text>###############
 <%text>##</%text> SSH LOGGING ##
 <%text>##</%text>###############
 
 <%text>##</%text> The default loggers use 'handler_console' that uses StreamHandler with
 <%text>##</%text> destination 'sys.stderr'. In the context of the SSH server process, these log
 <%text>##</%text> messages would be sent to the client, which is normally not what you want.
 <%text>##</%text> By default, when running ssh-serve, just use NullHandler and disable logging
 <%text>##</%text> completely. For other logging options, see:
 <%text>##</%text> https://docs.python.org/2/library/logging.handlers.html
 
 [ssh_serve:logger_root]
 level = CRITICAL
 handlers = null
 
 <%text>##</%text> Note: If logging is configured with other handlers, they might need similar
 <%text>##</%text> muting for ssh-serve too.

kallithea/lib/vcs/backends/hg/changeset.py

@@ -1,392 +1,392 b''
 import os
 import posixpath
 
 import mercurial.archival
 import mercurial.node
 import mercurial.obsutil
 
 from kallithea.lib.vcs.backends.base import BaseChangeset
 from kallithea.lib.vcs.conf import settings
 from kallithea.lib.vcs.exceptions import ChangesetDoesNotExistError, ChangesetError, ImproperArchiveTypeError, NodeDoesNotExistError, VCSError
 from kallithea.lib.vcs.nodes import (AddedFileNodesGenerator, ChangedFileNodesGenerator, DirNode, FileNode, NodeKind, RemovedFileNodesGenerator, RootNode,
                                      SubModuleNode)
 from kallithea.lib.vcs.utils import ascii_bytes, ascii_str, date_fromtimestamp, safe_bytes, safe_str
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 from kallithea.lib.vcs.utils.paths import get_dirs_for_path
 
 
 class MercurialChangeset(BaseChangeset):
     """
     Represents state of the repository at a revision.
     """
 
     def __init__(self, repository, revision):
         self.repository = repository
         assert isinstance(revision, str), repr(revision)
         self._ctx = repository._repo[ascii_bytes(revision)]
         self.raw_id = ascii_str(self._ctx.hex())
         self.revision = self._ctx._rev
         self.nodes = {}
 
     @LazyProperty
     def tags(self):
         return [safe_str(tag) for tag in self._ctx.tags()]
 
     @LazyProperty
     def branch(self):
         return safe_str(self._ctx.branch())
 
     @LazyProperty
     def branches(self):
         return [safe_str(self._ctx.branch())]
 
     @LazyProperty
     def closesbranch(self):
         return self._ctx.closesbranch()
 
     @LazyProperty
     def obsolete(self):
         return self._ctx.obsolete()
 
     @LazyProperty
     def bumped(self):
         return self._ctx.phasedivergent()
 
     @LazyProperty
     def divergent(self):
         return self._ctx.contentdivergent()
 
     @LazyProperty
     def extinct(self):
         return self._ctx.extinct()
 
     @LazyProperty
     def unstable(self):
         return self._ctx.orphan()
 
     @LazyProperty
     def phase(self):
         if(self._ctx.phase() == 1):
             return 'Draft'
         elif(self._ctx.phase() == 2):
             return 'Secret'
         else:
             return ''
 
     @LazyProperty
     def successors(self):
         successors = mercurial.obsutil.successorssets(self._ctx._repo, self._ctx.node(), closest=True)
         # flatten the list here handles both divergent (len > 1)
         # and the usual case (len = 1)
         return [safe_str(mercurial.node.hex(n)[:12]) for sub in successors for n in sub if n != self._ctx.node()]
 
     @LazyProperty
     def predecessors(self):
         return [safe_str(mercurial.node.hex(n)[:12]) for n in mercurial.obsutil.closestpredecessors(self._ctx._repo, self._ctx.node())]
 
     @LazyProperty
     def bookmarks(self):
         return [safe_str(bookmark) for bookmark in self._ctx.bookmarks()]
 
     @LazyProperty
     def message(self):
         return safe_str(self._ctx.description())
 
     @LazyProperty
     def committer(self):
         return safe_str(self.author)
 
     @LazyProperty
     def author(self):
         return safe_str(self._ctx.user())
 
     @LazyProperty
     def date(self):
         return date_fromtimestamp(*self._ctx.date())
 
     @LazyProperty
     def _timestamp(self):
         return self._ctx.date()[0]
 
     @LazyProperty
     def status(self):
         """
         Returns modified, added, removed, deleted files for current changeset
         """
         return self.repository._repo.status(self._ctx.p1().node(),
                                             self._ctx.node())
 
     @LazyProperty
     def _file_paths(self):
         return list(safe_str(f) for f in self._ctx)
 
     @LazyProperty
     def _dir_paths(self):
         p = list(set(get_dirs_for_path(*self._file_paths)))
         p.insert(0, '')
         return p
 
     @LazyProperty
     def _paths(self):
         return self._dir_paths + self._file_paths
 
     @LazyProperty
     def short_id(self):
         return self.raw_id[:12]
 
     @LazyProperty
     def parents(self):
         """
         Returns list of parents changesets.
         """
         return [self.repository.get_changeset(parent.rev())
                 for parent in self._ctx.parents() if parent.rev() >= 0]
 
     @LazyProperty
     def children(self):
         """
         Returns list of children changesets.
         """
         return [self.repository.get_changeset(child.rev())
                 for child in self._ctx.children() if child.rev() >= 0]
 
     def next(self, branch=None):
         if branch and self.branch != branch:
             raise VCSError('Branch option used on changeset not belonging '
                            'to that branch')
 
         cs = self
         while True:
             try:
                 next_ = cs.repository.revisions.index(cs.raw_id) + 1
                 next_rev = cs.repository.revisions[next_]
             except IndexError:
                 raise ChangesetDoesNotExistError
             cs = cs.repository.get_changeset(next_rev)
 
             if not branch or branch == cs.branch:
                 return cs
 
     def prev(self, branch=None):
         if branch and self.branch != branch:
             raise VCSError('Branch option used on changeset not belonging '
                            'to that branch')
 
         cs = self
         while True:
             try:
                 prev_ = cs.repository.revisions.index(cs.raw_id) - 1
                 if prev_ < 0:
                     raise IndexError
                 prev_rev = cs.repository.revisions[prev_]
             except IndexError:
                 raise ChangesetDoesNotExistError
             cs = cs.repository.get_changeset(prev_rev)
 
             if not branch or branch == cs.branch:
                 return cs
 
     def diff(self):
         # Only used to feed diffstat
         return b''.join(self._ctx.diff())
 
     def _get_kind(self, path):
         path = path.rstrip('/')
         if path in self._file_paths:
             return NodeKind.FILE
         elif path in self._dir_paths:
             return NodeKind.DIR
         else:
             raise ChangesetError("Node does not exist at the given path '%s'"
                 % (path))
 
     def _get_filectx(self, path):
         path = path.rstrip('/')
         if self._get_kind(path) != NodeKind.FILE:
             raise ChangesetError("File does not exist for revision %s at "
                 " '%s'" % (self.raw_id, path))
         return self._ctx.filectx(safe_bytes(path))
 
     def _extract_submodules(self):
         """
         returns a dictionary with submodule information from substate file
         of hg repository
         """
         return self._ctx.substate
 
     def get_file_mode(self, path):
         """
         Returns stat mode of the file at the given ``path``.
         """
         fctx = self._get_filectx(path)
         if b'x' in fctx.flags():
             return 0o100755
         else:
             return 0o100644
 
     def get_file_content(self, path):
         """
         Returns content of the file at given ``path``.
         """
         fctx = self._get_filectx(path)
         return fctx.data()
 
     def get_file_size(self, path):
         """
         Returns size of the file at given ``path``.
         """
         fctx = self._get_filectx(path)
         return fctx.size()
 
     def get_file_changeset(self, path):
         """
         Returns last commit of the file at the given ``path``.
         """
         return self.get_file_history(path, limit=1)[0]
 
     def get_file_history(self, path, limit=None):
         """
         Returns history of file as reversed list of ``Changeset`` objects for
         which file at given ``path`` has been modified.
         """
         fctx = self._get_filectx(path)
         hist = []
         cnt = 0
         for cs in reversed([x for x in fctx.filelog()]):
             cnt += 1
             hist.append(mercurial.node.hex(fctx.filectx(cs).node()))
             if limit is not None and cnt == limit:
                 break
 
         return [self.repository.get_changeset(node) for node in hist]
 
     def get_file_annotate(self, path):
         """
         Returns a generator of four element tuples with
             lineno, sha, changeset lazy loader and line
         """
         annotations = self._get_filectx(path).annotate()
         annotation_lines = [(annotateline.fctx, annotateline.text) for annotateline in annotations]
         for i, (fctx, line) in enumerate(annotation_lines):
             sha = ascii_str(fctx.hex())
             yield (i + 1, sha, lambda sha=sha: self.repository.get_changeset(sha), line)
 
     def fill_archive(self, stream=None, kind='tgz', prefix=None,
                      subrepos=False):
         """
         Fills up given stream.
 
         :param stream: file like object.
         :param kind: one of following: ``zip``, ``tgz`` or ``tbz2``.
             Default: ``tgz``.
         :param prefix: name of root directory in archive.
             Default is repository name and changeset's raw_id joined with dash
             (``repo-tip.<KIND>``).
         :param subrepos: include subrepos in this archive.
 
         :raise ImproperArchiveTypeError: If given kind is wrong.
         :raise VcsError: If given stream is None
         """
         allowed_kinds = settings.ARCHIVE_SPECS
         if kind not in allowed_kinds:
             raise ImproperArchiveTypeError('Archive kind not supported use one'
                 'of %s' % ' '.join(allowed_kinds))
 
         if stream is None:
             raise VCSError('You need to pass in a valid stream for filling'
                            ' with archival data')
 
         if prefix is None:
             prefix = '%s-%s' % (self.repository.name, self.short_id)
         elif prefix.startswith('/'):
             raise VCSError("Prefix cannot start with leading slash")
         elif prefix.strip() == '':
             raise VCSError("Prefix cannot be empty")
 
         mercurial.archival.archive(self.repository._repo, stream, ascii_bytes(self.raw_id),
                          safe_bytes(kind), prefix=safe_bytes(prefix), subrepos=subrepos)
 
     def get_nodes(self, path):
         """
         Returns combined ``DirNode`` and ``FileNode`` objects list representing
         state of changeset at the given ``path``. If node at the given ``path``
         is not instance of ``DirNode``, ChangesetError would be raised.
         """
 
         if self._get_kind(path) != NodeKind.DIR:
             raise ChangesetError("Directory does not exist for revision %s at "
                 " '%s'" % (self.revision, path))
         path = path.rstrip('/')
         filenodes = [FileNode(f, changeset=self) for f in self._file_paths
             if os.path.dirname(f) == path]
         dirs = path == '' and '' or [d for d in self._dir_paths
             if d and posixpath.dirname(d) == path]
         dirnodes = [DirNode(d, changeset=self) for d in dirs
             if os.path.dirname(d) == path]
 
         als = self.repository.alias
         for k, vals in self._extract_submodules().items():
             #vals = url,rev,type
             loc = vals[0]
             cs = vals[1]
-            dirnodes.append(SubModuleNode(k, url=loc, changeset=cs,
+            dirnodes.append(SubModuleNode(safe_str(k), url=safe_str(loc), changeset=cs,
                                           alias=als))
         nodes = dirnodes + filenodes
         for node in nodes:
             self.nodes[node.path] = node
         nodes.sort()
         return nodes
 
     def get_node(self, path):
         """
         Returns ``Node`` object from the given ``path``. If there is no node at
         the given ``path``, ``ChangesetError`` would be raised.
         """
         path = path.rstrip('/')
         if path not in self.nodes:
             if path in self._file_paths:
                 node = FileNode(path, changeset=self)
             elif path in self._dir_paths or path in self._dir_paths:
                 if path == '':
                     node = RootNode(changeset=self)
                 else:
                     node = DirNode(path, changeset=self)
             else:
                 raise NodeDoesNotExistError("There is no file nor directory "
                     "at the given path: '%s' at revision %s"
                     % (path, self.short_id))
             # cache node
             self.nodes[path] = node
         return self.nodes[path]
 
     @LazyProperty
     def affected_files(self):
         """
         Gets a fast accessible file changes for given changeset
         """
         return self._ctx.files()
 
     @property
     def added(self):
         """
         Returns list of added ``FileNode`` objects.
         """
         return AddedFileNodesGenerator([safe_str(n) for n in self.status.added], self)
 
     @property
     def changed(self):
         """
         Returns list of modified ``FileNode`` objects.
         """
         return ChangedFileNodesGenerator([safe_str(n) for n in self.status.modified], self)
 
     @property
     def removed(self):
         """
         Returns list of removed ``FileNode`` objects.
         """
         return RemovedFileNodesGenerator([safe_str(n) for n in self.status.removed], self)
 
     @LazyProperty
     def extra(self):
         return self._ctx.extra()

kallithea/lib/vcs/backends/hg/repository.py

@@ -1,618 +1,618 b''
 # -*- coding: utf-8 -*-
 """
     vcs.backends.hg.repository
     ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Mercurial repository implementation.
 
     :created_on: Apr 8, 2010
     :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
 """
 
 import datetime
 import logging
 import os
 import time
 import urllib.error
 import urllib.parse
 import urllib.request
 from collections import OrderedDict
 
 import mercurial.commands
 import mercurial.error
 import mercurial.exchange
 import mercurial.hg
 import mercurial.hgweb
 import mercurial.httppeer
 import mercurial.localrepo
 import mercurial.match
 import mercurial.mdiff
 import mercurial.node
 import mercurial.patch
 import mercurial.scmutil
 import mercurial.sshpeer
 import mercurial.tags
 import mercurial.ui
 import mercurial.url
 import mercurial.util
 
 from kallithea.lib.vcs.backends.base import BaseRepository, CollectionGenerator
 from kallithea.lib.vcs.exceptions import (BranchDoesNotExistError, ChangesetDoesNotExistError, EmptyRepositoryError, RepositoryError, TagAlreadyExistError,
                                           TagDoesNotExistError, VCSError)
 from kallithea.lib.vcs.utils import ascii_str, author_email, author_name, date_fromtimestamp, makedate, safe_bytes, safe_str
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 from kallithea.lib.vcs.utils.paths import abspath
 
 from .changeset import MercurialChangeset
 from .inmemory import MercurialInMemoryChangeset
 from .workdir import MercurialWorkdir
 
 
 log = logging.getLogger(__name__)
 
 
 class MercurialRepository(BaseRepository):
     """
     Mercurial repository backend
     """
     DEFAULT_BRANCH_NAME = 'default'
     scm = 'hg'
 
     def __init__(self, repo_path, create=False, baseui=None, src_url=None,
                  update_after_clone=False):
         """
         Raises RepositoryError if repository could not be find at the given
         ``repo_path``.
 
         :param repo_path: local path of the repository
         :param create=False: if set to True, would try to create repository if
            it does not exist rather than raising exception
         :param baseui=None: user data
         :param src_url=None: would try to clone repository from given location
         :param update_after_clone=False: sets update of working copy after
           making a clone
         """
 
         if not isinstance(repo_path, str):
             raise VCSError('Mercurial backend requires repository path to '
                            'be instance of <str> got %s instead' %
                            type(repo_path))
         self.path = abspath(repo_path)
         self.baseui = baseui or mercurial.ui.ui()
         # We've set path and ui, now we can set _repo itself
         self._repo = self._get_repo(create, src_url, update_after_clone)
 
     @property
     def _empty(self):
         """
         Checks if repository is empty ie. without any changesets
         """
         # TODO: Following raises errors when using InMemoryChangeset...
         # return len(self._repo.changelog) == 0
         return len(self.revisions) == 0
 
     @LazyProperty
     def revisions(self):
         """
         Returns list of revisions' ids, in ascending order.  Being lazy
         attribute allows external tools to inject shas from cache.
         """
         return self._get_all_revisions()
 
     @LazyProperty
     def name(self):
         return os.path.basename(self.path)
 
     @LazyProperty
     def branches(self):
         return self._get_branches()
 
     @LazyProperty
     def closed_branches(self):
         return self._get_branches(normal=False, closed=True)
 
     @LazyProperty
     def allbranches(self):
         """
         List all branches, including closed branches.
         """
         return self._get_branches(closed=True)
 
     def _get_branches(self, normal=True, closed=False):
         """
         Gets branches for this repository
         Returns only not closed branches by default
 
         :param closed: return also closed branches for mercurial
         :param normal: return also normal branches
         """
 
         if self._empty:
             return {}
 
         bt = OrderedDict()
         for bn, _heads, node, isclosed in sorted(self._repo.branchmap().iterbranches()):
             if isclosed:
                 if closed:
                     bt[safe_str(bn)] = ascii_str(mercurial.node.hex(node))
             else:
                 if normal:
                     bt[safe_str(bn)] = ascii_str(mercurial.node.hex(node))
         return bt
 
     @LazyProperty
     def tags(self):
         """
         Gets tags for this repository
         """
         return self._get_tags()
 
     def _get_tags(self):
         if self._empty:
             return {}
 
         return OrderedDict(sorted(
             ((safe_str(n), ascii_str(mercurial.node.hex(h))) for n, h in self._repo.tags().items()),
             reverse=True,
             key=lambda x: x[0],  # sort by name
         ))
 
     def tag(self, name, user, revision=None, message=None, date=None,
             **kwargs):
         """
         Creates and returns a tag for the given ``revision``.
 
         :param name: name for new tag
         :param user: full username, i.e.: "Joe Doe <[e-mail]>"
         :param revision: changeset id for which new tag would be created
         :param message: message of the tag's commit
         :param date: date of tag's commit
 
         :raises TagAlreadyExistError: if tag with same name already exists
         """
         if name in self.tags:
             raise TagAlreadyExistError("Tag %s already exists" % name)
         changeset = self.get_changeset(revision)
         local = kwargs.setdefault('local', False)
 
         if message is None:
             message = "Added tag %s for changeset %s" % (name,
                 changeset.short_id)
 
         if date is None:
             date = safe_bytes(datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S'))
 
         try:
             mercurial.tags.tag(self._repo, safe_bytes(name), changeset._ctx.node(), safe_bytes(message), local, safe_bytes(user), date)
         except mercurial.error.Abort as e:
             raise RepositoryError(e.args[0])
 
         # Reinitialize tags
         self.tags = self._get_tags()
         tag_id = self.tags[name]
 
         return self.get_changeset(revision=tag_id)
 
     def remove_tag(self, name, user, message=None, date=None):
         """
         Removes tag with the given ``name``.
 
         :param name: name of the tag to be removed
         :param user: full username, i.e.: "Joe Doe <[e-mail]>"
         :param message: message of the tag's removal commit
         :param date: date of tag's removal commit
 
         :raises TagDoesNotExistError: if tag with given name does not exists
         """
         if name not in self.tags:
             raise TagDoesNotExistError("Tag %s does not exist" % name)
         if message is None:
             message = "Removed tag %s" % name
         if date is None:
             date = safe_bytes(datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S'))
         local = False
 
         try:
             mercurial.tags.tag(self._repo, safe_bytes(name), mercurial.commands.nullid, safe_bytes(message), local, safe_bytes(user), date)
             self.tags = self._get_tags()
         except mercurial.error.Abort as e:
             raise RepositoryError(e.args[0])
 
     @LazyProperty
     def bookmarks(self):
         """
         Gets bookmarks for this repository
         """
         return self._get_bookmarks()
 
     def _get_bookmarks(self):
         if self._empty:
             return {}
 
         return OrderedDict(sorted(
-            ((safe_str(n), ascii_str(h)) for n, h in self._repo._bookmarks.items()),
+            ((safe_str(n), ascii_str(mercurial.node.hex(h))) for n, h in self._repo._bookmarks.items()),
             reverse=True,
             key=lambda x: x[0],  # sort by name
         ))
 
     def _get_all_revisions(self):
         return [ascii_str(self._repo[x].hex()) for x in self._repo.filtered(b'visible').changelog.revs()]
 
     def get_diff(self, rev1, rev2, path='', ignore_whitespace=False,
                   context=3):
         """
         Returns (git like) *diff*, as plain text. Shows changes introduced by
         ``rev2`` since ``rev1``.
 
         :param rev1: Entry point from which diff is shown. Can be
           ``self.EMPTY_CHANGESET`` - in this case, patch showing all
           the changes since empty state of the repository until ``rev2``
         :param rev2: Until which revision changes should be shown.
         :param ignore_whitespace: If set to ``True``, would not show whitespace
           changes. Defaults to ``False``.
         :param context: How many lines before/after changed lines should be
           shown. Defaults to ``3``. If negative value is passed-in, it will be
           set to ``0`` instead.
         """
 
         # Negative context values make no sense, and will result in
         # errors. Ensure this does not happen.
         if context < 0:
             context = 0
 
         if hasattr(rev1, 'raw_id'):
             rev1 = getattr(rev1, 'raw_id')
 
         if hasattr(rev2, 'raw_id'):
             rev2 = getattr(rev2, 'raw_id')
 
         # Check if given revisions are present at repository (may raise
         # ChangesetDoesNotExistError)
         if rev1 != self.EMPTY_CHANGESET:
             self.get_changeset(rev1)
         self.get_changeset(rev2)
         if path:
             file_filter = mercurial.match.exact([safe_bytes(path)])
         else:
             file_filter = None
 
         return b''.join(mercurial.patch.diff(self._repo, rev1, rev2, match=file_filter,
                           opts=mercurial.mdiff.diffopts(git=True,
                                         showfunc=True,
                                         ignorews=ignore_whitespace,
                                         context=context)))
 
     @classmethod
     def _check_url(cls, url, repoui=None):
         """
         Function will check given url and try to verify if it's a valid
         link. Sometimes it may happened that mercurial will issue basic
         auth request that can cause whole API to hang when used from python
         or other external calls.
 
         On failures it'll raise urllib2.HTTPError, exception is also thrown
         when the return code is non 200
         """
         # check first if it's not an local url
         url = safe_bytes(url)
         if os.path.isdir(url) or url.startswith(b'file:'):
             return True
 
         if url.startswith(b'ssh:'):
             # in case of invalid uri or authentication issues, sshpeer will
             # throw an exception.
             mercurial.sshpeer.instance(repoui or mercurial.ui.ui(), url, False).lookup(b'tip')
             return True
 
         url_prefix = None
         if b'+' in url[:url.find(b'://')]:
             url_prefix, url = url.split(b'+', 1)
 
         handlers = []
         url_obj = mercurial.util.url(url)
         test_uri, authinfo = url_obj.authinfo()
         url_obj.passwd = b'*****'
         cleaned_uri = str(url_obj)
 
         if authinfo:
             # create a password manager
             passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
             passmgr.add_password(*authinfo)
 
             handlers.extend((mercurial.url.httpbasicauthhandler(passmgr),
                              mercurial.url.httpdigestauthhandler(passmgr)))
 
         o = urllib.request.build_opener(*handlers)
         o.addheaders = [('Content-Type', 'application/mercurial-0.1'),
                         ('Accept', 'application/mercurial-0.1')]
 
         req = urllib.request.Request(
             "%s?%s" % (
                 test_uri,
                 urllib.parse.urlencode({
                     'cmd': 'between',
                     'pairs': "%s-%s" % ('0' * 40, '0' * 40),
                 })
             ))
 
         try:
             resp = o.open(req)
             if resp.code != 200:
                 raise Exception('Return Code is not 200')
         except Exception as e:
             # means it cannot be cloned
             raise urllib.error.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
 
         if not url_prefix: # skip svn+http://... (and git+... too)
             # now check if it's a proper hg repo
             try:
                 mercurial.httppeer.instance(repoui or mercurial.ui.ui(), url, False).lookup(b'tip')
             except Exception as e:
                 raise urllib.error.URLError(
                     "url [%s] does not look like an hg repo org_exc: %s"
                     % (cleaned_uri, e))
 
         return True
 
     def _get_repo(self, create, src_url=None, update_after_clone=False):
         """
         Function will check for mercurial repository in given path and return
         a localrepo object. If there is no repository in that path it will
         raise an exception unless ``create`` parameter is set to True - in
         that case repository would be created and returned.
         If ``src_url`` is given, would try to clone repository from the
         location at given clone_point. Additionally it'll make update to
         working copy accordingly to ``update_after_clone`` flag
         """
         try:
             if src_url:
                 url = safe_bytes(self._get_url(src_url))
                 opts = {}
                 if not update_after_clone:
                     opts.update({'noupdate': True})
                 MercurialRepository._check_url(url, self.baseui)
                 mercurial.commands.clone(self.baseui, url, safe_bytes(self.path), **opts)
 
                 # Don't try to create if we've already cloned repo
                 create = False
             return mercurial.localrepo.instance(self.baseui, safe_bytes(self.path), create=create)
         except (mercurial.error.Abort, mercurial.error.RepoError) as err:
             if create:
                 msg = "Cannot create repository at %s. Original error was %s" \
                     % (self.name, err)
             else:
                 msg = "Not valid repository at %s. Original error was %s" \
                     % (self.name, err)
             raise RepositoryError(msg)
 
     @LazyProperty
     def in_memory_changeset(self):
         return MercurialInMemoryChangeset(self)
 
     @LazyProperty
     def description(self):
         _desc = self._repo.ui.config(b'web', b'description', None, untrusted=True)
         return safe_str(_desc or b'unknown')
 
     @LazyProperty
     def contact(self):
         return safe_str(mercurial.hgweb.common.get_contact(self._repo.ui.config)
                             or b'Unknown')
 
     @LazyProperty
     def last_change(self):
         """
         Returns last change made on this repository as datetime object
         """
         return date_fromtimestamp(self._get_mtime(), makedate()[1])
 
     def _get_mtime(self):
         try:
             return time.mktime(self.get_changeset().date.timetuple())
         except RepositoryError:
             # fallback to filesystem
             cl_path = os.path.join(self.path, '.hg', "00changelog.i")
             st_path = os.path.join(self.path, '.hg', "store")
             if os.path.exists(cl_path):
                 return os.stat(cl_path).st_mtime
             else:
                 return os.stat(st_path).st_mtime
 
     def _get_revision(self, revision):
         """
         Given any revision identifier, returns a 40 char string with revision hash.
 
         :param revision: str or int or None
         """
         if self._empty:
             raise EmptyRepositoryError("There are no changesets yet")
 
         if revision in [-1, None]:
             revision = b'tip'
         elif isinstance(revision, str):
             revision = safe_bytes(revision)
 
         try:
             if isinstance(revision, int):
                 return ascii_str(self._repo[revision].hex())
             return ascii_str(mercurial.scmutil.revsymbol(self._repo, revision).hex())
         except (IndexError, ValueError, mercurial.error.RepoLookupError, TypeError):
             msg = "Revision %r does not exist for %s" % (safe_str(revision), self.name)
             raise ChangesetDoesNotExistError(msg)
         except (LookupError, ):
             msg = "Ambiguous identifier `%s` for %s" % (safe_str(revision), self.name)
             raise ChangesetDoesNotExistError(msg)
 
     def get_ref_revision(self, ref_type, ref_name):
         """
         Returns revision number for the given reference.
         """
         if ref_type == 'rev' and not ref_name.strip('0'):
             return self.EMPTY_CHANGESET
         # lookup up the exact node id
         _revset_predicates = {
                 'branch': 'branch',
                 'book': 'bookmark',
                 'tag': 'tag',
                 'rev': 'id',
             }
         # avoid expensive branch(x) iteration over whole repo
         rev_spec = "%%s & %s(%%s)" % _revset_predicates[ref_type]
         try:
             revs = self._repo.revs(rev_spec, ref_name, ref_name)
         except LookupError:
             msg = "Ambiguous identifier %s:%s for %s" % (ref_type, ref_name, self.name)
             raise ChangesetDoesNotExistError(msg)
         except mercurial.error.RepoLookupError:
             msg = "Revision %s:%s does not exist for %s" % (ref_type, ref_name, self.name)
             raise ChangesetDoesNotExistError(msg)
         if revs:
             revision = revs.last()
         else:
             # TODO: just report 'not found'?
             revision = ref_name
 
         return self._get_revision(revision)
 
     def _get_archives(self, archive_name='tip'):
         allowed = self.baseui.configlist(b"web", b"allow_archive",
                                          untrusted=True)
         for name, ext in [(b'zip', '.zip'), (b'gz', '.tar.gz'), (b'bz2', '.tar.bz2')]:
             if name in allowed or self._repo.ui.configbool(b"web",
                                                            b"allow" + name,
                                                            untrusted=True):
                 yield {"type": safe_str(name), "extension": ext, "node": archive_name}
 
     def _get_url(self, url):
         """
         Returns normalized url. If schema is not given, fall back to
         filesystem (``file:///``) schema.
         """
         if url != 'default' and '://' not in url:
             url = "file:" + urllib.request.pathname2url(url)
         return url
 
     def get_changeset(self, revision=None):
         """
         Returns ``MercurialChangeset`` object representing repository's
         changeset at the given ``revision``.
         """
         return MercurialChangeset(repository=self, revision=self._get_revision(revision))
 
     def get_changesets(self, start=None, end=None, start_date=None,
                        end_date=None, branch_name=None, reverse=False, max_revisions=None):
         """
         Returns iterator of ``MercurialChangeset`` objects from start to end
         (both are inclusive)
 
         :param start: None, str, int or mercurial lookup format
         :param end:  None, str, int or mercurial lookup format
         :param start_date:
         :param end_date:
         :param branch_name:
         :param reversed: return changesets in reversed order
         """
         start_raw_id = self._get_revision(start)
         start_pos = None if start is None else self.revisions.index(start_raw_id)
         end_raw_id = self._get_revision(end)
         end_pos = None if end is None else self.revisions.index(end_raw_id)
 
         if start_pos is not None and end_pos is not None and start_pos > end_pos:
             raise RepositoryError("Start revision '%s' cannot be "
                                   "after end revision '%s'" % (start, end))
 
         if branch_name and branch_name not in self.allbranches:
             msg = "Branch %r not found in %s" % (branch_name, self.name)
             raise BranchDoesNotExistError(msg)
         if end_pos is not None:
             end_pos += 1
         # filter branches
         filter_ = []
         if branch_name:
             filter_.append(b'branch("%s")' % safe_bytes(branch_name))
         if start_date:
             filter_.append(b'date(">%s")' % safe_bytes(str(start_date)))
         if end_date:
             filter_.append(b'date("<%s")' % safe_bytes(str(end_date)))
         if filter_ or max_revisions:
             if filter_:
                 revspec = b' and '.join(filter_)
             else:
                 revspec = b'all()'
             if max_revisions:
                 revspec = b'limit(%s, %d)' % (revspec, max_revisions)
             revisions = mercurial.scmutil.revrange(self._repo, [revspec])
         else:
             revisions = self.revisions
 
         # this is very much a hack to turn this into a list; a better solution
         # would be to get rid of this function entirely and use revsets
         revs = list(revisions)[start_pos:end_pos]
         if reverse:
             revs.reverse()
 
         return CollectionGenerator(self, revs)
 
     def pull(self, url):
         """
         Tries to pull changes from external location.
         """
         other = mercurial.hg.peer(self._repo, {}, safe_bytes(self._get_url(url)))
         try:
             mercurial.exchange.pull(self._repo, other, heads=None, force=None)
         except mercurial.error.Abort as err:
             # Propagate error but with vcs's type
             raise RepositoryError(str(err))
 
     @LazyProperty
     def workdir(self):
         """
         Returns ``Workdir`` instance for this repository.
         """
         return MercurialWorkdir(self)
 
     def get_config_value(self, section, name=None, config_file=None):
         """
         Returns configuration value for a given [``section``] and ``name``.
 
         :param section: Section we want to retrieve value from
         :param name: Name of configuration we want to retrieve
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         if config_file is None:
             config_file = []
         elif isinstance(config_file, str):
             config_file = [config_file]
 
         config = self._repo.ui
         if config_file:
             config = mercurial.ui.ui()
             for path in config_file:
                 config.readconfig(safe_bytes(path))
         value = config.config(safe_bytes(section), safe_bytes(name))
         return value if value is None else safe_str(value)
 
     def get_user_name(self, config_file=None):
         """
         Returns user's name from global configuration file.
 
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         username = self.get_config_value('ui', 'username', config_file=config_file)
         if username:
             return author_name(username)
         return None
 
     def get_user_email(self, config_file=None):
         """
         Returns user's email from global configuration file.
 
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         username = self.get_config_value('ui', 'username', config_file=config_file)
         if username:
             return author_email(username)
         return None

kallithea/lib/vcs/nodes.py

@@ -1,606 +1,606 b''
 # -*- coding: utf-8 -*-
 """
     vcs.nodes
     ~~~~~~~~~
 
     Module holding everything related to vcs nodes.
 
     :created_on: Apr 8, 2010
     :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
 """
 
 import functools
 import mimetypes
 import posixpath
 import stat
 
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.lib.vcs.exceptions import NodeError, RemovedFileNodeError
 from kallithea.lib.vcs.utils import safe_bytes, safe_str
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 
 
 class NodeKind:
     SUBMODULE = -1
     DIR = 1
     FILE = 2
 
 
 class NodeState:
     ADDED = 'added'
     CHANGED = 'changed'
     NOT_CHANGED = 'not changed'
     REMOVED = 'removed'
 
 
 class NodeGeneratorBase(object):
     """
     Base class for removed added and changed filenodes, it's a lazy generator
     class that will create filenodes only on iteration or call
 
     The len method doesn't need to create filenodes at all
     """
 
     def __init__(self, current_paths, cs):
         self.cs = cs
         self.current_paths = current_paths
 
     def __getitem__(self, key):
         assert isinstance(key, slice), key
         for p in self.current_paths[key]:
             yield self.cs.get_node(p)
 
     def __len__(self):
         return len(self.current_paths)
 
     def __iter__(self):
         for p in self.current_paths:
             yield self.cs.get_node(p)
 
 
 class AddedFileNodesGenerator(NodeGeneratorBase):
     """
     Class holding Added files for current changeset
     """
     pass
 
 
 class ChangedFileNodesGenerator(NodeGeneratorBase):
     """
     Class holding Changed files for current changeset
     """
     pass
 
 
 class RemovedFileNodesGenerator(NodeGeneratorBase):
     """
     Class holding removed files for current changeset
     """
     def __iter__(self):
         for p in self.current_paths:
             yield RemovedFileNode(path=p)
 
     def __getitem__(self, key):
         assert isinstance(key, slice), key
         for p in self.current_paths[key]:
             yield RemovedFileNode(path=p)
 
 
 @functools.total_ordering
 class Node(object):
     """
     Simplest class representing file or directory on repository.  SCM backends
     should use ``FileNode`` and ``DirNode`` subclasses rather than ``Node``
     directly.
 
     Node's ``path`` cannot start with slash as we operate on *relative* paths
     only. Moreover, every single node is identified by the ``path`` attribute,
     so it cannot end with slash, too. Otherwise, path could lead to mistakes.
     """
 
     def __init__(self, path, kind):
         if path.startswith('/'):
             raise NodeError("Cannot initialize Node objects with slash at "
                             "the beginning as only relative paths are supported")
         self.path = path.rstrip('/')
         if path == '' and kind != NodeKind.DIR:
             raise NodeError("Only DirNode and its subclasses may be "
                             "initialized with empty path")
         self.kind = kind
         #self.dirs, self.files = [], []
         if self.is_root() and not self.is_dir():
             raise NodeError("Root node cannot be FILE kind")
 
     @LazyProperty
     def parent(self):
         parent_path = self.get_parent_path()
         if parent_path:
             if self.changeset:
                 return self.changeset.get_node(parent_path)
             return DirNode(parent_path)
         return None
 
     @LazyProperty
     def name(self):
         """
         Returns name of the node so if its path
         then only last part is returned.
         """
         return self.path.rstrip('/').split('/')[-1]
 
     def __eq__(self, other):
         if type(self) is not type(other):
             return False
         if self.kind != other.kind:
             return False
         if self.path != other.path:
             return False
 
     def __lt__(self, other):
         if self.kind < other.kind:
             return True
         if self.kind > other.kind:
             return False
         if self.path < other.path:
             return True
         if self.path > other.path:
             return False
 
     def __repr__(self):
         return '<%s %r>' % (self.__class__.__name__, self.path)
 
     def get_parent_path(self):
         """
         Returns node's parent path or empty string if node is root.
         """
         if self.is_root():
             return ''
         return posixpath.dirname(self.path.rstrip('/')) + '/'
 
     def is_file(self):
         """
         Returns ``True`` if node's kind is ``NodeKind.FILE``, ``False``
         otherwise.
         """
         return self.kind == NodeKind.FILE
 
     def is_dir(self):
         """
         Returns ``True`` if node's kind is ``NodeKind.DIR``, ``False``
         otherwise.
         """
         return self.kind == NodeKind.DIR
 
     def is_root(self):
         """
         Returns ``True`` if node is a root node and ``False`` otherwise.
         """
         return self.kind == NodeKind.DIR and self.path == ''
 
     def is_submodule(self):
         """
         Returns ``True`` if node's kind is ``NodeKind.SUBMODULE``, ``False``
         otherwise.
         """
         return self.kind == NodeKind.SUBMODULE
 
     @LazyProperty
     def added(self):
         return self.state is NodeState.ADDED
 
     @LazyProperty
     def changed(self):
         return self.state is NodeState.CHANGED
 
     @LazyProperty
     def not_changed(self):
         return self.state is NodeState.NOT_CHANGED
 
     @LazyProperty
     def removed(self):
         return self.state is NodeState.REMOVED
 
 
 class FileNode(Node):
     """
     Class representing file nodes.
 
     :attribute: path: path to the node, relative to repository's root
     :attribute: content: if given arbitrary sets content of the file
     :attribute: changeset: if given, first time content is accessed, callback
     :attribute: mode: octal stat mode for a node. Default is 0100644.
     """
 
     def __init__(self, path, content=None, changeset=None, mode=None):
         """
         Only one of ``content`` and ``changeset`` may be given. Passing both
         would raise ``NodeError`` exception.
 
         :param path: relative path to the node
         :param content: content may be passed to constructor
         :param changeset: if given, will use it to lazily fetch content
         :param mode: octal representation of ST_MODE (i.e. 0100644)
         """
 
         if content and changeset:
             raise NodeError("Cannot use both content and changeset")
         super(FileNode, self).__init__(path, kind=NodeKind.FILE)
         self.changeset = changeset
         if not isinstance(content, bytes) and content is not None:
             # File content is one thing that inherently must be bytes ... but
             # VCS module tries to be "user friendly" and support unicode ...
             content = safe_bytes(content)
         self._content = content
         self._mode = mode or 0o100644
 
     def __eq__(self, other):
         eq = super(FileNode, self).__eq__(other)
         if eq is not None:
             return eq
         return self.content == other.content
 
     def __lt__(self, other):
         lt = super(FileNode, self).__lt__(other)
         if lt is not None:
             return lt
         return self.content < other.content
 
     @LazyProperty
     def mode(self):
         """
         Returns lazily mode of the FileNode. If ``changeset`` is not set, would
         use value given at initialization or 0100644 (default).
         """
         if self.changeset:
             mode = self.changeset.get_file_mode(self.path)
         else:
             mode = self._mode
         return mode
 
     @property
     def content(self):
         """
         Returns lazily byte content of the FileNode.
         """
         if self.changeset:
             content = self.changeset.get_file_content(self.path)
         else:
             content = self._content
         return content
 
     @LazyProperty
     def size(self):
         if self.changeset:
             return self.changeset.get_file_size(self.path)
         raise NodeError("Cannot retrieve size of the file without related "
             "changeset attribute")
 
     @LazyProperty
     def message(self):
         if self.changeset:
             return self.last_changeset.message
         raise NodeError("Cannot retrieve message of the file without related "
             "changeset attribute")
 
     @LazyProperty
     def last_changeset(self):
         if self.changeset:
             return self.changeset.get_file_changeset(self.path)
         raise NodeError("Cannot retrieve last changeset of the file without "
             "related changeset attribute")
 
     def get_mimetype(self):
         """
         Mimetype is calculated based on the file's content.
         """
 
         mtype, encoding = mimetypes.guess_type(self.name)
 
         if mtype is None:
             if self.is_binary:
                 mtype = 'application/octet-stream'
                 encoding = None
             else:
                 mtype = 'text/plain'
                 encoding = None
 
                 # try with pygments
                 from pygments import lexers
                 try:
                     mt = lexers.get_lexer_for_filename(self.name).mimetypes
                 except lexers.ClassNotFound:
                     mt = None
 
                 if mt:
                     mtype = mt[0]
 
         return mtype, encoding
 
     @LazyProperty
     def mimetype(self):
         """
         Wrapper around full mimetype info. It returns only type of fetched
         mimetype without the encoding part. use get_mimetype function to fetch
         full set of (type,encoding)
         """
         return self.get_mimetype()[0]
 
     @LazyProperty
     def mimetype_main(self):
         return self.mimetype.split('/')[0]
 
     @LazyProperty
     def lexer(self):
         """
         Returns pygment's lexer class. Would try to guess lexer taking file's
         content, name and mimetype.
         """
         from pygments import lexers
         try:
             lexer = lexers.guess_lexer_for_filename(self.name, safe_str(self.content), stripnl=False)
         except lexers.ClassNotFound:
             lexer = lexers.TextLexer(stripnl=False)
         # returns first alias
         return lexer
 
     @LazyProperty
     def lexer_alias(self):
         """
         Returns first alias of the lexer guessed for this file.
         """
         return self.lexer.aliases[0]
 
     @LazyProperty
     def history(self):
         """
         Returns a list of changeset for this file in which the file was changed
         """
         if self.changeset is None:
             raise NodeError('Unable to get changeset for this FileNode')
         return self.changeset.get_file_history(self.path)
 
     @LazyProperty
     def annotate(self):
         """
         Returns a list of three element tuples with lineno,changeset and line
         """
         if self.changeset is None:
             raise NodeError('Unable to get changeset for this FileNode')
         return self.changeset.get_file_annotate(self.path)
 
     @LazyProperty
     def state(self):
         if not self.changeset:
             raise NodeError("Cannot check state of the node if it's not "
                 "linked with changeset")
         elif self.path in (node.path for node in self.changeset.added):
             return NodeState.ADDED
         elif self.path in (node.path for node in self.changeset.changed):
             return NodeState.CHANGED
         else:
             return NodeState.NOT_CHANGED
 
     @property
     def is_binary(self):
         """
         Returns True if file has binary content.
         """
         return b'\0' in self.content
 
     def is_browser_compatible_image(self):
         return self.mimetype in [
             "image/gif",
             "image/jpeg",
             "image/png",
             "image/bmp"
         ]
 
     @LazyProperty
     def extension(self):
         """Returns filenode extension"""
         return self.name.split('.')[-1]
 
     @property
     def is_executable(self):
         """
         Returns ``True`` if file has executable flag turned on.
         """
         return bool(self.mode & stat.S_IXUSR)
 
     def __repr__(self):
         return '<%s %r @ %s>' % (self.__class__.__name__, self.path,
                                  getattr(self.changeset, 'short_id', ''))
 
 
 class RemovedFileNode(FileNode):
     """
     Dummy FileNode class - trying to access any public attribute except path,
     name, kind or state (or methods/attributes checking those two) would raise
     RemovedFileNodeError.
     """
     ALLOWED_ATTRIBUTES = [
         'name', 'path', 'state', 'is_root', 'is_file', 'is_dir', 'kind',
         'added', 'changed', 'not_changed', 'removed'
     ]
 
     def __init__(self, path):
         """
         :param path: relative path to the node
         """
         super(RemovedFileNode, self).__init__(path=path)
 
     def __getattribute__(self, attr):
         if attr.startswith('_') or attr in RemovedFileNode.ALLOWED_ATTRIBUTES:
             return super(RemovedFileNode, self).__getattribute__(attr)
         raise RemovedFileNodeError("Cannot access attribute %s on "
             "RemovedFileNode" % attr)
 
     @LazyProperty
     def state(self):
         return NodeState.REMOVED
 
 
 class DirNode(Node):
     """
     DirNode stores list of files and directories within this node.
     Nodes may be used standalone but within repository context they
     lazily fetch data within same repository's changeset.
     """
 
     def __init__(self, path, nodes=(), changeset=None):
         """
         Only one of ``nodes`` and ``changeset`` may be given. Passing both
         would raise ``NodeError`` exception.
 
         :param path: relative path to the node
         :param nodes: content may be passed to constructor
         :param changeset: if given, will use it to lazily fetch content
         :param size: always 0 for ``DirNode``
         """
         if nodes and changeset:
             raise NodeError("Cannot use both nodes and changeset")
         super(DirNode, self).__init__(path, NodeKind.DIR)
         self.changeset = changeset
         self._nodes = nodes
 
     def __eq__(self, other):
         eq = super(DirNode, self).__eq__(other)
         if eq is not None:
             return eq
         # check without entering each dir
         self_nodes_paths = list(sorted(n.path for n in self.nodes))
         other_nodes_paths = list(sorted(n.path for n in self.nodes))
         return self_nodes_paths == other_nodes_paths
 
     def __lt__(self, other):
         lt = super(DirNode, self).__lt__(other)
         if lt is not None:
             return lt
         # check without entering each dir
         self_nodes_paths = list(sorted(n.path for n in self.nodes))
         other_nodes_paths = list(sorted(n.path for n in self.nodes))
         return self_nodes_paths < other_nodes_paths
 
     @LazyProperty
     def nodes(self):
         if self.changeset:
             nodes = self.changeset.get_nodes(self.path)
         else:
             nodes = self._nodes
         self._nodes_dict = dict((node.path, node) for node in nodes)
         return sorted(nodes)
 
     @LazyProperty
     def files(self):
         return sorted((node for node in self.nodes if node.is_file()))
 
     @LazyProperty
     def dirs(self):
         return sorted((node for node in self.nodes if node.is_dir()))
 
     def __iter__(self):
         for node in self.nodes:
             yield node
 
     def get_node(self, path):
         """
         Returns node from within this particular ``DirNode``, so it is now
         allowed to fetch, i.e. node located at 'docs/api/index.rst' from node
         'docs'. In order to access deeper nodes one must fetch nodes between
         them first - this would work::
 
            docs = root.get_node('docs')
            docs.get_node('api').get_node('index.rst')
 
         :param: path - relative to the current node
 
         .. note::
            To access lazily (as in example above) node have to be initialized
            with related changeset object - without it node is out of
            context and may know nothing about anything else than nearest
            (located at same level) nodes.
         """
         try:
             path = path.rstrip('/')
             if path == '':
                 raise NodeError("Cannot retrieve node without path")
             self.nodes  # access nodes first in order to set _nodes_dict
             paths = path.split('/')
             if len(paths) == 1:
                 if not self.is_root():
                     path = '/'.join((self.path, paths[0]))
                 else:
                     path = paths[0]
                 return self._nodes_dict[path]
             elif len(paths) > 1:
                 if self.changeset is None:
                     raise NodeError("Cannot access deeper "
                                     "nodes without changeset")
                 else:
                     path1, path2 = paths[0], '/'.join(paths[1:])
                     return self.get_node(path1).get_node(path2)
             else:
                 raise KeyError
         except KeyError:
             raise NodeError("Node does not exist at %s" % path)
 
     @LazyProperty
     def state(self):
         raise NodeError("Cannot access state of DirNode")
 
     @LazyProperty
     def size(self):
         size = 0
         for root, dirs, files in self.changeset.walk(self.path):
             for f in files:
                 size += f.size
 
         return size
 
     def __repr__(self):
         return '<%s %r @ %s>' % (self.__class__.__name__, self.path,
                                  getattr(self.changeset, 'short_id', ''))
 
 
 class RootNode(DirNode):
     """
     DirNode being the root node of the repository.
     """
 
     def __init__(self, nodes=(), changeset=None):
         super(RootNode, self).__init__(path='', nodes=nodes,
             changeset=changeset)
 
     def __repr__(self):
         return '<%s>' % self.__class__.__name__
 
 
 class SubModuleNode(Node):
     """
     represents a SubModule of Git or SubRepo of Mercurial
     """
     is_binary = False
     size = 0
 
     def __init__(self, name, url, changeset=None, alias=None):
         # Note: Doesn't call Node.__init__!
         self.path = name.rstrip('/')
         self.kind = NodeKind.SUBMODULE
         self.alias = alias
         # we have to use emptyChangeset here since this can point to svn/git/hg
         # submodules we cannot get from repository
         self.changeset = EmptyChangeset(changeset, alias=alias)
         self.url = url
 
     def __repr__(self):
         return '<%s %r @ %s>' % (self.__class__.__name__, self.path,
                                  getattr(self.changeset, 'short_id', ''))
 
     @LazyProperty
     def name(self):
         """
         Returns name of the node so if its path
         then only last part is returned.
         """
         org = self.path.rstrip('/').rsplit('/', 1)[-1]
-        return '%s @ %s' % (org, self.changeset.short_id)
+        return '%s @ %s' % (org, safe_str(self.changeset.short_id))