imports: always use CONFIG as kallithea.CONFIG

kallithea/controllers/feed.py

@@ -1,134 +1,134 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.controllers.feed
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Feed controller for Kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 23, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 
 import logging
 
 from beaker.cache import cache_region
 from tg import response
 from tg import tmpl_context as c
 from tg.i18n import ugettext as _
 
-from kallithea import CONFIG
+import kallithea
 from kallithea.lib import feeds
 from kallithea.lib import helpers as h
 from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired
 from kallithea.lib.base import BaseRepoController
 from kallithea.lib.diffs import DiffProcessor
 from kallithea.lib.utils2 import asbool, safe_int, safe_str
 
 
 log = logging.getLogger(__name__)
 
 
 class FeedController(BaseRepoController):
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def _before(self, *args, **kwargs):
         super(FeedController, self)._before(*args, **kwargs)
 
     def _get_title(self, cs):
         return h.shorter(cs.message, 160)
 
     def __get_desc(self, cs):
         desc_msg = [(_('%s committed on %s')
                      % (h.person(cs.author), h.fmt_date(cs.date))) + '<br/>']
         # branches, tags, bookmarks
         for branch in cs.branches:
             desc_msg.append('branch: %s<br/>' % branch)
         for book in cs.bookmarks:
             desc_msg.append('bookmark: %s<br/>' % book)
         for tag in cs.tags:
             desc_msg.append('tag: %s<br/>' % tag)
 
         changes = []
-        diff_limit = safe_int(CONFIG.get('rss_cut_off_limit', 32 * 1024))
+        diff_limit = safe_int(kallithea.CONFIG.get('rss_cut_off_limit', 32 * 1024))
         raw_diff = cs.diff()
         diff_processor = DiffProcessor(raw_diff,
                                        diff_limit=diff_limit,
                                        inline_diff=False)
 
         for st in diff_processor.parsed:
             st.update({'added': st['stats']['added'],
                        'removed': st['stats']['deleted']})
             changes.append('\n %(operation)s %(filename)s '
                            '(%(added)s lines added, %(removed)s lines removed)'
                             % st)
         if diff_processor.limited_diff:
             changes = changes + ['\n ' +
                                  _('Changeset was too big and was cut off...')]
 
         # rev link
         _url = h.canonical_url('changeset_home', repo_name=c.db_repo.repo_name,
                    revision=cs.raw_id)
         desc_msg.append('changeset: <a href="%s">%s</a>' % (_url, cs.raw_id[:8]))
 
         desc_msg.append('<pre>')
         desc_msg.append(h.urlify_text(cs.message))
         desc_msg.append('\n')
         desc_msg.extend(changes)
-        if asbool(CONFIG.get('rss_include_diff', False)):
+        if asbool(kallithea.CONFIG.get('rss_include_diff', False)):
             desc_msg.append('\n\n')
             desc_msg.append(safe_str(raw_diff))
         desc_msg.append('</pre>')
         return desc_msg
 
     def _feed(self, repo_name, feeder):
         """Produce a simple feed"""
 
         @cache_region('long_term_file', '_get_feed_from_cache')
         def _get_feed_from_cache(*_cache_keys):  # parameters are not really used - only as caching key
             header = dict(
                 title=_('%s %s feed') % (c.site_name, repo_name),
                 link=h.canonical_url('summary_home', repo_name=repo_name),
                 description=_('Changes on %s repository') % repo_name,
             )
 
-            rss_items_per_page = safe_int(CONFIG.get('rss_items_per_page', 20))
+            rss_items_per_page = safe_int(kallithea.CONFIG.get('rss_items_per_page', 20))
             entries=[]
             for cs in reversed(list(c.db_repo_scm_instance[-rss_items_per_page:])):
                 entries.append(dict(
                     title=self._get_title(cs),
                     link=h.canonical_url('changeset_home', repo_name=repo_name, revision=cs.raw_id),
                     author_email=cs.author_email,
                     author_name=cs.author_name,
                     description=''.join(self.__get_desc(cs)),
                     pubdate=cs.date,
                 ))
             return feeder.render(header, entries)
 
         response.content_type = feeder.content_type
         return _get_feed_from_cache(repo_name, feeder.__name__)
 
     def atom(self, repo_name):
         """Produce a simple atom-1.0 feed"""
         return self._feed(repo_name, feeds.AtomFeed)
 
     def rss(self, repo_name):
         """Produce a simple rss2 feed"""
         return self._feed(repo_name, feeds.RssFeed)

kallithea/controllers/files.py

@@ -1,746 +1,746 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.controllers.files
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Files controller for Kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 21, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 import logging
 import os
 import posixpath
 import shutil
 import tempfile
 import traceback
 from collections import OrderedDict
 
 from tg import request, response
 from tg import tmpl_context as c
 from tg.i18n import ugettext as _
 from webob.exc import HTTPFound, HTTPNotFound
 
+import kallithea
 from kallithea.config.routing import url
 from kallithea.lib import diffs
 from kallithea.lib import helpers as h
 from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired
 from kallithea.lib.base import BaseRepoController, jsonify, render
 from kallithea.lib.exceptions import NonRelativePathError
 from kallithea.lib.utils import action_logger
 from kallithea.lib.utils2 import asbool, convert_line_endings, detect_mode, safe_str
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.lib.vcs.conf import settings
 from kallithea.lib.vcs.exceptions import (ChangesetDoesNotExistError, ChangesetError, EmptyRepositoryError, ImproperArchiveTypeError, NodeAlreadyExistsError,
                                           NodeDoesNotExistError, NodeError, RepositoryError, VCSError)
 from kallithea.lib.vcs.nodes import FileNode
 from kallithea.model import db
 from kallithea.model.repo import RepoModel
 from kallithea.model.scm import ScmModel
 
 
 log = logging.getLogger(__name__)
 
 
 class FilesController(BaseRepoController):
 
     def _before(self, *args, **kwargs):
         super(FilesController, self)._before(*args, **kwargs)
 
     def __get_cs(self, rev, silent_empty=False):
         """
         Safe way to get changeset if error occur it redirects to tip with
         proper message
 
         :param rev: revision to fetch
         :silent_empty: return None if repository is empty
         """
 
         try:
             return c.db_repo_scm_instance.get_changeset(rev)
         except EmptyRepositoryError as e:
             if silent_empty:
                 return None
             url_ = url('files_add_home',
                        repo_name=c.repo_name,
                        revision=0, f_path='', anchor='edit')
             add_new = h.link_to(_('Click here to add new file'), url_, class_="alert-link")
             h.flash(_('There are no files yet.') + ' ' + add_new, category='warning')
             raise HTTPNotFound()
         except (ChangesetDoesNotExistError, LookupError):
             msg = _('Such revision does not exist for this repository')
             h.flash(msg, category='error')
             raise HTTPNotFound()
         except RepositoryError as e:
             h.flash(e, category='error')
             raise HTTPNotFound()
 
     def __get_filenode(self, cs, path):
         """
         Returns file_node or raise HTTP error.
 
         :param cs: given changeset
         :param path: path to lookup
         """
 
         try:
             file_node = cs.get_node(path)
             if file_node.is_dir():
                 raise RepositoryError('given path is a directory')
         except ChangesetDoesNotExistError:
             msg = _('Such revision does not exist for this repository')
             h.flash(msg, category='error')
             raise HTTPNotFound()
         except RepositoryError as e:
             h.flash(e, category='error')
             raise HTTPNotFound()
 
         return file_node
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def index(self, repo_name, revision, f_path, annotate=False):
         # redirect to given revision from form if given
         post_revision = request.POST.get('at_rev', None)
         if post_revision:
             cs = self.__get_cs(post_revision) # FIXME - unused!
 
         c.revision = revision
         c.changeset = self.__get_cs(revision)
         c.branch = request.GET.get('branch', None)
         c.f_path = f_path
         c.annotate = annotate
         cur_rev = c.changeset.revision
         # used in files_source.html:
         c.cut_off_limit = self.cut_off_limit
         c.fulldiff = request.GET.get('fulldiff')
 
         # prev link
         try:
             prev_rev = c.db_repo_scm_instance.get_changeset(cur_rev).prev(c.branch)
             c.url_prev = url('files_home', repo_name=c.repo_name,
                          revision=prev_rev.raw_id, f_path=f_path)
             if c.branch:
                 c.url_prev += '?branch=%s' % c.branch
         except (ChangesetDoesNotExistError, VCSError):
             c.url_prev = '#'
 
         # next link
         try:
             next_rev = c.db_repo_scm_instance.get_changeset(cur_rev).next(c.branch)
             c.url_next = url('files_home', repo_name=c.repo_name,
                      revision=next_rev.raw_id, f_path=f_path)
             if c.branch:
                 c.url_next += '?branch=%s' % c.branch
         except (ChangesetDoesNotExistError, VCSError):
             c.url_next = '#'
 
         # files or dirs
         try:
             c.file = c.changeset.get_node(f_path)
 
             if c.file.is_submodule():
                 raise HTTPFound(location=c.file.url)
             elif c.file.is_file():
                 c.load_full_history = False
                 # determine if we're on branch head
                 _branches = c.db_repo_scm_instance.branches
                 c.on_branch_head = revision in _branches or revision in _branches.values()
                 _hist = []
                 c.file_history = []
                 if c.load_full_history:
                     c.file_history, _hist = self._get_node_history(c.changeset, f_path)
 
                 c.authors = []
                 for a in set([x.author for x in _hist]):
                     c.authors.append((h.email(a), h.person(a)))
             else:
                 c.authors = c.file_history = []
         except RepositoryError as e:
             h.flash(e, category='error')
             raise HTTPNotFound()
 
         if request.environ.get('HTTP_X_PARTIAL_XHR'):
             return render('files/files_ypjax.html')
 
         # TODO: tags and bookmarks?
         c.revision_options = [(c.changeset.raw_id,
                               _('%s at %s') % (b, h.short_id(c.changeset.raw_id))) for b in c.changeset.branches] + \
             [(n, b) for b, n in c.db_repo_scm_instance.branches.items()]
         if c.db_repo_scm_instance.closed_branches:
             prefix = _('(closed)') + ' '
             c.revision_options += [('-', '-')] + \
                 [(n, prefix + b) for b, n in c.db_repo_scm_instance.closed_branches.items()]
 
         return render('files/files.html')
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     @jsonify
     def history(self, repo_name, revision, f_path):
         changeset = self.__get_cs(revision)
         _file = changeset.get_node(f_path)
         if _file.is_file():
             file_history, _hist = self._get_node_history(changeset, f_path)
 
             res = []
             for obj in file_history:
                 res.append({
                     'text': obj[1],
                     'children': [{'id': o[0], 'text': o[1]} for o in obj[0]]
                 })
 
             data = {
                 'more': False,
                 'results': res
             }
             return data
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def authors(self, repo_name, revision, f_path):
         changeset = self.__get_cs(revision)
         _file = changeset.get_node(f_path)
         if _file.is_file():
             file_history, _hist = self._get_node_history(changeset, f_path)
             c.authors = []
             for a in set([x.author for x in _hist]):
                 c.authors.append((h.email(a), h.person(a)))
             return render('files/files_history_box.html')
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def rawfile(self, repo_name, revision, f_path):
         cs = self.__get_cs(revision)
         file_node = self.__get_filenode(cs, f_path)
 
         response.content_disposition = \
             'attachment; filename=%s' % f_path.split(db.URL_SEP)[-1]
 
         response.content_type = file_node.mimetype
         return file_node.content
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def raw(self, repo_name, revision, f_path):
         cs = self.__get_cs(revision)
         file_node = self.__get_filenode(cs, f_path)
 
         raw_mimetype_mapping = {
             # map original mimetype to a mimetype used for "show as raw"
             # you can also provide a content-disposition to override the
             # default "attachment" disposition.
             # orig_type: (new_type, new_dispo)
 
             # show images inline:
             'image/x-icon': ('image/x-icon', 'inline'),
             'image/png': ('image/png', 'inline'),
             'image/gif': ('image/gif', 'inline'),
             'image/jpeg': ('image/jpeg', 'inline'),
             'image/svg+xml': ('image/svg+xml', 'inline'),
         }
 
         mimetype = file_node.mimetype
         try:
             mimetype, dispo = raw_mimetype_mapping[mimetype]
         except KeyError:
             # we don't know anything special about this, handle it safely
             if file_node.is_binary:
                 # do same as download raw for binary files
                 mimetype, dispo = 'application/octet-stream', 'attachment'
             else:
                 # do not just use the original mimetype, but force text/plain,
                 # otherwise it would serve text/html and that might be unsafe.
                 # Note: underlying vcs library fakes text/plain mimetype if the
                 # mimetype can not be determined and it thinks it is not
                 # binary.This might lead to erroneous text display in some
                 # cases, but helps in other cases, like with text files
                 # without extension.
                 mimetype, dispo = 'text/plain', 'inline'
 
         if dispo == 'attachment':
             dispo = 'attachment; filename=%s' % f_path.split(os.sep)[-1]
 
         response.content_disposition = dispo
         response.content_type = mimetype
         return file_node.content
 
     @LoginRequired()
     @HasRepoPermissionLevelDecorator('write')
     def delete(self, repo_name, revision, f_path):
         repo = c.db_repo
         # check if revision is a branch identifier- basically we cannot
         # create multiple heads via file editing
         _branches = repo.scm_instance.branches
         # check if revision is a branch name or branch hash
         if revision not in _branches and revision not in _branches.values():
             h.flash(_('You can only delete files with revision '
                       'being a valid branch'), category='warning')
             raise HTTPFound(location=h.url('files_home',
                                   repo_name=repo_name, revision='tip',
                                   f_path=f_path))
 
         r_post = request.POST
 
         c.cs = self.__get_cs(revision)
         c.file = self.__get_filenode(c.cs, f_path)
 
         c.default_message = _('Deleted file %s via Kallithea') % (f_path)
         c.f_path = f_path
         node_path = f_path
         author = request.authuser.full_contact
 
         if r_post:
             message = r_post.get('message') or c.default_message
 
             try:
                 nodes = {
                     node_path: {
                         'content': ''
                     }
                 }
                 self.scm_model.delete_nodes(
                     user=request.authuser.user_id,
                     ip_addr=request.ip_addr,
                     repo=c.db_repo,
                     message=message,
                     nodes=nodes,
                     parent_cs=c.cs,
                     author=author,
                 )
 
                 h.flash(_('Successfully deleted file %s') % f_path,
                         category='success')
             except Exception:
                 log.error(traceback.format_exc())
                 h.flash(_('Error occurred during commit'), category='error')
             raise HTTPFound(location=url('changeset_home',
                                 repo_name=c.repo_name, revision='tip'))
 
         return render('files/files_delete.html')
 
     @LoginRequired()
     @HasRepoPermissionLevelDecorator('write')
     def edit(self, repo_name, revision, f_path):
         repo = c.db_repo
         # check if revision is a branch identifier- basically we cannot
         # create multiple heads via file editing
         _branches = repo.scm_instance.branches
         # check if revision is a branch name or branch hash
         if revision not in _branches and revision not in _branches.values():
             h.flash(_('You can only edit files with revision '
                       'being a valid branch'), category='warning')
             raise HTTPFound(location=h.url('files_home',
                                   repo_name=repo_name, revision='tip',
                                   f_path=f_path))
 
         r_post = request.POST
 
         c.cs = self.__get_cs(revision)
         c.file = self.__get_filenode(c.cs, f_path)
 
         if c.file.is_binary:
             raise HTTPFound(location=url('files_home', repo_name=c.repo_name,
                             revision=c.cs.raw_id, f_path=f_path))
         c.default_message = _('Edited file %s via Kallithea') % (f_path)
         c.f_path = f_path
 
         if r_post:
             old_content = safe_str(c.file.content)
             sl = old_content.splitlines(1)
             first_line = sl[0] if sl else ''
             # modes:  0 - Unix, 1 - Mac, 2 - DOS
             mode = detect_mode(first_line, 0)
             content = convert_line_endings(r_post.get('content', ''), mode)
 
             message = r_post.get('message') or c.default_message
             author = request.authuser.full_contact
 
             if content == old_content:
                 h.flash(_('No changes'), category='warning')
                 raise HTTPFound(location=url('changeset_home', repo_name=c.repo_name,
                                     revision='tip'))
             try:
                 self.scm_model.commit_change(repo=c.db_repo_scm_instance,
                                              repo_name=repo_name, cs=c.cs,
                                              user=request.authuser.user_id,
                                              ip_addr=request.ip_addr,
                                              author=author, message=message,
                                              content=content, f_path=f_path)
                 h.flash(_('Successfully committed to %s') % f_path,
                         category='success')
             except Exception:
                 log.error(traceback.format_exc())
                 h.flash(_('Error occurred during commit'), category='error')
             raise HTTPFound(location=url('changeset_home',
                                 repo_name=c.repo_name, revision='tip'))
 
         return render('files/files_edit.html')
 
     @LoginRequired()
     @HasRepoPermissionLevelDecorator('write')
     def add(self, repo_name, revision, f_path):
 
         repo = c.db_repo
         r_post = request.POST
         c.cs = self.__get_cs(revision, silent_empty=True)
         if c.cs is None:
             c.cs = EmptyChangeset(alias=c.db_repo_scm_instance.alias)
         c.default_message = (_('Added file via Kallithea'))
         c.f_path = f_path
 
         if r_post:
             unix_mode = 0
             content = convert_line_endings(r_post.get('content', ''), unix_mode)
 
             message = r_post.get('message') or c.default_message
             filename = r_post.get('filename')
             location = r_post.get('location', '')
             file_obj = r_post.get('upload_file', None)
 
             if file_obj is not None and hasattr(file_obj, 'filename'):
                 filename = file_obj.filename
                 content = file_obj.file
 
                 if hasattr(content, 'file'):
                     # non posix systems store real file under file attr
                     content = content.file
 
             if not content:
                 h.flash(_('No content'), category='warning')
                 raise HTTPFound(location=url('changeset_home', repo_name=c.repo_name,
                                     revision='tip'))
             if not filename:
                 h.flash(_('No filename'), category='warning')
                 raise HTTPFound(location=url('changeset_home', repo_name=c.repo_name,
                                     revision='tip'))
             # strip all crap out of file, just leave the basename
             filename = os.path.basename(filename)
             node_path = posixpath.join(location, filename)
             author = request.authuser.full_contact
 
             try:
                 nodes = {
                     node_path: {
                         'content': content
                     }
                 }
                 self.scm_model.create_nodes(
                     user=request.authuser.user_id,
                     ip_addr=request.ip_addr,
                     repo=c.db_repo,
                     message=message,
                     nodes=nodes,
                     parent_cs=c.cs,
                     author=author,
                 )
 
                 h.flash(_('Successfully committed to %s') % node_path,
                         category='success')
             except NonRelativePathError as e:
                 h.flash(_('Location must be relative path and must not '
                           'contain .. in path'), category='warning')
                 raise HTTPFound(location=url('changeset_home', repo_name=c.repo_name,
                                     revision='tip'))
             except (NodeError, NodeAlreadyExistsError) as e:
                 h.flash(_(e), category='error')
             except Exception:
                 log.error(traceback.format_exc())
                 h.flash(_('Error occurred during commit'), category='error')
             raise HTTPFound(location=url('changeset_home',
                                 repo_name=c.repo_name, revision='tip'))
 
         return render('files/files_add.html')
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def archivefile(self, repo_name, fname):
         fileformat = None
         revision = None
         ext = None
         subrepos = request.GET.get('subrepos') == 'true'
 
         for a_type, ext_data in settings.ARCHIVE_SPECS.items():
             archive_spec = fname.split(ext_data[1])
             if len(archive_spec) == 2 and archive_spec[1] == '':
                 fileformat = a_type or ext_data[1]
                 revision = archive_spec[0]
                 ext = ext_data[1]
 
         try:
             dbrepo = RepoModel().get_by_repo_name(repo_name)
             if not dbrepo.enable_downloads:
                 return _('Downloads disabled') # TODO: do something else?
 
             if c.db_repo_scm_instance.alias == 'hg':
                 # patch and reset hooks section of UI config to not run any
                 # hooks on fetching archives with subrepos
                 for k, v in c.db_repo_scm_instance._repo.ui.configitems('hooks'):
                     c.db_repo_scm_instance._repo.ui.setconfig('hooks', k, None)
 
             cs = c.db_repo_scm_instance.get_changeset(revision)
             content_type = settings.ARCHIVE_SPECS[fileformat][0]
         except ChangesetDoesNotExistError:
             return _('Unknown revision %s') % revision
         except EmptyRepositoryError:
             return _('Empty repository')
         except (ImproperArchiveTypeError, KeyError):
             return _('Unknown archive type')
 
-        from kallithea import CONFIG
         rev_name = cs.raw_id[:12]
         archive_name = '%s-%s%s' % (repo_name.replace('/', '_'), rev_name, ext)
 
         archive_path = None
         cached_archive_path = None
-        archive_cache_dir = CONFIG.get('archive_cache_dir')
+        archive_cache_dir = kallithea.CONFIG.get('archive_cache_dir')
         if archive_cache_dir and not subrepos: # TODO: subrepo caching?
             if not os.path.isdir(archive_cache_dir):
                 os.makedirs(archive_cache_dir)
             cached_archive_path = os.path.join(archive_cache_dir, archive_name)
             if os.path.isfile(cached_archive_path):
                 log.debug('Found cached archive in %s', cached_archive_path)
                 archive_path = cached_archive_path
             else:
                 log.debug('Archive %s is not yet cached', archive_name)
 
         if archive_path is None:
             # generate new archive
             fd, archive_path = tempfile.mkstemp()
             log.debug('Creating new temp archive in %s', archive_path)
             with os.fdopen(fd, 'wb') as stream:
                 cs.fill_archive(stream=stream, kind=fileformat, subrepos=subrepos)
                 # stream (and thus fd) has been closed by cs.fill_archive
             if cached_archive_path is not None:
                 # we generated the archive - move it to cache
                 log.debug('Storing new archive in %s', cached_archive_path)
                 shutil.move(archive_path, cached_archive_path)
                 archive_path = cached_archive_path
 
         def get_chunked_archive(archive_path):
             stream = open(archive_path, 'rb')
             while True:
                 data = stream.read(16 * 1024)
                 if not data:
                     break
                 yield data
             stream.close()
             if archive_path != cached_archive_path:
                 log.debug('Destroying temp archive %s', archive_path)
                 os.remove(archive_path)
 
         action_logger(user=request.authuser,
                       action='user_downloaded_archive:%s' % (archive_name),
                       repo=repo_name, ipaddr=request.ip_addr, commit=True)
 
         response.content_disposition = str('attachment; filename=%s' % (archive_name))
         response.content_type = str(content_type)
         return get_chunked_archive(archive_path)
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def diff(self, repo_name, f_path):
         ignore_whitespace_diff = h.get_ignore_whitespace_diff(request.GET)
         diff_context_size = h.get_diff_context_size(request.GET)
         diff2 = request.GET.get('diff2', '')
         diff1 = request.GET.get('diff1', '') or diff2
         c.action = request.GET.get('diff')
         c.no_changes = diff1 == diff2
         c.f_path = f_path
         c.big_diff = False
         fulldiff = request.GET.get('fulldiff')
         c.changes = OrderedDict()
         c.changes[diff2] = []
 
         # special case if we want a show rev only, it's impl here
         # to reduce JS and callbacks
 
         if request.GET.get('show_rev'):
             if asbool(request.GET.get('annotate', 'False')):
                 _url = url('files_annotate_home', repo_name=c.repo_name,
                            revision=diff1, f_path=c.f_path)
             else:
                 _url = url('files_home', repo_name=c.repo_name,
                            revision=diff1, f_path=c.f_path)
 
             raise HTTPFound(location=_url)
         try:
             if diff1 not in ['', None, 'None', '0' * 12, '0' * 40]:
                 c.changeset_1 = c.db_repo_scm_instance.get_changeset(diff1)
                 try:
                     node1 = c.changeset_1.get_node(f_path)
                     if node1.is_dir():
                         raise NodeError('%s path is a %s not a file'
                                         % (node1, type(node1)))
                 except NodeDoesNotExistError:
                     c.changeset_1 = EmptyChangeset(cs=diff1,
                                                    revision=c.changeset_1.revision,
                                                    repo=c.db_repo_scm_instance)
                     node1 = FileNode(f_path, '', changeset=c.changeset_1)
             else:
                 c.changeset_1 = EmptyChangeset(repo=c.db_repo_scm_instance)
                 node1 = FileNode(f_path, '', changeset=c.changeset_1)
 
             if diff2 not in ['', None, 'None', '0' * 12, '0' * 40]:
                 c.changeset_2 = c.db_repo_scm_instance.get_changeset(diff2)
                 try:
                     node2 = c.changeset_2.get_node(f_path)
                     if node2.is_dir():
                         raise NodeError('%s path is a %s not a file'
                                         % (node2, type(node2)))
                 except NodeDoesNotExistError:
                     c.changeset_2 = EmptyChangeset(cs=diff2,
                                                    revision=c.changeset_2.revision,
                                                    repo=c.db_repo_scm_instance)
                     node2 = FileNode(f_path, '', changeset=c.changeset_2)
             else:
                 c.changeset_2 = EmptyChangeset(repo=c.db_repo_scm_instance)
                 node2 = FileNode(f_path, '', changeset=c.changeset_2)
         except (RepositoryError, NodeError):
             log.error(traceback.format_exc())
             raise HTTPFound(location=url('files_home', repo_name=c.repo_name,
                                 f_path=f_path))
 
         if c.action == 'download':
             raw_diff = diffs.get_gitdiff(node1, node2,
                                       ignore_whitespace=ignore_whitespace_diff,
                                       context=diff_context_size)
             diff_name = '%s_vs_%s.diff' % (diff1, diff2)
             response.content_type = 'text/plain'
             response.content_disposition = (
                 'attachment; filename=%s' % diff_name
             )
             return raw_diff
 
         elif c.action == 'raw':
             raw_diff = diffs.get_gitdiff(node1, node2,
                                       ignore_whitespace=ignore_whitespace_diff,
                                       context=diff_context_size)
             response.content_type = 'text/plain'
             return raw_diff
 
         else:
             fid = h.FID(diff2, node2.path)
             diff_limit = None if fulldiff else self.cut_off_limit
             c.a_rev, c.cs_rev, a_path, diff, st, op = diffs.wrapped_diff(filenode_old=node1,
                                          filenode_new=node2,
                                          diff_limit=diff_limit,
                                          ignore_whitespace=ignore_whitespace_diff,
                                          line_context=diff_context_size,
                                          enable_comments=False)
             c.file_diff_data = [(fid, fid, op, a_path, node2.path, diff, st)]
             return render('files/file_diff.html')
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def diff_2way(self, repo_name, f_path):
         diff1 = request.GET.get('diff1', '')
         diff2 = request.GET.get('diff2', '')
         try:
             if diff1 not in ['', None, 'None', '0' * 12, '0' * 40]:
                 c.changeset_1 = c.db_repo_scm_instance.get_changeset(diff1)
                 try:
                     node1 = c.changeset_1.get_node(f_path)
                     if node1.is_dir():
                         raise NodeError('%s path is a %s not a file'
                                         % (node1, type(node1)))
                 except NodeDoesNotExistError:
                     c.changeset_1 = EmptyChangeset(cs=diff1,
                                                    revision=c.changeset_1.revision,
                                                    repo=c.db_repo_scm_instance)
                     node1 = FileNode(f_path, '', changeset=c.changeset_1)
             else:
                 c.changeset_1 = EmptyChangeset(repo=c.db_repo_scm_instance)
                 node1 = FileNode(f_path, '', changeset=c.changeset_1)
 
             if diff2 not in ['', None, 'None', '0' * 12, '0' * 40]:
                 c.changeset_2 = c.db_repo_scm_instance.get_changeset(diff2)
                 try:
                     node2 = c.changeset_2.get_node(f_path)
                     if node2.is_dir():
                         raise NodeError('%s path is a %s not a file'
                                         % (node2, type(node2)))
                 except NodeDoesNotExistError:
                     c.changeset_2 = EmptyChangeset(cs=diff2,
                                                    revision=c.changeset_2.revision,
                                                    repo=c.db_repo_scm_instance)
                     node2 = FileNode(f_path, '', changeset=c.changeset_2)
             else:
                 c.changeset_2 = EmptyChangeset(repo=c.db_repo_scm_instance)
                 node2 = FileNode(f_path, '', changeset=c.changeset_2)
         except ChangesetDoesNotExistError as e:
             msg = _('Such revision does not exist for this repository')
             h.flash(msg, category='error')
             raise HTTPNotFound()
         c.node1 = node1
         c.node2 = node2
         c.cs1 = c.changeset_1
         c.cs2 = c.changeset_2
 
         return render('files/diff_2way.html')
 
     def _get_node_history(self, cs, f_path, changesets=None):
         """
         get changesets history for given node
 
         :param cs: changeset to calculate history
         :param f_path: path for node to calculate history for
         :param changesets: if passed don't calculate history and take
             changesets defined in this list
         """
         # calculate history based on tip
         tip_cs = c.db_repo_scm_instance.get_changeset()
         if changesets is None:
             try:
                 changesets = tip_cs.get_file_history(f_path)
             except (NodeDoesNotExistError, ChangesetError):
                 # this node is not present at tip !
                 changesets = cs.get_file_history(f_path)
         hist_l = []
 
         changesets_group = ([], _("Changesets"))
         branches_group = ([], _("Branches"))
         tags_group = ([], _("Tags"))
         for chs in changesets:
             # TODO: loop over chs.branches ... but that will not give all the bogus None branches for Git ...
             _branch = chs.branch
             n_desc = '%s (%s)' % (h.show_id(chs), _branch)
             changesets_group[0].append((chs.raw_id, n_desc,))
         hist_l.append(changesets_group)
 
         for name, chs in c.db_repo_scm_instance.branches.items():
             branches_group[0].append((chs, name),)
         hist_l.append(branches_group)
 
         for name, chs in c.db_repo_scm_instance.tags.items():
             tags_group[0].append((chs, name),)
         hist_l.append(tags_group)
 
         return hist_l, changesets
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     @jsonify
     def nodelist(self, repo_name, revision, f_path):
         if request.environ.get('HTTP_X_PARTIAL_XHR'):
             cs = self.__get_cs(revision)
             _d, _f = ScmModel().get_nodes(repo_name, cs.raw_id, f_path,
                                           flat=False)
             return {'nodes': _d + _f}

kallithea/lib/helpers.py

@@ -1,1392 +1,1389 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 Helper functions
 
 Consists of functions to typically be used within templates, but also
 available to Controllers. This module is available to both as 'h'.
 """
 import hashlib
 import json
 import logging
 import random
 import re
 import textwrap
 import urllib.parse
 
 from beaker.cache import cache_region
 from pygments import highlight as code_highlight
 from pygments.formatters.html import HtmlFormatter
 from tg.i18n import ugettext as _
 from webhelpers2.html import HTML, escape, literal
 from webhelpers2.html.tags import NotGiven, Option, Options, _input, _make_safe_id_component, checkbox, end_form
 from webhelpers2.html.tags import form as insecure_form
 from webhelpers2.html.tags import hidden, link_to, password, radio
 from webhelpers2.html.tags import select as webhelpers2_select
 from webhelpers2.html.tags import submit, text, textarea
 from webhelpers2.number import format_byte_size
 from webhelpers2.text import chop_at, truncate, wrap_paragraphs
 
+import kallithea
 from kallithea.config.routing import url
 from kallithea.lib.annotate import annotate_highlight
 #==============================================================================
 # PERMS
 #==============================================================================
 from kallithea.lib.auth import HasPermissionAny, HasRepoGroupPermissionLevel, HasRepoPermissionLevel
 from kallithea.lib.markup_renderer import url_re
 from kallithea.lib.pygmentsutils import get_custom_lexer
 from kallithea.lib.utils2 import MENTIONS_REGEX, AttributeDict
 from kallithea.lib.utils2 import age as _age
 from kallithea.lib.utils2 import asbool, credentials_filter, safe_bytes, safe_int, safe_str, time_to_datetime
 from kallithea.lib.vcs.backends.base import BaseChangeset, EmptyChangeset
 from kallithea.lib.vcs.exceptions import ChangesetDoesNotExistError
 #==============================================================================
 # SCM FILTERS available via h.
 #==============================================================================
 from kallithea.lib.vcs.utils import author_email, author_name
 
 
 # mute pyflakes "imported but unused"
 assert Option
 assert checkbox
 assert end_form
 assert password
 assert radio
 assert submit
 assert text
 assert textarea
 assert format_byte_size
 assert chop_at
 assert wrap_paragraphs
 assert HasPermissionAny
 assert HasRepoGroupPermissionLevel
 assert HasRepoPermissionLevel
 assert time_to_datetime
 assert EmptyChangeset
 
 
 log = logging.getLogger(__name__)
 
 
 def canonical_url(*args, **kargs):
     '''Like url(x, qualified=True), but returns url that not only is qualified
     but also canonical, as configured in canonical_url'''
-    from kallithea import CONFIG
     try:
-        parts = CONFIG.get('canonical_url', '').split('://', 1)
+        parts = kallithea.CONFIG.get('canonical_url', '').split('://', 1)
         kargs['host'] = parts[1]
         kargs['protocol'] = parts[0]
     except IndexError:
         kargs['qualified'] = True
     return url(*args, **kargs)
 
 
 def canonical_hostname():
     '''Return canonical hostname of system'''
-    from kallithea import CONFIG
     try:
-        parts = CONFIG.get('canonical_url', '').split('://', 1)
+        parts = kallithea.CONFIG.get('canonical_url', '').split('://', 1)
         return parts[1].split('/', 1)[0]
     except IndexError:
         parts = url('home', qualified=True).split('://', 1)
         return parts[1].split('/', 1)[0]
 
 
 def html_escape(s):
     """Return string with all html escaped.
     This is also safe for javascript in html but not necessarily correct.
     """
     return (s
         .replace('&', '&amp;')
         .replace(">", "&gt;")
         .replace("<", "&lt;")
         .replace('"', "&quot;")
         .replace("'", "&apos;") # Note: this is HTML5 not HTML4 and might not work in mails
         )
 
 def js(value):
     """Convert Python value to the corresponding JavaScript representation.
 
     This is necessary to safely insert arbitrary values into HTML <script>
     sections e.g. using Mako template expression substitution.
 
     Note: Rather than using this function, it's preferable to avoid the
     insertion of values into HTML <script> sections altogether. Instead,
     data should (to the extent possible) be passed to JavaScript using
     data attributes or AJAX calls, eliminating the need for JS specific
     escaping.
 
     Note: This is not safe for use in attributes (e.g. onclick), because
     quotes are not escaped.
 
     Because the rules for parsing <script> varies between XHTML (where
     normal rules apply for any special characters) and HTML (where
     entities are not interpreted, but the literal string "</script>"
     is forbidden), the function ensures that the result never contains
     '&', '<' and '>', thus making it safe in both those contexts (but
     not in attributes).
     """
     return literal(
         ('(' + json.dumps(value) + ')')
         # In JSON, the following can only appear in string literals.
         .replace('&', r'\x26')
         .replace('<', r'\x3c')
         .replace('>', r'\x3e')
     )
 
 
 def jshtml(val):
     """HTML escapes a string value, then converts the resulting string
     to its corresponding JavaScript representation (see `js`).
 
     This is used when a plain-text string (possibly containing special
     HTML characters) will be used by a script in an HTML context (e.g.
     element.innerHTML or jQuery's 'html' method).
 
     If in doubt, err on the side of using `jshtml` over `js`, since it's
     better to escape too much than too little.
     """
     return js(escape(val))
 
 
 def shorter(s, size=20, firstline=False, postfix='...'):
     """Truncate s to size, including the postfix string if truncating.
     If firstline, truncate at newline.
     """
     if firstline:
         s = s.split('\n', 1)[0].rstrip()
     if len(s) > size:
         return s[:size - len(postfix)] + postfix
     return s
 
 
 def reset(name, value, id=NotGiven, **attrs):
     """Create a reset button, similar to webhelpers2.html.tags.submit ."""
     return _input("reset", name, value, id, attrs)
 
 
 def select(name, selected_values, options, id=NotGiven, **attrs):
     """Convenient wrapper of webhelpers2 to let it accept options as a tuple list"""
     if isinstance(options, list):
         option_list = options
         # Handle old value,label lists ... where value also can be value,label lists
         options = Options()
         for x in option_list:
             if isinstance(x, tuple) and len(x) == 2:
                 value, label = x
             elif isinstance(x, str):
                 value = label = x
             else:
                 log.error('invalid select option %r', x)
                 raise
             if isinstance(value, list):
                 og = options.add_optgroup(label)
                 for x in value:
                     if isinstance(x, tuple) and len(x) == 2:
                         group_value, group_label = x
                     elif isinstance(x, str):
                         group_value = group_label = x
                     else:
                         log.error('invalid select option %r', x)
                         raise
                     og.add_option(group_label, group_value)
             else:
                 options.add_option(label, value)
     return webhelpers2_select(name, selected_values, options, id=id, **attrs)
 
 
 safeid = _make_safe_id_component
 
 
 def FID(raw_id, path):
     """
     Creates a unique ID for filenode based on it's hash of path and revision
     it's safe to use in urls
     """
     return 'C-%s-%s' % (short_id(raw_id), hashlib.md5(safe_bytes(path)).hexdigest()[:12])
 
 
 def get_ignore_whitespace_diff(GET):
     """Return true if URL requested whitespace to be ignored"""
     return bool(GET.get('ignorews'))
 
 
 def ignore_whitespace_link(GET, anchor=None):
     """Return snippet with link to current URL with whitespace ignoring toggled"""
     params = dict(GET)  # ignoring duplicates
     if get_ignore_whitespace_diff(GET):
         params.pop('ignorews')
         title = _("Show whitespace changes")
     else:
         params['ignorews'] = '1'
         title = _("Ignore whitespace changes")
     params['anchor'] = anchor
     return link_to(
         literal('<i class="icon-strike"></i>'),
         url.current(**params),
         title=title,
         **{'data-toggle': 'tooltip'})
 
 
 def get_diff_context_size(GET):
     """Return effective context size requested in URL"""
     return safe_int(GET.get('context'), default=3)
 
 
 def increase_context_link(GET, anchor=None):
     """Return snippet with link to current URL with double context size"""
     context = get_diff_context_size(GET) * 2
     params = dict(GET)  # ignoring duplicates
     params['context'] = str(context)
     params['anchor'] = anchor
     return link_to(
         literal('<i class="icon-sort"></i>'),
         url.current(**params),
         title=_('Increase diff context to %(num)s lines') % {'num': context},
         **{'data-toggle': 'tooltip'})
 
 
 class _FilesBreadCrumbs(object):
 
     def __call__(self, repo_name, rev, paths):
         url_l = [link_to(repo_name, url('files_home',
                                         repo_name=repo_name,
                                         revision=rev, f_path=''),
                          class_='ypjax-link')]
         paths_l = paths.split('/')
         for cnt, p in enumerate(paths_l):
             if p != '':
                 url_l.append(link_to(p,
                                      url('files_home',
                                          repo_name=repo_name,
                                          revision=rev,
                                          f_path='/'.join(paths_l[:cnt + 1])
                                          ),
                                      class_='ypjax-link'
                                      )
                              )
 
         return literal('/'.join(url_l))
 
 
 files_breadcrumbs = _FilesBreadCrumbs()
 
 
 class CodeHtmlFormatter(HtmlFormatter):
     """
     My code Html Formatter for source codes
     """
 
     def wrap(self, source, outfile):
         return self._wrap_div(self._wrap_pre(self._wrap_code(source)))
 
     def _wrap_code(self, source):
         for cnt, it in enumerate(source):
             i, t = it
             t = '<span id="L%s">%s</span>' % (cnt + 1, t)
             yield i, t
 
     def _wrap_tablelinenos(self, inner):
         inner_lines = []
         lncount = 0
         for t, line in inner:
             if t:
                 lncount += 1
             inner_lines.append(line)
 
         fl = self.linenostart
         mw = len(str(lncount + fl - 1))
         sp = self.linenospecial
         st = self.linenostep
         la = self.lineanchors
         aln = self.anchorlinenos
         nocls = self.noclasses
         if sp:
             lines = []
 
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if i % sp == 0:
                         if aln:
                             lines.append('<a href="#%s%d" class="special">%*d</a>' %
                                          (la, i, mw, i))
                         else:
                             lines.append('<span class="special">%*d</span>' % (mw, i))
                     else:
                         if aln:
                             lines.append('<a href="#%s%d">%*d</a>' % (la, i, mw, i))
                         else:
                             lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
         else:
             lines = []
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if aln:
                         lines.append('<a href="#%s%d">%*d</a>' % (la, i, mw, i))
                     else:
                         lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
 
         # in case you wonder about the seemingly redundant <div> here: since the
         # content in the other cell also is wrapped in a div, some browsers in
         # some configurations seem to mess up the formatting...
         if nocls:
             yield 0, ('<table class="%stable">' % self.cssclass +
                       '<tr><td><div class="linenodiv">'
                       '<pre>' + ls + '</pre></div></td>'
                       '<td id="hlcode" class="code">')
         else:
             yield 0, ('<table class="%stable">' % self.cssclass +
                       '<tr><td class="linenos"><div class="linenodiv">'
                       '<pre>' + ls + '</pre></div></td>'
                       '<td id="hlcode" class="code">')
         yield 0, ''.join(inner_lines)
         yield 0, '</td></tr></table>'
 
 
 _whitespace_re = re.compile(r'(\t)|( )(?=\n|</div>)')
 
 
 def _markup_whitespace(m):
     groups = m.groups()
     if groups[0]:
         return '<u>\t</u>'
     if groups[1]:
         return ' <i></i>'
 
 
 def markup_whitespace(s):
     return _whitespace_re.sub(_markup_whitespace, s)
 
 
 def pygmentize(filenode, **kwargs):
     """
     pygmentize function using pygments
 
     :param filenode:
     """
     lexer = get_custom_lexer(filenode.extension) or filenode.lexer
     return literal(markup_whitespace(
         code_highlight(safe_str(filenode.content), lexer, CodeHtmlFormatter(**kwargs))))
 
 
 def hsv_to_rgb(h, s, v):
     if s == 0.0:
         return v, v, v
     i = int(h * 6.0)  # XXX assume int() truncates!
     f = (h * 6.0) - i
     p = v * (1.0 - s)
     q = v * (1.0 - s * f)
     t = v * (1.0 - s * (1.0 - f))
     i = i % 6
     if i == 0:
         return v, t, p
     if i == 1:
         return q, v, p
     if i == 2:
         return p, v, t
     if i == 3:
         return p, q, v
     if i == 4:
         return t, p, v
     if i == 5:
         return v, p, q
 
 
 def gen_color(n=10000):
     """generator for getting n of evenly distributed colors using
     hsv color and golden ratio. It always return same order of colors
 
     :returns: RGB tuple
     """
 
     golden_ratio = 0.618033988749895
     h = 0.22717784590367374
 
     for _unused in range(n):
         h += golden_ratio
         h %= 1
         HSV_tuple = [h, 0.95, 0.95]
         RGB_tuple = hsv_to_rgb(*HSV_tuple)
         yield [str(int(x * 256)) for x in RGB_tuple]
 
 
 def pygmentize_annotation(repo_name, filenode, **kwargs):
     """
     pygmentize function for annotation
 
     :param filenode:
     """
     cgenerator = gen_color()
     color_dict = {}
 
     def get_color_string(cs):
         if cs in color_dict:
             col = color_dict[cs]
         else:
             col = color_dict[cs] = next(cgenerator)
         return "color: rgb(%s)! important;" % (', '.join(col))
 
     def url_func(changeset):
         author = escape(changeset.author)
         date = changeset.date
         message = escape(changeset.message)
         tooltip_html = ("<b>Author:</b> %s<br/>"
                         "<b>Date:</b> %s</b><br/>"
                         "<b>Message:</b> %s") % (author, date, message)
 
         lnk_format = show_id(changeset)
         uri = link_to(
                 lnk_format,
                 url('changeset_home', repo_name=repo_name,
                     revision=changeset.raw_id),
                 style=get_color_string(changeset.raw_id),
                 **{'data-toggle': 'popover',
                    'data-content': tooltip_html}
               )
 
         uri += '\n'
         return uri
 
     return literal(markup_whitespace(annotate_highlight(filenode, url_func, **kwargs)))
 
 
 class _Message(object):
     """A message returned by ``pop_flash_messages()``.
 
     Converting the message to a string returns the message text. Instances
     also have the following attributes:
 
     * ``category``: the category specified when the message was created.
     * ``message``: the html-safe message text.
     """
 
     def __init__(self, category, message):
         self.category = category
         self.message = message
 
 
 def _session_flash_messages(append=None, clear=False):
     """Manage a message queue in tg.session: return the current message queue
     after appending the given message, and possibly clearing the queue."""
     key = 'flash'
     from tg import session
     if key in session:
         flash_messages = session[key]
     else:
         if append is None:  # common fast path - also used for clearing empty queue
             return []  # don't bother saving
         flash_messages = []
         session[key] = flash_messages
     if append is not None and append not in flash_messages:
         flash_messages.append(append)
     if clear:
         session.pop(key, None)
     session.save()
     return flash_messages
 
 
 def flash(message, category, logf=None):
     """
     Show a message to the user _and_ log it through the specified function
 
     category: notice (default), warning, error, success
     logf: a custom log function - such as log.debug
 
     logf defaults to log.info, unless category equals 'success', in which
     case logf defaults to log.debug.
     """
     assert category in ('error', 'success', 'warning'), category
     if hasattr(message, '__html__'):
         # render to HTML for storing in cookie
         safe_message = str(message)
     else:
         # Apply str - the message might be an exception with __str__
         # Escape, so we can trust the result without further escaping, without any risk of injection
         safe_message = html_escape(str(message))
     if logf is None:
         logf = log.info
         if category == 'success':
             logf = log.debug
 
     logf('Flash %s: %s', category, safe_message)
 
     _session_flash_messages(append=(category, safe_message))
 
 
 def pop_flash_messages():
     """Return all accumulated messages and delete them from the session.
 
     The return value is a list of ``Message`` objects.
     """
     return [_Message(category, message) for category, message in _session_flash_messages(clear=True)]
 
 
 def age(x, y=False):
     return _age(x, y)
 
 def capitalize(x):
     return x.capitalize()
 
 email = author_email
 
 def short_id(x):
     return x[:12]
 
 def hide_credentials(x):
     return ''.join(credentials_filter(x))
 
 
 def show_id(cs):
     """
     Configurable function that shows ID
     by default it's r123:fffeeefffeee
 
     :param cs: changeset instance
     """
-    from kallithea import CONFIG
-    def_len = safe_int(CONFIG.get('show_sha_length', 12))
-    show_rev = asbool(CONFIG.get('show_revision_number', False))
+    def_len = safe_int(kallithea.CONFIG.get('show_sha_length', 12))
+    show_rev = asbool(kallithea.CONFIG.get('show_revision_number', False))
 
     raw_id = cs.raw_id[:def_len]
     if show_rev:
         return 'r%s:%s' % (cs.revision, raw_id)
     else:
         return raw_id
 
 
 def fmt_date(date):
     if date:
         return date.strftime("%Y-%m-%d %H:%M:%S")
     return ""
 
 
 def is_git(repository):
     if hasattr(repository, 'alias'):
         _type = repository.alias
     elif hasattr(repository, 'repo_type'):
         _type = repository.repo_type
     else:
         _type = repository
     return _type == 'git'
 
 
 def is_hg(repository):
     if hasattr(repository, 'alias'):
         _type = repository.alias
     elif hasattr(repository, 'repo_type'):
         _type = repository.repo_type
     else:
         _type = repository
     return _type == 'hg'
 
 
 @cache_region('long_term', 'user_attr_or_none')
 def user_attr_or_none(author, show_attr):
     """Try to match email part of VCS committer string with a local user and return show_attr
     - or return None if user not found"""
     email = author_email(author)
     if email:
         from kallithea.model.db import User
         user = User.get_by_email(email)
         if user is not None:
             return getattr(user, show_attr)
     return None
 
 
 def email_or_none(author):
     """Try to match email part of VCS committer string with a local user.
     Return primary email of user, email part of the specified author name, or None."""
     if not author:
         return None
     email = user_attr_or_none(author, 'email')
     if email is not None:
         return email # always use user's main email address - not necessarily the one used to find user
 
     # extract email from the commit string
     email = author_email(author)
     if email:
         return email
 
     # No valid email, not a valid user in the system, none!
     return None
 
 
 def person(author, show_attr="username"):
     """Find the user identified by 'author', return one of the users attributes,
     default to the username attribute, None if there is no user"""
     from kallithea.model.db import User
 
     # if author is already an instance use it for extraction
     if isinstance(author, User):
         return getattr(author, show_attr)
 
     value = user_attr_or_none(author, show_attr)
     if value is not None:
         return value
 
     # Still nothing?  Just pass back the author name if any, else the email
     return author_name(author) or email(author)
 
 
 def person_by_id(id_, show_attr="username"):
     from kallithea.model.db import User
 
     # maybe it's an ID ?
     if str(id_).isdigit() or isinstance(id_, int):
         id_ = int(id_)
         user = User.get(id_)
         if user is not None:
             return getattr(user, show_attr)
     return id_
 
 
 def boolicon(value):
     """Returns boolean value of a value, represented as small html image of true/false
     icons
 
     :param value: value
     """
 
     if value:
         return HTML.tag('i', class_="icon-ok")
     else:
         return HTML.tag('i', class_="icon-minus-circled")
 
 
 def action_parser(user_log, feed=False, parse_cs=False):
     """
     This helper will action_map the specified string action into translated
     fancy names with icons and links
 
     :param user_log: user log instance
     :param feed: use output for feeds (no html and fancy icons)
     :param parse_cs: parse Changesets into VCS instances
     """
 
     action = user_log.action
     action_params = ' '
 
     x = action.split(':')
 
     if len(x) > 1:
         action, action_params = x
 
     def get_cs_links():
         revs_limit = 3  # display this amount always
         revs_top_limit = 50  # show upto this amount of changesets hidden
         revs_ids = action_params.split(',')
         deleted = user_log.repository is None
         if deleted:
             return ','.join(revs_ids)
 
         repo_name = user_log.repository.repo_name
 
         def lnk(rev, repo_name):
             lazy_cs = False
             title_ = None
             url_ = '#'
             if isinstance(rev, BaseChangeset) or isinstance(rev, AttributeDict):
                 if rev.op and rev.ref_name:
                     if rev.op == 'delete_branch':
                         lbl = _('Deleted branch: %s') % rev.ref_name
                     elif rev.op == 'tag':
                         lbl = _('Created tag: %s') % rev.ref_name
                     else:
                         lbl = 'Unknown operation %s' % rev.op
                 else:
                     lazy_cs = True
                     lbl = rev.short_id[:8]
                     url_ = url('changeset_home', repo_name=repo_name,
                                revision=rev.raw_id)
             else:
                 # changeset cannot be found - it might have been stripped or removed
                 lbl = rev[:12]
                 title_ = _('Changeset %s not found') % lbl
             if parse_cs:
                 return link_to(lbl, url_, title=title_, **{'data-toggle': 'tooltip'})
             return link_to(lbl, url_, class_='lazy-cs' if lazy_cs else '',
                            **{'data-raw_id': rev.raw_id, 'data-repo_name': repo_name})
 
         def _get_op(rev_txt):
             _op = None
             _name = rev_txt
             if len(rev_txt.split('=>')) == 2:
                 _op, _name = rev_txt.split('=>')
             return _op, _name
 
         revs = []
         if len([v for v in revs_ids if v != '']) > 0:
             repo = None
             for rev in revs_ids[:revs_top_limit]:
                 _op, _name = _get_op(rev)
 
                 # we want parsed changesets, or new log store format is bad
                 if parse_cs:
                     try:
                         if repo is None:
                             repo = user_log.repository.scm_instance
                         _rev = repo.get_changeset(rev)
                         revs.append(_rev)
                     except ChangesetDoesNotExistError:
                         log.error('cannot find revision %s in this repo', rev)
                         revs.append(rev)
                 else:
                     _rev = AttributeDict({
                         'short_id': rev[:12],
                         'raw_id': rev,
                         'message': '',
                         'op': _op,
                         'ref_name': _name
                     })
                     revs.append(_rev)
         cs_links = [" " + ', '.join(
             [lnk(rev, repo_name) for rev in revs[:revs_limit]]
         )]
         _op1, _name1 = _get_op(revs_ids[0])
         _op2, _name2 = _get_op(revs_ids[-1])
 
         _rev = '%s...%s' % (_name1, _name2)
 
         compare_view = (
             ' <div class="compare_view" data-toggle="tooltip" title="%s">'
             '<a href="%s">%s</a> </div>' % (
                 _('Show all combined changesets %s->%s') % (
                     revs_ids[0][:12], revs_ids[-1][:12]
                 ),
                 url('changeset_home', repo_name=repo_name,
                     revision=_rev
                 ),
                 _('Compare view')
             )
         )
 
         # if we have exactly one more than normally displayed
         # just display it, takes less space than displaying
         # "and 1 more revisions"
         if len(revs_ids) == revs_limit + 1:
             cs_links.append(", " + lnk(revs[revs_limit], repo_name))
 
         # hidden-by-default ones
         if len(revs_ids) > revs_limit + 1:
             uniq_id = revs_ids[0]
             html_tmpl = (
                 '<span> %s <a class="show_more" id="_%s" '
                 'href="#more">%s</a> %s</span>'
             )
             if not feed:
                 cs_links.append(html_tmpl % (
                       _('and'),
                       uniq_id, _('%s more') % (len(revs_ids) - revs_limit),
                       _('revisions')
                     )
                 )
 
             if not feed:
                 html_tmpl = '<span id="%s" style="display:none">, %s </span>'
             else:
                 html_tmpl = '<span id="%s"> %s </span>'
 
             morelinks = ', '.join(
               [lnk(rev, repo_name) for rev in revs[revs_limit:]]
             )
 
             if len(revs_ids) > revs_top_limit:
                 morelinks += ', ...'
 
             cs_links.append(html_tmpl % (uniq_id, morelinks))
         if len(revs) > 1:
             cs_links.append(compare_view)
         return ''.join(cs_links)
 
     def get_fork_name():
         repo_name = action_params
         url_ = url('summary_home', repo_name=repo_name)
         return _('Fork name %s') % link_to(action_params, url_)
 
     def get_user_name():
         user_name = action_params
         return user_name
 
     def get_users_group():
         group_name = action_params
         return group_name
 
     def get_pull_request():
         from kallithea.model.db import PullRequest
         pull_request_id = action_params
         nice_id = PullRequest.make_nice_id(pull_request_id)
 
         deleted = user_log.repository is None
         if deleted:
             repo_name = user_log.repository_name
         else:
             repo_name = user_log.repository.repo_name
 
         return link_to(_('Pull request %s') % nice_id,
                     url('pullrequest_show', repo_name=repo_name,
                     pull_request_id=pull_request_id))
 
     def get_archive_name():
         archive_name = action_params
         return archive_name
 
     # action : translated str, callback(extractor), icon
     action_map = {
         'user_deleted_repo':           (_('[deleted] repository'),
                                         None, 'icon-trashcan'),
         'user_created_repo':           (_('[created] repository'),
                                         None, 'icon-plus'),
         'user_created_fork':           (_('[created] repository as fork'),
                                         None, 'icon-fork'),
         'user_forked_repo':            (_('[forked] repository'),
                                         get_fork_name, 'icon-fork'),
         'user_updated_repo':           (_('[updated] repository'),
                                         None, 'icon-pencil'),
         'user_downloaded_archive':      (_('[downloaded] archive from repository'),
                                         get_archive_name, 'icon-download-cloud'),
         'admin_deleted_repo':          (_('[delete] repository'),
                                         None, 'icon-trashcan'),
         'admin_created_repo':          (_('[created] repository'),
                                         None, 'icon-plus'),
         'admin_forked_repo':           (_('[forked] repository'),
                                         None, 'icon-fork'),
         'admin_updated_repo':          (_('[updated] repository'),
                                         None, 'icon-pencil'),
         'admin_created_user':          (_('[created] user'),
                                         get_user_name, 'icon-user'),
         'admin_updated_user':          (_('[updated] user'),
                                         get_user_name, 'icon-user'),
         'admin_created_users_group':   (_('[created] user group'),
                                         get_users_group, 'icon-pencil'),
         'admin_updated_users_group':   (_('[updated] user group'),
                                         get_users_group, 'icon-pencil'),
         'user_commented_revision':     (_('[commented] on revision in repository'),
                                         get_cs_links, 'icon-comment'),
         'user_commented_pull_request': (_('[commented] on pull request for'),
                                         get_pull_request, 'icon-comment'),
         'user_closed_pull_request':    (_('[closed] pull request for'),
                                         get_pull_request, 'icon-ok'),
         'push':                        (_('[pushed] into'),
                                         get_cs_links, 'icon-move-up'),
         'push_local':                  (_('[committed via Kallithea] into repository'),
                                         get_cs_links, 'icon-pencil'),
         'push_remote':                 (_('[pulled from remote] into repository'),
                                         get_cs_links, 'icon-move-up'),
         'pull':                        (_('[pulled] from'),
                                         None, 'icon-move-down'),
         'started_following_repo':      (_('[started following] repository'),
                                         None, 'icon-heart'),
         'stopped_following_repo':      (_('[stopped following] repository'),
                                         None, 'icon-heart-empty'),
     }
 
     action_str = action_map.get(action, action)
     if feed:
         action = action_str[0].replace('[', '').replace(']', '')
     else:
         action = action_str[0] \
             .replace('[', '<b>') \
             .replace(']', '</b>')
 
     action_params_func = action_str[1] if callable(action_str[1]) else (lambda: "")
 
     def action_parser_icon():
         action = user_log.action
         action_params = None
         x = action.split(':')
 
         if len(x) > 1:
             action, action_params = x
 
         ico = action_map.get(action, ['', '', ''])[2]
         html = """<i class="%s"></i>""" % ico
         return literal(html)
 
     # returned callbacks we need to call to get
     return [lambda: literal(action), action_params_func, action_parser_icon]
 
 
 #==============================================================================
 # GRAVATAR URL
 #==============================================================================
 def gravatar_div(email_address, cls='', size=30, **div_attributes):
     """Return an html literal with a span around a gravatar if they are enabled.
     Extra keyword parameters starting with 'div_' will get the prefix removed
     and '_' changed to '-' and be used as attributes on the div. The default
     class is 'gravatar'.
     """
     from tg import tmpl_context as c
     if not c.visual.use_gravatar:
         return ''
     if 'div_class' not in div_attributes:
         div_attributes['div_class'] = "gravatar"
     attributes = []
     for k, v in sorted(div_attributes.items()):
         assert k.startswith('div_'), k
         attributes.append(' %s="%s"' % (k[4:].replace('_', '-'), escape(v)))
     return literal("""<span%s>%s</span>""" %
                    (''.join(attributes),
                     gravatar(email_address, cls=cls, size=size)))
 
 
 def gravatar(email_address, cls='', size=30):
     """return html element of the gravatar
 
     This method will return an <img> with the resolution double the size (for
     retina screens) of the image. If the url returned from gravatar_url is
     empty then we fallback to using an icon.
 
     """
     from tg import tmpl_context as c
     if not c.visual.use_gravatar:
         return ''
 
     src = gravatar_url(email_address, size * 2)
 
     if src:
         # here it makes sense to use style="width: ..." (instead of, say, a
         # stylesheet) because we using this to generate a high-res (retina) size
         html = ('<i class="icon-gravatar {cls}"'
                 ' style="font-size: {size}px;background-size: {size}px;background-image: url(\'{src}\')"'
                 '></i>').format(cls=cls, size=size, src=src)
 
     else:
         # if src is empty then there was no gravatar, so we use a font icon
         html = ("""<i class="icon-user {cls}" style="font-size: {size}px;"></i>"""
             .format(cls=cls, size=size))
 
     return literal(html)
 
 
 def gravatar_url(email_address, size=30, default=''):
     from tg import tmpl_context as c
 
     if not c.visual.use_gravatar:
         return ""
 
     _def = '[e-mail]'  # default gravatar
     email_address = email_address or _def
 
     if email_address == _def:
         return default
 
     # re-import url so tests can mock it
     from kallithea.config.routing import url
     from kallithea.model.db import User
 
     parsed_url = urllib.parse.urlparse(url.current(qualified=True))
     url = (c.visual.gravatar_url or User.DEFAULT_GRAVATAR_URL) \
                .replace('{email}', email_address) \
                .replace('{md5email}', hashlib.md5(safe_bytes(email_address).lower()).hexdigest()) \
                .replace('{netloc}', parsed_url.netloc) \
                .replace('{scheme}', parsed_url.scheme) \
                .replace('{size}', str(size))
     return url
 
 
 def changed_tooltip(nodes):
     """
     Generates a html string for changed nodes in changeset page.
     It limits the output to 30 entries
 
     :param nodes: LazyNodesGenerator
     """
     if nodes:
         pref = ': <br/> '
         suf = ''
         if len(nodes) > 30:
             suf = '<br/>' + _(' and %s more') % (len(nodes) - 30)
         return literal(pref + '<br/> '.join([x.path
                                              for x in nodes[:30]]) + suf)
     else:
         return ': ' + _('No files')
 
 
 def fancy_file_stats(stats):
     """
     Displays a fancy two colored bar for number of added/deleted
     lines of code on file
 
     :param stats: two element list of added/deleted lines of code
     """
     from kallithea.lib.diffs import BIN_FILENODE, CHMOD_FILENODE, DEL_FILENODE, MOD_FILENODE, NEW_FILENODE, RENAMED_FILENODE
 
     a, d = stats['added'], stats['deleted']
     width = 100
 
     if stats['binary']:
         # binary mode
         lbl = ''
         bin_op = 1
 
         if BIN_FILENODE in stats['ops']:
             lbl = 'bin+'
 
         if NEW_FILENODE in stats['ops']:
             lbl += _('new file')
             bin_op = NEW_FILENODE
         elif MOD_FILENODE in stats['ops']:
             lbl += _('mod')
             bin_op = MOD_FILENODE
         elif DEL_FILENODE in stats['ops']:
             lbl += _('del')
             bin_op = DEL_FILENODE
         elif RENAMED_FILENODE in stats['ops']:
             lbl += _('rename')
             bin_op = RENAMED_FILENODE
 
         # chmod can go with other operations
         if CHMOD_FILENODE in stats['ops']:
             _org_lbl = _('chmod')
             lbl += _org_lbl if lbl.endswith('+') else '+%s' % _org_lbl
 
         #import ipdb;ipdb.set_trace()
         b_d = '<div class="bin bin%s progress-bar" style="width:100%%">%s</div>' % (bin_op, lbl)
         b_a = '<div class="bin bin1" style="width:0%"></div>'
         return literal('<div style="width:%spx" class="progress">%s%s</div>' % (width, b_a, b_d))
 
     t = stats['added'] + stats['deleted']
     unit = float(width) / (t or 1)
 
     # needs > 9% of width to be visible or 0 to be hidden
     a_p = max(9, unit * a) if a > 0 else 0
     d_p = max(9, unit * d) if d > 0 else 0
     p_sum = a_p + d_p
 
     if p_sum > width:
         # adjust the percentage to be == 100% since we adjusted to 9
         if a_p > d_p:
             a_p = a_p - (p_sum - width)
         else:
             d_p = d_p - (p_sum - width)
 
     a_v = a if a > 0 else ''
     d_v = d if d > 0 else ''
 
     d_a = '<div class="added progress-bar" style="width:%s%%">%s</div>' % (
         a_p, a_v
     )
     d_d = '<div class="deleted progress-bar" style="width:%s%%">%s</div>' % (
         d_p, d_v
     )
     return literal('<div class="progress" style="width:%spx">%s%s</div>' % (width, d_a, d_d))
 
 
 _URLIFY_RE = re.compile(r'''
 # URL markup
 (?P<url>%s) |
 # @mention markup
 (?P<mention>%s) |
 # Changeset hash markup
 (?<!\w|[-_])
   (?P<hash>[0-9a-f]{12,40})
 (?!\w|[-_]) |
 # Markup of *bold text*
 (?:
   (?:^|(?<=\s))
   (?P<bold> [*] (?!\s) [^*\n]* (?<!\s) [*] )
   (?![*\w])
 ) |
 # "Stylize" markup
 \[see\ \=&gt;\ *(?P<seen>[a-zA-Z0-9\/\=\?\&\ \:\/\.\-]*)\] |
 \[license\ \=&gt;\ *(?P<license>[a-zA-Z0-9\/\=\?\&\ \:\/\.\-]*)\] |
 \[(?P<tagtype>requires|recommends|conflicts|base)\ \=&gt;\ *(?P<tagvalue>[a-zA-Z0-9\-\/]*)\] |
 \[(?:lang|language)\ \=&gt;\ *(?P<lang>[a-zA-Z\-\/\#\+]*)\] |
 \[(?P<tag>[a-z]+)\]
 ''' % (url_re.pattern, MENTIONS_REGEX.pattern),
     re.VERBOSE | re.MULTILINE | re.IGNORECASE)
 
 
 def urlify_text(s, repo_name=None, link_=None, truncate=None, stylize=False, truncatef=truncate):
     """
     Parses given text message and make literal html with markup.
     The text will be truncated to the specified length.
     Hashes are turned into changeset links to specified repository.
     URLs links to what they say.
     Issues are linked to given issue-server.
     If link_ is provided, all text not already linking somewhere will link there.
     >>> urlify_text("Urlify http://example.com/ and 'https://example.com' *and* <b>markup/b>")
     literal('Urlify <a href="http://example.com/">http://example.com/</a> and &#39;<a href="https://example.com&apos">https://example.com&apos</a>; <b>*and*</b> &lt;b&gt;markup/b&gt;')
     """
 
     def _replace(match_obj):
         url = match_obj.group('url')
         if url is not None:
             return '<a href="%(url)s">%(url)s</a>' % {'url': url}
         mention = match_obj.group('mention')
         if mention is not None:
             return '<b>%s</b>' % mention
         hash_ = match_obj.group('hash')
         if hash_ is not None and repo_name is not None:
             from kallithea.config.routing import url  # doh, we need to re-import url to mock it later
             return '<a class="changeset_hash" href="%(url)s">%(hash)s</a>' % {
                  'url': url('changeset_home', repo_name=repo_name, revision=hash_),
                  'hash': hash_,
                 }
         bold = match_obj.group('bold')
         if bold is not None:
             return '<b>*%s*</b>' % _urlify(bold[1:-1])
         if stylize:
             seen = match_obj.group('seen')
             if seen:
                 return '<div class="label label-meta" data-tag="see">see =&gt; %s</div>' % seen
             license = match_obj.group('license')
             if license:
                 return '<div class="label label-meta" data-tag="license"><a href="http://www.opensource.org/licenses/%s">%s</a></div>' % (license, license)
             tagtype = match_obj.group('tagtype')
             if tagtype:
                 tagvalue = match_obj.group('tagvalue')
                 return '<div class="label label-meta" data-tag="%s">%s =&gt; <a href="/%s">%s</a></div>' % (tagtype, tagtype, tagvalue, tagvalue)
             lang = match_obj.group('lang')
             if lang:
                 return '<div class="label label-meta" data-tag="lang">%s</div>' % lang
             tag = match_obj.group('tag')
             if tag:
                 return '<div class="label label-meta" data-tag="%s">%s</div>' % (tag, tag)
         return match_obj.group(0)
 
     def _urlify(s):
         """
         Extract urls from text and make html links out of them
         """
         return _URLIFY_RE.sub(_replace, s)
 
     if truncate is None:
         s = s.rstrip()
     else:
         s = truncatef(s, truncate, whole_word=True)
     s = html_escape(s)
     s = _urlify(s)
     if repo_name is not None:
         s = urlify_issues(s, repo_name)
     if link_ is not None:
         # make href around everything that isn't a href already
         s = linkify_others(s, link_)
     s = s.replace('\r\n', '<br/>').replace('\n', '<br/>')
     # Turn HTML5 into more valid HTML4 as required by some mail readers.
     # (This is not done in one step in html_escape, because character codes like
     # &#123; risk to be seen as an issue reference due to the presence of '#'.)
     s = s.replace("&apos;", "&#39;")
     return literal(s)
 
 
 def linkify_others(t, l):
     """Add a default link to html with links.
     HTML doesn't allow nesting of links, so the outer link must be broken up
     in pieces and give space for other links.
     """
     urls = re.compile(r'(\<a.*?\<\/a\>)',)
     links = []
     for e in urls.split(t):
         if e.strip() and not urls.match(e):
             links.append('<a class="message-link" href="%s">%s</a>' % (l, e))
         else:
             links.append(e)
 
     return ''.join(links)
 
 
 # Global variable that will hold the actual urlify_issues function body.
 # Will be set on first use when the global configuration has been read.
 _urlify_issues_f = None
 
 
 def urlify_issues(newtext, repo_name):
     """Urlify issue references according to .ini configuration"""
     global _urlify_issues_f
     if _urlify_issues_f is None:
-        from kallithea import CONFIG
         from kallithea.model.db import URL_SEP
-        assert CONFIG['sqlalchemy.url'] # make sure config has been loaded
+        assert kallithea.CONFIG['sqlalchemy.url'] # make sure config has been loaded
 
         # Build chain of urlify functions, starting with not doing any transformation
         def tmp_urlify_issues_f(s):
             return s
 
         issue_pat_re = re.compile(r'issue_pat(.*)')
-        for k in CONFIG:
+        for k in kallithea.CONFIG:
             # Find all issue_pat* settings that also have corresponding server_link and prefix configuration
             m = issue_pat_re.match(k)
             if m is None:
                 continue
             suffix = m.group(1)
-            issue_pat = CONFIG.get(k)
-            issue_server_link = CONFIG.get('issue_server_link%s' % suffix)
-            issue_sub = CONFIG.get('issue_sub%s' % suffix)
-            issue_prefix = CONFIG.get('issue_prefix%s' % suffix)
+            issue_pat = kallithea.CONFIG.get(k)
+            issue_server_link = kallithea.CONFIG.get('issue_server_link%s' % suffix)
+            issue_sub = kallithea.CONFIG.get('issue_sub%s' % suffix)
+            issue_prefix = kallithea.CONFIG.get('issue_prefix%s' % suffix)
             if issue_prefix:
                 log.error('found unsupported issue_prefix%s = %r - use issue_sub%s instead', suffix, issue_prefix, suffix)
             if not issue_pat:
                 log.error('skipping incomplete issue pattern %r: it needs a regexp', k)
                 continue
             if not issue_server_link:
                 log.error('skipping incomplete issue pattern %r: it needs issue_server_link%s', k, suffix)
                 continue
             if issue_sub is None: # issue_sub can be empty but should be present
                 log.error('skipping incomplete issue pattern %r: it needs (a potentially empty) issue_sub%s', k, suffix)
                 continue
 
             # Wrap tmp_urlify_issues_f with substitution of this pattern, while making sure all loop variables (and compiled regexpes) are bound
             try:
                 issue_re = re.compile(issue_pat)
             except re.error as e:
                 log.error('skipping invalid issue pattern %r: %r -> %r %r. Error: %s', k, issue_pat, issue_server_link, issue_sub, str(e))
                 continue
 
             log.debug('issue pattern %r: %r -> %r %r', k, issue_pat, issue_server_link, issue_sub)
 
             def issues_replace(match_obj,
                                issue_server_link=issue_server_link, issue_sub=issue_sub):
                 try:
                     issue_url = match_obj.expand(issue_server_link)
                 except (IndexError, re.error) as e:
                     log.error('invalid issue_url setting %r -> %r %r. Error: %s', issue_pat, issue_server_link, issue_sub, str(e))
                     issue_url = issue_server_link
                 issue_url = issue_url.replace('{repo}', repo_name)
                 issue_url = issue_url.replace('{repo_name}', repo_name.split(URL_SEP)[-1])
                 # if issue_sub is empty use the matched issue reference verbatim
                 if not issue_sub:
                     issue_text = match_obj.group()
                 else:
                     try:
                         issue_text = match_obj.expand(issue_sub)
                     except (IndexError, re.error) as e:
                         log.error('invalid issue_sub setting %r -> %r %r. Error: %s', issue_pat, issue_server_link, issue_sub, str(e))
                         issue_text = match_obj.group()
 
                 return (
                     '<a class="issue-tracker-link" href="%(url)s">'
                     '%(text)s'
                     '</a>'
                     ) % {
                      'url': issue_url,
                      'text': issue_text,
                     }
 
             def tmp_urlify_issues_f(s, issue_re=issue_re, issues_replace=issues_replace, chain_f=tmp_urlify_issues_f):
                 return issue_re.sub(issues_replace, chain_f(s))
 
         # Set tmp function globally - atomically
         _urlify_issues_f = tmp_urlify_issues_f
 
     return _urlify_issues_f(newtext)
 
 
 def render_w_mentions(source, repo_name=None):
     """
     Render plain text with revision hashes and issue references urlified
     and with @mention highlighting.
     """
     s = safe_str(source)
     s = urlify_text(s, repo_name=repo_name)
     return literal('<div class="formatted-fixed">%s</div>' % s)
 
 
 def short_ref(ref_type, ref_name):
     if ref_type == 'rev':
         return short_id(ref_name)
     return ref_name
 
 
 def link_to_ref(repo_name, ref_type, ref_name, rev=None):
     """
     Return full markup for a href to changeset_home for a changeset.
     If ref_type is branch it will link to changelog.
     ref_name is shortened if ref_type is 'rev'.
     if rev is specified show it too, explicitly linking to that revision.
     """
     txt = short_ref(ref_type, ref_name)
     if ref_type == 'branch':
         u = url('changelog_home', repo_name=repo_name, branch=ref_name)
     else:
         u = url('changeset_home', repo_name=repo_name, revision=ref_name)
     l = link_to(repo_name + '#' + txt, u)
     if rev and ref_type != 'rev':
         l = literal('%s (%s)' % (l, link_to(short_id(rev), url('changeset_home', repo_name=repo_name, revision=rev))))
     return l
 
 
 def changeset_status(repo, revision):
     from kallithea.model.changeset_status import ChangesetStatusModel
     return ChangesetStatusModel().get_status(repo, revision)
 
 
 def changeset_status_lbl(changeset_status):
     from kallithea.model.db import ChangesetStatus
     return ChangesetStatus.get_status_lbl(changeset_status)
 
 
 def get_permission_name(key):
     from kallithea.model.db import Permission
     return dict(Permission.PERMS).get(key)
 
 
 def journal_filter_help():
     return _(textwrap.dedent('''
         Example filter terms:
             repository:vcs
             username:developer
             action:*push*
             ip:127.0.0.1
             date:20120101
             date:[20120101100000 TO 20120102]
 
         Generate wildcards using '*' character:
             "repository:vcs*" - search everything starting with 'vcs'
             "repository:*vcs*" - search for repository containing 'vcs'
 
         Optional AND / OR operators in queries
             "repository:vcs OR repository:test"
             "username:test AND repository:test*"
     '''))
 
 
 def not_mapped_error(repo_name):
     flash(_('%s repository is not mapped to db perhaps'
             ' it was created or renamed from the filesystem'
             ' please run the application again'
             ' in order to rescan repositories') % repo_name, category='error')
 
 
 def ip_range(ip_addr):
     from kallithea.model.db import UserIpMap
     s, e = UserIpMap._get_ip_range(ip_addr)
     return '%s - %s' % (s, e)
 
 
 session_csrf_secret_name = "_session_csrf_secret_token"
 
 def session_csrf_secret_token():
     """Return (and create) the current session's CSRF protection token."""
     from tg import session
     if not session_csrf_secret_name in session:
         session[session_csrf_secret_name] = str(random.getrandbits(128))
         session.save()
     return session[session_csrf_secret_name]
 
 def form(url, method="post", **attrs):
     """Like webhelpers.html.tags.form , but automatically adding
     session_csrf_secret_token for POST. The secret is thus never leaked in GET
     URLs.
     """
     form = insecure_form(url, method, **attrs)
     if method.lower() == 'get':
         return form
     return form + HTML.div(hidden(session_csrf_secret_name, session_csrf_secret_token()), style="display: none;")

kallithea/tests/functional/test_search_indexing.py

@@ -1,188 +1,188 b''
 import mock
 
-from kallithea import CONFIG
+import kallithea
 from kallithea.config.conf import INDEX_FILENAMES
 from kallithea.model.meta import Session
 from kallithea.model.repo import RepoModel
 from kallithea.model.repo_group import RepoGroupModel
 from kallithea.tests import base
 from kallithea.tests.fixture import Fixture, create_test_index
 
 
 fixture = Fixture()
 
 
 def init_indexing_test(repo):
     prev = fixture.commit_change(repo.repo_name,
                                  filename='this_should_be_unique_filename.txt',
                                  content='this_should_be_unique_content\n',
                                  message='this_should_be_unique_commit_log',
                                  vcs_type='hg',
                                  newfile=True)
 
 def init_stopword_test(repo):
     prev = fixture.commit_change(repo.repo_name,
                                  filename='this/is/it',
                                  content='def test\n',
                                  message='bother to ask where - in folder',
                                  vcs_type='hg',
                                  newfile=True)
     prev = fixture.commit_change(repo.repo_name,
                                  filename='join.us',
                                  content='def test\n',
                                  message='bother to ask where - top level',
                                  author='this is it <[e-mail]>',
                                  vcs_type='hg',
                                  parent=prev,
                                  newfile=True)
 
 
 repos = [
     # reponame,              init func or fork base, groupname
     ('indexing_test',       init_indexing_test,     None),
     ('indexing_test-fork',  'indexing_test',       None),
     ('group/indexing_test', 'indexing_test',       'group'),
     ('this-is-it',          'indexing_test',       None),
     ('indexing_test-foo',   'indexing_test',       None),
     ('stopword_test',       init_stopword_test,     None),
 ]
 
 
 # map: name => id
 repoids = {}
 groupids = {}
 
 
 def rebuild_index(full_index):
     with mock.patch('kallithea.lib.indexers.daemon.log.debug',
                     lambda *args, **kwargs: None):
         # The more revisions managed repositories have, the more
         # memory capturing "log.debug()" output in "indexers.daemon"
         # requires. This may cause unintentional failure of subsequent
         # tests, if ENOMEM at forking "git" prevents from rebuilding
         # index for search.
         # Therefore, "log.debug()" is disabled regardless of logging
         # level while rebuilding index.
         # (FYI, ENOMEM occurs at forking "git" with python 2.7.3,
         # Linux 3.2.78-1 x86_64, 3GB memory, and no ulimit
         # configuration for memory)
-        create_test_index(base.TESTS_TMP_PATH, CONFIG, full_index=full_index)
+        create_test_index(base.TESTS_TMP_PATH, kallithea.CONFIG, full_index=full_index)
 
 
 class TestSearchControllerIndexing(base.TestController):
     @classmethod
     def setup_class(cls):
         for reponame, init_or_fork, groupname in repos:
             if groupname and groupname not in groupids:
                 group = fixture.create_repo_group(groupname)
                 groupids[groupname] = group.group_id
             if callable(init_or_fork):
                 repo = fixture.create_repo(reponame,
                                            repo_group=groupname)
                 init_or_fork(repo)
             else:
                 repo = fixture.create_fork(init_or_fork, reponame,
                                            repo_group=groupname)
             repoids[reponame] = repo.repo_id
 
         # treat "it" as indexable filename
         filenames_mock = list(INDEX_FILENAMES)
         filenames_mock.append('it')
         with mock.patch('kallithea.lib.indexers.daemon.INDEX_FILENAMES',
                         filenames_mock):
             rebuild_index(full_index=False) # only for newly added repos
 
     @classmethod
     def teardown_class(cls):
         # delete in reversed order, to delete fork destination at first
         for reponame, init_or_fork, groupname in reversed(repos):
             RepoModel().delete(repoids[reponame])
 
         for reponame, init_or_fork, groupname in reversed(repos):
             if groupname in groupids:
                 RepoGroupModel().delete(groupids.pop(groupname),
                                         force_delete=True)
 
         Session().commit()
         Session.remove()
 
         rebuild_index(full_index=True) # rebuild fully for subsequent tests
 
     @base.parametrize('reponame', [
         ('indexing_test'),
         ('indexing_test-fork'),
         ('group/indexing_test'),
         ('this-is-it'),
         ('*-fork'),
         ('group/*'),
     ])
     @base.parametrize('searchtype,query,hit', [
         ('content', 'this_should_be_unique_content', 1),
         ('commit', 'this_should_be_unique_commit_log', 1),
         ('path', 'this_should_be_unique_filename.txt', 1),
     ])
     def test_repository_tokenization(self, reponame, searchtype, query, hit):
         self.log_user()
 
         q = 'repository:%s %s' % (reponame, query)
         response = self.app.get(base.url(controller='search', action='index'),
                                 {'q': q, 'type': searchtype})
         response.mustcontain('>%d results' % hit)
 
     @base.parametrize('reponame', [
         ('indexing_test'),
         ('indexing_test-fork'),
         ('group/indexing_test'),
         ('this-is-it'),
     ])
     @base.parametrize('searchtype,query,hit', [
         ('content', 'this_should_be_unique_content', 1),
         ('commit', 'this_should_be_unique_commit_log', 1),
         ('path', 'this_should_be_unique_filename.txt', 1),
     ])
     def test_searching_under_repository(self, reponame, searchtype, query, hit):
         self.log_user()
 
         response = self.app.get(base.url(controller='search', action='index',
                                     repo_name=reponame),
                                 {'q': query, 'type': searchtype})
         response.mustcontain('>%d results' % hit)
 
     @base.parametrize('searchtype,query,hit', [
         ('content', 'path:this/is/it def test', 1),
         ('commit', 'added:this/is/it bother to ask where', 1),
         # this condition matches against files below, because
         # "path:" condition is also applied on "repository path".
         # - "this/is/it" in "stopword_test" repo
         # - "this_should_be_unique_filename.txt" in "this-is-it" repo
         ('path', 'this/is/it', 2),
 
         ('content', 'extension:us', 1),
         ('path', 'extension:us', 1),
     ])
     def test_filename_stopword(self, searchtype, query, hit):
         response = self.app.get(base.url(controller='search', action='index'),
                                 {'q': query, 'type': searchtype})
 
         response.mustcontain('>%d results' % hit)
 
     @base.parametrize('searchtype,query,hit', [
         # matching against both 2 files
         ('content', 'owner:"this is it"', 0),
         ('content', 'owner:this-is-it', 0),
         ('path', 'owner:"this is it"', 0),
         ('path', 'owner:this-is-it', 0),
 
         # matching against both 2 revisions
         ('commit', 'owner:"this is it"', 0),
         ('commit', 'owner:"this-is-it"', 0),
 
         # matching against only 1 revision
         ('commit', 'author:"this is it"', 1),
         ('commit', 'author:"this-is-it"', 1),
     ])
     def test_mailaddr_stopword(self, searchtype, query, hit):
         response = self.app.get(base.url(controller='search', action='index'),
                                 {'q': query, 'type': searchtype})
 
         response.mustcontain('>%d results' % hit)

kallithea/tests/other/test_vcs_operations.py

@@ -1,642 +1,642 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 Test suite for vcs push/pull operations.
 
 The tests need Git > 1.8.1.
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Dec 30, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 
 """
 
 import json
 import os
 import re
 import tempfile
 import time
 import urllib.request
 from subprocess import PIPE, Popen
 from tempfile import _RandomNameSequence
 
 import pytest
 
-from kallithea import CONFIG
+import kallithea
 from kallithea.lib.utils2 import ascii_bytes, safe_str
 from kallithea.model.db import Repository, Ui, User, UserIpMap, UserLog
 from kallithea.model.meta import Session
 from kallithea.model.ssh_key import SshKeyModel
 from kallithea.model.user import UserModel
 from kallithea.tests import base
 from kallithea.tests.fixture import Fixture
 
 
 DEBUG = True
 HOST = '127.0.0.1:4999'  # test host
 
 fixture = Fixture()
 
 
 # Parameterize different kinds of VCS testing - both the kind of VCS and the
 # access method (HTTP/SSH)
 
 # Mixin for using HTTP and SSH URLs
 class HttpVcsTest(object):
     @staticmethod
     def repo_url_param(webserver, repo_name, **kwargs):
         return webserver.repo_url(repo_name, **kwargs)
 
 class SshVcsTest(object):
     public_keys = {
         base.TEST_USER_REGULAR_LOGIN: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQC6Ycnc2oUZHQnQwuqgZqTTdMDZD7ataf3JM7oG2Fw8JR6cdmz4QZLe5mfDwaFwG2pWHLRpVqzfrD/Pn3rIO++bgCJH5ydczrl1WScfryV1hYMJ/4EzLGM657J1/q5EI+b9SntKjf4ax+KP322L0TNQGbZUHLbfG2MwHMrYBQpHUQ== kallithea@localhost',
         base.TEST_USER_ADMIN_LOGIN: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQC6Ycnc2oUZHQnQwuqgZqTTdMDZD7ataf3JM7oG2Fw8JR6cdmz4QZLe5mfDwaFwG2pWHLRpVqzfrD/Pn3rIO++bgCJH5ydczrl1WScfryV1hYMJ/4EzLGM657J1/q5EI+b9SntKjf4ax+KP322L0TNQGbZUHLbfG2MwHMrYBQpHUq== kallithea@localhost',
     }
 
     @classmethod
     def repo_url_param(cls, webserver, repo_name, username=base.TEST_USER_ADMIN_LOGIN, password=base.TEST_USER_ADMIN_PASS, client_ip=base.IP_ADDR):
         user = User.get_by_username(username)
         if user.ssh_keys:
             ssh_key = user.ssh_keys[0]
         else:
             sshkeymodel = SshKeyModel()
             ssh_key = sshkeymodel.create(user, 'test key', cls.public_keys[user.username])
             Session().commit()
 
         return cls._ssh_param(repo_name, user, ssh_key, client_ip)
 
 # Mixins for using Mercurial and Git
 class HgVcsTest(object):
     repo_type = 'hg'
     repo_name = base.HG_REPO
 
 class GitVcsTest(object):
     repo_type = 'git'
     repo_name = base.GIT_REPO
 
 # Combine mixins to give the combinations we want to parameterize tests with
 class HgHttpVcsTest(HgVcsTest, HttpVcsTest):
     pass
 
 class GitHttpVcsTest(GitVcsTest, HttpVcsTest):
     pass
 
 class HgSshVcsTest(HgVcsTest, SshVcsTest):
     @staticmethod
     def _ssh_param(repo_name, user, ssh_key, client_ip):
         # Specify a custom ssh command on the command line
         return r"""--config ui.ssh="bash -c 'SSH_ORIGINAL_COMMAND=\"\$2\" SSH_CONNECTION=\"%s 1024 127.0.0.1 22\" kallithea-cli ssh-serve -c %s %s %s' --" ssh://someuser@somehost/%s""" % (
             client_ip,
-            CONFIG['__file__'],
+            kallithea.CONFIG['__file__'],
             user.user_id,
             ssh_key.user_ssh_key_id,
             repo_name)
 
 class GitSshVcsTest(GitVcsTest, SshVcsTest):
     @staticmethod
     def _ssh_param(repo_name, user, ssh_key, client_ip):
         # Set a custom ssh command in the global environment
         os.environ['GIT_SSH_COMMAND'] = r"""bash -c 'SSH_ORIGINAL_COMMAND="$2" SSH_CONNECTION="%s 1024 127.0.0.1 22" kallithea-cli ssh-serve -c %s %s %s' --""" % (
             client_ip,
-            CONFIG['__file__'],
+            kallithea.CONFIG['__file__'],
             user.user_id,
             ssh_key.user_ssh_key_id)
         return "ssh://someuser@somehost/%s""" % repo_name
 
 parametrize_vcs_test = base.parametrize('vt', [
     HgHttpVcsTest,
     GitHttpVcsTest,
     HgSshVcsTest,
     GitSshVcsTest,
 ])
 parametrize_vcs_test_hg = base.parametrize('vt', [
     HgHttpVcsTest,
     HgSshVcsTest,
 ])
 parametrize_vcs_test_http = base.parametrize('vt', [
     HgHttpVcsTest,
     GitHttpVcsTest,
 ])
 
 class Command(object):
 
     def __init__(self, cwd):
         self.cwd = cwd
 
     def execute(self, *args, **environ):
         """
         Runs command on the system with given ``args`` using simple space
         join without safe quoting.
         """
         command = ' '.join(args)
         ignoreReturnCode = environ.pop('ignoreReturnCode', False)
         if DEBUG:
             print('*** CMD %s ***' % command)
         testenv = dict(os.environ)
         testenv['LANG'] = 'en_US.UTF-8'
         testenv['LANGUAGE'] = 'en_US:en'
         testenv['HGPLAIN'] = ''
         testenv['HGRCPATH'] = ''
         testenv.update(environ)
         p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE, cwd=self.cwd, env=testenv)
         stdout, stderr = p.communicate()
         if DEBUG:
             if stdout:
                 print('stdout:', stdout)
             if stderr:
                 print('stderr:', stderr)
         if not ignoreReturnCode:
             assert p.returncode == 0
         return safe_str(stdout), safe_str(stderr)
 
 
 def _get_tmp_dir(prefix='vcs_operations-', suffix=''):
     return tempfile.mkdtemp(dir=base.TESTS_TMP_PATH, prefix=prefix, suffix=suffix)
 
 
 def _add_files(vcs, dest_dir, files_no=3):
     """
     Generate some files, add it to dest_dir repo and push back
     vcs is git or hg and defines what VCS we want to make those files for
 
     :param vcs:
     :param dest_dir:
     """
     added_file = '%ssetup.py' % next(_RandomNameSequence())
     open(os.path.join(dest_dir, added_file), 'a').close()
     Command(dest_dir).execute(vcs, 'add', added_file)
 
     email = '[e-mail]'
     if os.name == 'nt':
         author_str = 'User <%s>' % email
     else:
         author_str = 'User ǝɯɐᴎ <%s>' % email
     for i in range(files_no):
         cmd = """echo "added_line%s" >> %s""" % (i, added_file)
         Command(dest_dir).execute(cmd)
         if vcs == 'hg':
             cmd = """hg commit -m "committed new %s" -u "%s" "%s" """ % (
                 i, author_str, added_file
             )
         elif vcs == 'git':
             cmd = """git commit -m "committed new %s" --author "%s" "%s" """ % (
                 i, author_str, added_file
             )
         # git commit needs EMAIL on some machines
         Command(dest_dir).execute(cmd, EMAIL=email)
 
 def _add_files_and_push(webserver, vt, dest_dir, clone_url, ignoreReturnCode=False, files_no=3):
     _add_files(vt.repo_type, dest_dir, files_no=files_no)
     # PUSH it back
     stdout = stderr = None
     if vt.repo_type == 'hg':
         stdout, stderr = Command(dest_dir).execute('hg push -f --verbose', clone_url, ignoreReturnCode=ignoreReturnCode)
     elif vt.repo_type == 'git':
         stdout, stderr = Command(dest_dir).execute('git push -f --verbose', clone_url, "master", ignoreReturnCode=ignoreReturnCode)
 
     return stdout, stderr
 
 
 def _check_outgoing(vcs, cwd, clone_url):
     if vcs == 'hg':
         # hg removes the password from default URLs, so we have to provide it here via the clone_url
         return Command(cwd).execute('hg -q outgoing', clone_url, ignoreReturnCode=True)
     elif vcs == 'git':
         Command(cwd).execute('git remote update')
         return Command(cwd).execute('git log origin/master..master')
 
 
 def set_anonymous_access(enable=True):
     user = User.get_default_user()
     user.active = enable
     Session().commit()
     if enable != User.get_default_user().active:
         raise Exception('Cannot set anonymous access')
 
 
 #==============================================================================
 # TESTS
 #==============================================================================
 
 
 def _check_proper_git_push(stdout, stderr):
     assert 'fatal' not in stderr
     assert 'rejected' not in stderr
     assert 'Pushing to' in stderr
     assert 'master -> master' in stderr
 
 
 @pytest.mark.usefixtures("test_context_fixture")
 class TestVCSOperations(base.TestController):
 
     @classmethod
     def setup_class(cls):
         # DISABLE ANONYMOUS ACCESS
         set_anonymous_access(False)
 
     @pytest.fixture()
     def testhook_cleanup(self):
         yield
         # remove hook
         for hook in ['prechangegroup', 'pretxnchangegroup', 'preoutgoing', 'changegroup', 'outgoing', 'incoming']:
             entry = Ui.get_by_key('hooks', '%s.testhook' % hook)
             if entry:
                 Session().delete(entry)
         Session().commit()
 
     @pytest.fixture(scope="module")
     def testfork(self):
         # create fork so the repo stays untouched
         git_fork_name = '%s_fork%s' % (base.GIT_REPO, next(_RandomNameSequence()))
         fixture.create_fork(base.GIT_REPO, git_fork_name)
         hg_fork_name = '%s_fork%s' % (base.HG_REPO, next(_RandomNameSequence()))
         fixture.create_fork(base.HG_REPO, hg_fork_name)
         return {'git': git_fork_name, 'hg': hg_fork_name}
 
     @parametrize_vcs_test
     def test_clone_repo_by_admin(self, webserver, vt):
         clone_url = vt.repo_url_param(webserver, vt.repo_name)
         stdout, stderr = Command(base.TESTS_TMP_PATH).execute(vt.repo_type, 'clone', clone_url, _get_tmp_dir())
 
         if vt.repo_type == 'git':
             assert 'Cloning into' in stdout + stderr
             assert stderr == '' or stdout == ''
         elif vt.repo_type == 'hg':
             assert 'requesting all changes' in stdout
             assert 'adding changesets' in stdout
             assert 'adding manifests' in stdout
             assert 'adding file changes' in stdout
             assert stderr == ''
 
     @parametrize_vcs_test_http
     def test_clone_wrong_credentials(self, webserver, vt):
         clone_url = vt.repo_url_param(webserver, vt.repo_name, password='bad!')
         stdout, stderr = Command(base.TESTS_TMP_PATH).execute(vt.repo_type, 'clone', clone_url, _get_tmp_dir(), ignoreReturnCode=True)
         if vt.repo_type == 'git':
             assert 'fatal: Authentication failed' in stderr
         elif vt.repo_type == 'hg':
             assert 'abort: authorization failed' in stderr
 
     def test_clone_git_dir_as_hg(self, webserver):
         clone_url = HgHttpVcsTest.repo_url_param(webserver, base.GIT_REPO)
         stdout, stderr = Command(base.TESTS_TMP_PATH).execute('hg clone', clone_url, _get_tmp_dir(), ignoreReturnCode=True)
         assert 'HTTP Error 404: Not Found' in stderr or "not a valid repository" in stdout and 'abort:' in stderr
 
     def test_clone_hg_repo_as_git(self, webserver):
         clone_url = GitHttpVcsTest.repo_url_param(webserver, base.HG_REPO)
         stdout, stderr = Command(base.TESTS_TMP_PATH).execute('git clone', clone_url, _get_tmp_dir(), ignoreReturnCode=True)
         assert 'not found' in stderr
 
     @parametrize_vcs_test
     def test_clone_non_existing_path(self, webserver, vt):
         clone_url = vt.repo_url_param(webserver, 'trololo')
         stdout, stderr = Command(base.TESTS_TMP_PATH).execute(vt.repo_type, 'clone', clone_url, _get_tmp_dir(), ignoreReturnCode=True)
         if vt.repo_type == 'git':
             assert 'not found' in stderr or 'abort: Access to %r denied' % 'trololo' in stderr
         elif vt.repo_type == 'hg':
             assert 'HTTP Error 404: Not Found' in stderr or 'abort: no suitable response from remote hg' in stderr and 'remote: abort: Access to %r denied' % 'trololo' in stdout + stderr
 
     @parametrize_vcs_test
     def test_push_new_repo(self, webserver, vt):
         # Clear the log so we know what is added
         UserLog.query().delete()
         Session().commit()
 
         # Create an empty server repo using the API
         repo_name = 'new_%s_%s' % (vt.repo_type, next(_RandomNameSequence()))
         usr = User.get_by_username(base.TEST_USER_ADMIN_LOGIN)
         params = {
             "id": 7,
             "api_key": usr.api_key,
             "method": 'create_repo',
             "args": dict(repo_name=repo_name,
                          owner=base.TEST_USER_ADMIN_LOGIN,
                          repo_type=vt.repo_type),
         }
         req = urllib.request.Request(
             'http://%s:%s/_admin/api' % webserver.server_address,
             data=ascii_bytes(json.dumps(params)),
             headers={'content-type': 'application/json'})
         response = urllib.request.urlopen(req)
         result = json.loads(response.read())
         # Expect something like:
         # {u'result': {u'msg': u'Created new repository `new_XXX`', u'task': None, u'success': True}, u'id': 7, u'error': None}
         assert result['result']['success']
 
         # Create local clone of the empty server repo
         local_clone_dir = _get_tmp_dir()
         clone_url = vt.repo_url_param(webserver, repo_name)
         stdout, stderr = Command(base.TESTS_TMP_PATH).execute(vt.repo_type, 'clone', clone_url, local_clone_dir)
 
         # Make 3 commits and push to the empty server repo.
         # The server repo doesn't have any other heads than the
         # refs/heads/master we are pushing, but the `git log` in the push hook
         # should still list the 3 commits.
         stdout, stderr = _add_files_and_push(webserver, vt, local_clone_dir, clone_url=clone_url)
         if vt.repo_type == 'git':
             _check_proper_git_push(stdout, stderr)
         elif vt.repo_type == 'hg':
             assert 'pushing to ' in stdout
             assert 'remote: added ' in stdout
 
         # Verify that we got the right events in UserLog. Expect something like:
         # <UserLog('id:new_git_XXX:started_following_repo')>
         # <UserLog('id:new_git_XXX:user_created_repo')>
         # <UserLog('id:new_git_XXX:pull')>
         # <UserLog('id:new_git_XXX:push:aed9d4c1732a1927da3be42c47eb9afdc200d427,d38b083a07af10a9f44193486959a96a23db78da,4841ff9a2b385bec995f4679ef649adb3f437622')>
         Session.close()  # make sure SA fetches all new log entries (apparently only needed for MariaDB/MySQL ...)
         action_parts = [ul.action.split(':', 1) for ul in UserLog.query().order_by(UserLog.user_log_id)]
         assert [(t[0], (t[1].count(',') + 1) if len(t) == 2 else 0) for t in action_parts] == ([
             ('started_following_repo', 0),
             ('user_created_repo', 0),
             ('pull', 0),
             ('push', 3)]
             if vt.repo_type == 'git' else [
             ('started_following_repo', 0),
             ('user_created_repo', 0),
             # (u'pull', 0), # Mercurial outgoing hook is not called for empty clones
             ('push', 3)])
 
     @parametrize_vcs_test
     def test_push_new_file(self, webserver, testfork, vt):
         UserLog.query().delete()
         Session().commit()
 
         dest_dir = _get_tmp_dir()
         clone_url = vt.repo_url_param(webserver, vt.repo_name)
         stdout, stderr = Command(base.TESTS_TMP_PATH).execute(vt.repo_type, 'clone', clone_url, dest_dir)
 
         clone_url = vt.repo_url_param(webserver, testfork[vt.repo_type])
         stdout, stderr = _add_files_and_push(webserver, vt, dest_dir, clone_url=clone_url)
 
         if vt.repo_type == 'git':
             _check_proper_git_push(stdout, stderr)
         elif vt.repo_type == 'hg':
             assert 'pushing to' in stdout
             assert 'Repository size' in stdout
             assert 'Last revision is now' in stdout
 
         Session.close()  # make sure SA fetches all new log entries (apparently only needed for MariaDB/MySQL ...)
         action_parts = [ul.action.split(':', 1) for ul in UserLog.query().order_by(UserLog.user_log_id)]
         assert [(t[0], (t[1].count(',') + 1) if len(t) == 2 else 0) for t in action_parts] == \
             [('pull', 0), ('push', 3)]
 
     @parametrize_vcs_test
     def test_pull(self, webserver, testfork, vt):
         UserLog.query().delete()
         Session().commit()
 
         dest_dir = _get_tmp_dir()
         stdout, stderr = Command(base.TESTS_TMP_PATH).execute(vt.repo_type, 'init', dest_dir)
 
         clone_url = vt.repo_url_param(webserver, vt.repo_name)
         stdout, stderr = Command(dest_dir).execute(vt.repo_type, 'pull', clone_url)
         Session.close()  # make sure SA fetches all new log entries (apparently only needed for MariaDB/MySQL ...)
 
         if vt.repo_type == 'git':
             assert 'FETCH_HEAD' in stderr
         elif vt.repo_type == 'hg':
             assert 'new changesets' in stdout
 
         action_parts = [ul.action for ul in UserLog.query().order_by(UserLog.user_log_id)]
         assert action_parts == ['pull']
 
         # Test handling of URLs with extra '/' around repo_name
         stdout, stderr = Command(dest_dir).execute(vt.repo_type, 'pull', clone_url.replace('/' + vt.repo_name, '/./%s/' % vt.repo_name), ignoreReturnCode=True)
         if issubclass(vt, HttpVcsTest):
             if vt.repo_type == 'git':
                 # NOTE: when pulling from http://hostname/./vcs_test_git/ , the git client will normalize that and issue an HTTP request to /vcs_test_git/info/refs
                 assert 'Already up to date.' in stdout
             else:
                 assert vt.repo_type == 'hg'
                 assert "abort: HTTP Error 404: Not Found" in stderr
         else:
             assert issubclass(vt, SshVcsTest)
             if vt.repo_type == 'git':
                 assert "abort: Access to './%s' denied" % vt.repo_name in stderr
             else:
                 assert "abort: Access to './%s' denied" % vt.repo_name in stdout + stderr
 
         stdout, stderr = Command(dest_dir).execute(vt.repo_type, 'pull', clone_url.replace('/' + vt.repo_name, '/%s/' % vt.repo_name), ignoreReturnCode=True)
         if vt.repo_type == 'git':
             assert 'Already up to date.' in stdout
         else:
             assert vt.repo_type == 'hg'
             assert "no changes found" in stdout
         assert "denied" not in stderr
         assert "denied" not in stdout
         assert "404" not in stdout
 
     @parametrize_vcs_test
     def test_push_invalidates_cache(self, webserver, testfork, vt):
         pre_cached_tip = [repo.get_api_data()['last_changeset']['short_id'] for repo in Repository.query().filter(Repository.repo_name == testfork[vt.repo_type])]
 
         dest_dir = _get_tmp_dir()
         clone_url = vt.repo_url_param(webserver, testfork[vt.repo_type])
         stdout, stderr = Command(base.TESTS_TMP_PATH).execute(vt.repo_type, 'clone', clone_url, dest_dir)
 
         stdout, stderr = _add_files_and_push(webserver, vt, dest_dir, files_no=1, clone_url=clone_url)
 
         if vt.repo_type == 'git':
             _check_proper_git_push(stdout, stderr)
 
         Session.close()  # expire session to make sure SA fetches new Repository instances after last_changeset has been updated by server side hook in another process
         post_cached_tip = [repo.get_api_data()['last_changeset']['short_id'] for repo in Repository.query().filter(Repository.repo_name == testfork[vt.repo_type])]
         assert pre_cached_tip != post_cached_tip
 
     @parametrize_vcs_test_http
     def test_push_wrong_credentials(self, webserver, vt):
         dest_dir = _get_tmp_dir()
         clone_url = vt.repo_url_param(webserver, vt.repo_name)
         stdout, stderr = Command(base.TESTS_TMP_PATH).execute(vt.repo_type, 'clone', clone_url, dest_dir)
 
         clone_url = webserver.repo_url(vt.repo_name, username='bad', password='name')
         stdout, stderr = _add_files_and_push(webserver, vt, dest_dir,
                                              clone_url=clone_url, ignoreReturnCode=True)
 
         if vt.repo_type == 'git':
             assert 'fatal: Authentication failed' in stderr
         elif vt.repo_type == 'hg':
             assert 'abort: authorization failed' in stderr
 
     @parametrize_vcs_test
     def test_push_with_readonly_credentials(self, webserver, vt):
         UserLog.query().delete()
         Session().commit()
 
         dest_dir = _get_tmp_dir()
         clone_url = vt.repo_url_param(webserver, vt.repo_name, username=base.TEST_USER_REGULAR_LOGIN, password=base.TEST_USER_REGULAR_PASS)
         stdout, stderr = Command(base.TESTS_TMP_PATH).execute(vt.repo_type, 'clone', clone_url, dest_dir)
 
         stdout, stderr = _add_files_and_push(webserver, vt, dest_dir, ignoreReturnCode=True, clone_url=clone_url)
 
         if vt.repo_type == 'git':
             assert 'The requested URL returned error: 403' in stderr or 'abort: Push access to %r denied' % str(vt.repo_name) in stderr
         elif vt.repo_type == 'hg':
             assert 'abort: HTTP Error 403: Forbidden' in stderr or 'abort: push failed on remote' in stderr and 'remote: Push access to %r denied' % str(vt.repo_name) in stdout
 
         Session.close()  # make sure SA fetches all new log entries (apparently only needed for MariaDB/MySQL ...)
         action_parts = [ul.action.split(':', 1) for ul in UserLog.query().order_by(UserLog.user_log_id)]
         assert [(t[0], (t[1].count(',') + 1) if len(t) == 2 else 0) for t in action_parts] == \
             [('pull', 0)]
 
     @parametrize_vcs_test
     def test_push_back_to_wrong_url(self, webserver, vt):
         dest_dir = _get_tmp_dir()
         clone_url = vt.repo_url_param(webserver, vt.repo_name)
         stdout, stderr = Command(base.TESTS_TMP_PATH).execute(vt.repo_type, 'clone', clone_url, dest_dir)
 
         stdout, stderr = _add_files_and_push(
             webserver, vt, dest_dir, clone_url='http://%s:%s/tmp' % (
                 webserver.server_address[0], webserver.server_address[1]),
             ignoreReturnCode=True)
 
         if vt.repo_type == 'git':
             assert 'not found' in stderr
         elif vt.repo_type == 'hg':
             assert 'HTTP Error 404: Not Found' in stderr
 
     @parametrize_vcs_test
     def test_ip_restriction(self, webserver, vt):
         user_model = UserModel()
         try:
             # Add IP constraint that excludes the test context:
             user_model.add_extra_ip(base.TEST_USER_ADMIN_LOGIN, '10.10.10.10/32')
             Session().commit()
             # IP permissions are cached, need to wait for the cache in the server process to expire
             time.sleep(1.5)
             clone_url = vt.repo_url_param(webserver, vt.repo_name)
             stdout, stderr = Command(base.TESTS_TMP_PATH).execute(vt.repo_type, 'clone', clone_url, _get_tmp_dir(), ignoreReturnCode=True)
             if vt.repo_type == 'git':
                 # The message apparently changed in Git 1.8.3, so match it loosely.
                 assert re.search(r'\b403\b', stderr) or 'abort: User test_admin from 127.0.0.127 cannot be authorized' in stderr
             elif vt.repo_type == 'hg':
                 assert 'abort: HTTP Error 403: Forbidden' in stderr or 'remote: abort: User test_admin from 127.0.0.127 cannot be authorized' in stdout + stderr
         finally:
             # release IP restrictions
             for ip in UserIpMap.query():
                 UserIpMap.delete(ip.ip_id)
             Session().commit()
             # IP permissions are cached, need to wait for the cache in the server process to expire
             time.sleep(1.5)
 
         clone_url = vt.repo_url_param(webserver, vt.repo_name)
         stdout, stderr = Command(base.TESTS_TMP_PATH).execute(vt.repo_type, 'clone', clone_url, _get_tmp_dir())
 
         if vt.repo_type == 'git':
             assert 'Cloning into' in stdout + stderr
             assert stderr == '' or stdout == ''
         elif vt.repo_type == 'hg':
             assert 'requesting all changes' in stdout
             assert 'adding changesets' in stdout
             assert 'adding manifests' in stdout
             assert 'adding file changes' in stdout
 
             assert stderr == ''
 
     @parametrize_vcs_test_hg # git hooks doesn't work like hg hooks
     def test_custom_hooks_preoutgoing(self, testhook_cleanup, webserver, testfork, vt):
         # set prechangegroup to failing hook (returns True)
         Ui.create_or_update_hook('preoutgoing.testhook', 'python:kallithea.tests.fixture.failing_test_hook')
         Session().commit()
         # clone repo
         clone_url = vt.repo_url_param(webserver, testfork[vt.repo_type], username=base.TEST_USER_ADMIN_LOGIN, password=base.TEST_USER_ADMIN_PASS)
         dest_dir = _get_tmp_dir()
         stdout, stderr = Command(base.TESTS_TMP_PATH) \
             .execute(vt.repo_type, 'clone', clone_url, dest_dir, ignoreReturnCode=True)
         if vt.repo_type == 'hg':
             assert 'preoutgoing.testhook hook failed' in stdout
         elif vt.repo_type == 'git':
             assert 'error: 406' in stderr
 
     @parametrize_vcs_test_hg # git hooks doesn't work like hg hooks
     def test_custom_hooks_prechangegroup(self, testhook_cleanup, webserver, testfork, vt):
         # set prechangegroup to failing hook (returns exit code 1)
         Ui.create_or_update_hook('prechangegroup.testhook', 'python:kallithea.tests.fixture.failing_test_hook')
         Session().commit()
         # clone repo
         clone_url = vt.repo_url_param(webserver, testfork[vt.repo_type], username=base.TEST_USER_ADMIN_LOGIN, password=base.TEST_USER_ADMIN_PASS)
         dest_dir = _get_tmp_dir()
         stdout, stderr = Command(base.TESTS_TMP_PATH).execute(vt.repo_type, 'clone', clone_url, dest_dir)
 
         stdout, stderr = _add_files_and_push(webserver, vt, dest_dir, clone_url,
                                              ignoreReturnCode=True)
         assert 'failing_test_hook failed' in stdout + stderr
         assert 'Traceback' not in stdout + stderr
         assert 'prechangegroup.testhook hook failed' in stdout + stderr
         # there are still outgoing changesets
         stdout, stderr = _check_outgoing(vt.repo_type, dest_dir, clone_url)
         assert stdout != ''
 
         # set prechangegroup hook to exception throwing method
         Ui.create_or_update_hook('prechangegroup.testhook', 'python:kallithea.tests.fixture.exception_test_hook')
         Session().commit()
         # re-try to push
         stdout, stderr = Command(dest_dir).execute('%s push' % vt.repo_type, clone_url, ignoreReturnCode=True)
         if vt is HgHttpVcsTest:
             # like with 'hg serve...' 'HTTP Error 500: INTERNAL SERVER ERROR' should be returned
             assert 'HTTP Error 500: INTERNAL SERVER ERROR' in stderr
         elif vt is HgSshVcsTest:
             assert 'remote: Exception: exception_test_hook threw an exception' in stdout
         else: assert False
         # there are still outgoing changesets
         stdout, stderr = _check_outgoing(vt.repo_type, dest_dir, clone_url)
         assert stdout != ''
 
         # set prechangegroup hook to method that returns False
         Ui.create_or_update_hook('prechangegroup.testhook', 'python:kallithea.tests.fixture.passing_test_hook')
         Session().commit()
         # re-try to push
         stdout, stderr = Command(dest_dir).execute('%s push' % vt.repo_type, clone_url, ignoreReturnCode=True)
         assert 'passing_test_hook succeeded' in stdout + stderr
         assert 'Traceback' not in stdout + stderr
         assert 'prechangegroup.testhook hook failed' not in stdout + stderr
         # no more outgoing changesets
         stdout, stderr = _check_outgoing(vt.repo_type, dest_dir, clone_url)
         assert stdout == ''
         assert stderr == ''
 
     def test_add_submodule_git(self, webserver, testfork):
         dest_dir = _get_tmp_dir()
         clone_url = GitHttpVcsTest.repo_url_param(webserver, base.GIT_REPO)
 
         fork_url = GitHttpVcsTest.repo_url_param(webserver, testfork['git'])
 
         # add submodule
         stdout, stderr = Command(base.TESTS_TMP_PATH).execute('git clone', fork_url, dest_dir)
         stdout, stderr = Command(dest_dir).execute('git submodule add', clone_url, 'testsubmodule')
         stdout, stderr = Command(dest_dir).execute('git commit -am "added testsubmodule pointing to', clone_url, '"', EMAIL=base.TEST_USER_ADMIN_EMAIL)
         stdout, stderr = Command(dest_dir).execute('git push', fork_url, 'master')
 
         # check for testsubmodule link in files page
         self.log_user()
         response = self.app.get(base.url(controller='files', action='index',
                                     repo_name=testfork['git'],
                                     revision='tip',
                                     f_path='/'))
         # check _repo_files_url that will be used to reload as AJAX
         response.mustcontain('var _repo_files_url = ("/%s/files/");' % testfork['git'])
 
         response.mustcontain('<a class="submodule-dir" href="%s" target="_blank"><i class="icon-file-submodule"></i><span>testsubmodule @ ' % clone_url)
 
         # check that following a submodule link actually works - and redirects
         response = self.app.get(base.url(controller='files', action='index',
                                     repo_name=testfork['git'],
                                     revision='tip',
                                     f_path='/testsubmodule'),
                                 status=302)
         assert response.location == clone_url