vcs: always return bytes from node.content

We will rather have the unicode conversions explicit.

Note: Py3 bytes doesn't have .startswith - replace that with a regexp.

kallithea/controllers/admin/gists.py

@@ -1,264 +1,264 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.controllers.admin.gists
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 gist controller for Kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: May 9, 2013
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 import logging
 import traceback
 
 import formencode.htmlfill
 from sqlalchemy.sql.expression import or_
 from tg import request, response
 from tg import tmpl_context as c
 from tg.i18n import ugettext as _
 from webob.exc import HTTPForbidden, HTTPFound, HTTPNotFound
 
 from kallithea.config.routing import url
 from kallithea.lib import helpers as h
 from kallithea.lib.auth import LoginRequired
 from kallithea.lib.base import BaseController, jsonify, render
 from kallithea.lib.page import Page
 from kallithea.lib.utils2 import safe_int, safe_unicode, time_to_datetime
 from kallithea.lib.vcs.exceptions import NodeNotChangedError, VCSError
 from kallithea.model.db import Gist
 from kallithea.model.forms import GistForm
 from kallithea.model.gist import GistModel
 from kallithea.model.meta import Session
 
 
 log = logging.getLogger(__name__)
 
 
 class GistsController(BaseController):
     """REST Controller styled on the Atom Publishing Protocol"""
 
     def __load_defaults(self, extra_values=None):
         c.lifetime_values = [
             (str(-1), _('Forever')),
             (str(5), _('5 minutes')),
             (str(60), _('1 hour')),
             (str(60 * 24), _('1 day')),
             (str(60 * 24 * 30), _('1 month')),
         ]
         if extra_values:
             c.lifetime_values.append(extra_values)
         c.lifetime_options = [(c.lifetime_values, _("Lifetime"))]
 
     @LoginRequired(allow_default_user=True)
     def index(self):
         not_default_user = not request.authuser.is_default_user
         c.show_private = request.GET.get('private') and not_default_user
         c.show_public = request.GET.get('public') and not_default_user
         url_params = {}
         if c.show_public:
             url_params['public'] = 1
         elif c.show_private:
             url_params['private'] = 1
 
         gists = Gist().query() \
             .filter_by(is_expired=False) \
             .order_by(Gist.created_on.desc())
 
         # MY private
         if c.show_private and not c.show_public:
             gists = gists.filter(Gist.gist_type == Gist.GIST_PRIVATE) \
                              .filter(Gist.owner_id == request.authuser.user_id)
         # MY public
         elif c.show_public and not c.show_private:
             gists = gists.filter(Gist.gist_type == Gist.GIST_PUBLIC) \
                              .filter(Gist.owner_id == request.authuser.user_id)
 
         # MY public+private
         elif c.show_private and c.show_public:
             gists = gists.filter(or_(Gist.gist_type == Gist.GIST_PUBLIC,
                                      Gist.gist_type == Gist.GIST_PRIVATE)) \
                              .filter(Gist.owner_id == request.authuser.user_id)
 
         # default show ALL public gists
         if not c.show_public and not c.show_private:
             gists = gists.filter(Gist.gist_type == Gist.GIST_PUBLIC)
 
         c.gists = gists
         p = safe_int(request.GET.get('page'), 1)
         c.gists_pager = Page(c.gists, page=p, items_per_page=10,
                              **url_params)
         return render('admin/gists/index.html')
 
     @LoginRequired()
     def create(self):
         self.__load_defaults()
         gist_form = GistForm([x[0] for x in c.lifetime_values])()
         try:
             form_result = gist_form.to_python(dict(request.POST))
             # TODO: multiple files support, from the form
             filename = form_result['filename'] or Gist.DEFAULT_FILENAME
             nodes = {
                 filename: {
                     'content': form_result['content'],
                     'lexer': form_result['mimetype']  # None is autodetect
                 }
             }
             _public = form_result['public']
             gist_type = Gist.GIST_PUBLIC if _public else Gist.GIST_PRIVATE
             gist = GistModel().create(
                 description=form_result['description'],
                 owner=request.authuser.user_id,
                 ip_addr=request.ip_addr,
                 gist_mapping=nodes,
                 gist_type=gist_type,
                 lifetime=form_result['lifetime']
             )
             Session().commit()
             new_gist_id = gist.gist_access_id
         except formencode.Invalid as errors:
             defaults = errors.value
 
             return formencode.htmlfill.render(
                 render('admin/gists/new.html'),
                 defaults=defaults,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False)
 
         except Exception as e:
             log.error(traceback.format_exc())
             h.flash(_('Error occurred during gist creation'), category='error')
             raise HTTPFound(location=url('new_gist'))
         raise HTTPFound(location=url('gist', gist_id=new_gist_id))
 
     @LoginRequired()
     def new(self, format='html'):
         self.__load_defaults()
         return render('admin/gists/new.html')
 
     @LoginRequired()
     def delete(self, gist_id):
         gist = GistModel().get_gist(gist_id)
         owner = gist.owner_id == request.authuser.user_id
         if h.HasPermissionAny('hg.admin')() or owner:
             GistModel().delete(gist)
             Session().commit()
             h.flash(_('Deleted gist %s') % gist.gist_access_id, category='success')
         else:
             raise HTTPForbidden()
 
         raise HTTPFound(location=url('gists'))
 
     @LoginRequired(allow_default_user=True)
     def show(self, gist_id, revision='tip', format='html', f_path=None):
         c.gist = Gist.get_or_404(gist_id)
 
         if c.gist.is_expired:
             log.error('Gist expired at %s',
                       time_to_datetime(c.gist.gist_expires))
             raise HTTPNotFound()
         try:
             c.file_changeset, c.files = GistModel().get_gist_files(gist_id,
                                                             revision=revision)
         except VCSError:
             log.error(traceback.format_exc())
             raise HTTPNotFound()
         if format == 'raw':
-            content = '\n\n'.join([f.content for f in c.files if (f_path is None or safe_unicode(f.path) == f_path)])
+            content = '\n\n'.join([safe_unicode(f.content) for f in c.files if (f_path is None or safe_unicode(f.path) == f_path)])
             response.content_type = 'text/plain'
             return content
         return render('admin/gists/show.html')
 
     @LoginRequired()
     def edit(self, gist_id, format='html'):
         c.gist = Gist.get_or_404(gist_id)
 
         if c.gist.is_expired:
             log.error('Gist expired at %s',
                       time_to_datetime(c.gist.gist_expires))
             raise HTTPNotFound()
         try:
             c.file_changeset, c.files = GistModel().get_gist_files(gist_id)
         except VCSError:
             log.error(traceback.format_exc())
             raise HTTPNotFound()
 
         self.__load_defaults(extra_values=('0', _('Unmodified')))
         rendered = render('admin/gists/edit.html')
 
         if request.POST:
             rpost = request.POST
             nodes = {}
             for org_filename, filename, mimetype, content in zip(
                                                     rpost.getall('org_files'),
                                                     rpost.getall('files'),
                                                     rpost.getall('mimetypes'),
                                                     rpost.getall('contents')):
 
                 nodes[org_filename] = {
                     'org_filename': org_filename,
                     'filename': filename,
                     'content': content,
                     'lexer': mimetype,
                 }
             try:
                 GistModel().update(
                     gist=c.gist,
                     description=rpost['description'],
                     owner=c.gist.owner, # FIXME: request.authuser.user_id ?
                     ip_addr=request.ip_addr,
                     gist_mapping=nodes,
                     gist_type=c.gist.gist_type,
                     lifetime=rpost['lifetime']
                 )
 
                 Session().commit()
                 h.flash(_('Successfully updated gist content'), category='success')
             except NodeNotChangedError:
                 # raised if nothing was changed in repo itself. We anyway then
                 # store only DB stuff for gist
                 Session().commit()
                 h.flash(_('Successfully updated gist data'), category='success')
             except Exception:
                 log.error(traceback.format_exc())
                 h.flash(_('Error occurred during update of gist %s') % gist_id,
                         category='error')
 
             raise HTTPFound(location=url('gist', gist_id=gist_id))
 
         return rendered
 
     @LoginRequired()
     @jsonify
     def check_revision(self, gist_id):
         c.gist = Gist.get_or_404(gist_id)
         last_rev = c.gist.scm_instance.get_changeset()
         success = True
         revision = request.POST.get('revision')
 
         # TODO: maybe move this to model ?
         if revision != last_rev.raw_id:
             log.error('Last revision %s is different than submitted %s',
                       revision, last_rev)
             # our gist has newer version than we
             success = False
 
         return {'success': success}

kallithea/controllers/compare.py

@@ -1,289 +1,289 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.controllers.compare
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 compare controller showing differences between two
 repos, branches, bookmarks or tips
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: May 6, 2012
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 
 import logging
 import re
 
 from tg import request
 from tg import tmpl_context as c
 from tg.i18n import ugettext as _
 from webob.exc import HTTPBadRequest, HTTPFound, HTTPNotFound
 
 from kallithea.config.routing import url
 from kallithea.controllers.changeset import _context_url, _ignorews_url
 from kallithea.lib import diffs
 from kallithea.lib import helpers as h
 from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired
 from kallithea.lib.base import BaseRepoController, render
 from kallithea.lib.graphmod import graph_data
 from kallithea.lib.utils2 import safe_int, safe_str
 from kallithea.lib.vcs.utils.hgcompat import unionrepo
 from kallithea.model.db import Repository
 
 
 log = logging.getLogger(__name__)
 
 
 class CompareController(BaseRepoController):
 
     def _before(self, *args, **kwargs):
         super(CompareController, self)._before(*args, **kwargs)
 
         # The base repository has already been retrieved.
         c.a_repo = c.db_repo
 
         # Retrieve the "changeset" repository (default: same as base).
         other_repo = request.GET.get('other_repo', None)
         if other_repo is None:
             c.cs_repo = c.a_repo
         else:
             c.cs_repo = Repository.get_by_repo_name(other_repo)
             if c.cs_repo is None:
                 msg = _('Could not find other repository %s') % other_repo
                 h.flash(msg, category='error')
                 raise HTTPFound(location=url('compare_home', repo_name=c.a_repo.repo_name))
 
         # Verify that it's even possible to compare these two repositories.
         if c.a_repo.scm_instance.alias != c.cs_repo.scm_instance.alias:
             msg = _('Cannot compare repositories of different types')
             h.flash(msg, category='error')
             raise HTTPFound(location=url('compare_home', repo_name=c.a_repo.repo_name))
 
     @staticmethod
     def _get_changesets(alias, org_repo, org_rev, other_repo, other_rev):
         """
         Returns lists of changesets that can be merged from org_repo@org_rev
         to other_repo@other_rev
         ... and the other way
         ... and the ancestors that would be used for merge
 
         :param org_repo: repo object, that is most likely the original repo we forked from
         :param org_rev: the revision we want our compare to be made
         :param other_repo: repo object, most likely the fork of org_repo. It has
             all changesets that we need to obtain
         :param other_rev: revision we want out compare to be made on other_repo
         """
         ancestors = None
         if org_rev == other_rev:
             org_changesets = []
             other_changesets = []
 
         elif alias == 'hg':
             # case two independent repos
             if org_repo != other_repo:
                 hgrepo = unionrepo.makeunionrepository(other_repo.baseui,
                                                        other_repo.path,
                                                        org_repo.path)
                 # all ancestors of other_rev will be in other_repo and
                 # rev numbers from hgrepo can be used in other_repo - org_rev ancestors cannot
 
             # no remote compare do it on the same repository
             else:
                 hgrepo = other_repo._repo
 
             ancestors = [hgrepo[ancestor].hex() for ancestor in
                          hgrepo.revs("id(%s) & ::id(%s)", other_rev, org_rev)]
             if ancestors:
                 log.debug("shortcut found: %s is already an ancestor of %s", other_rev, org_rev)
             else:
                 log.debug("no shortcut found: %s is not an ancestor of %s", other_rev, org_rev)
                 ancestors = [hgrepo[ancestor].hex() for ancestor in
                              hgrepo.revs("heads(::id(%s) & ::id(%s))", org_rev, other_rev)] # FIXME: expensive!
 
             other_revs = hgrepo.revs("ancestors(id(%s)) and not ancestors(id(%s)) and not id(%s)",
                                      other_rev, org_rev, org_rev)
             other_changesets = [other_repo.get_changeset(rev) for rev in other_revs]
             org_revs = hgrepo.revs("ancestors(id(%s)) and not ancestors(id(%s)) and not id(%s)",
                                    org_rev, other_rev, other_rev)
             org_changesets = [org_repo.get_changeset(hgrepo[rev].hex()) for rev in org_revs]
 
         elif alias == 'git':
             if org_repo != other_repo:
                 from dulwich.repo import Repo
                 from dulwich.client import SubprocessGitClient
 
                 gitrepo = Repo(org_repo.path)
                 SubprocessGitClient(thin_packs=False).fetch(safe_str(other_repo.path), gitrepo)
 
                 gitrepo_remote = Repo(other_repo.path)
                 SubprocessGitClient(thin_packs=False).fetch(safe_str(org_repo.path), gitrepo_remote)
 
                 revs = [
                     x.commit.id
                     for x in gitrepo_remote.get_walker(include=[other_rev],
                                                        exclude=[org_rev])
                 ]
                 other_changesets = [other_repo.get_changeset(rev) for rev in reversed(revs)]
                 if other_changesets:
                     ancestors = [other_changesets[0].parents[0].raw_id]
                 else:
                     # no changesets from other repo, ancestor is the other_rev
                     ancestors = [other_rev]
 
                 gitrepo.close()
                 gitrepo_remote.close()
 
             else:
                 so = org_repo.run_git_command(
                     ['log', '--reverse', '--pretty=format:%H',
                      '-s', '%s..%s' % (org_rev, other_rev)]
                 )
                 other_changesets = [org_repo.get_changeset(cs)
                               for cs in re.findall(r'[0-9a-fA-F]{40}', so)]
                 so = org_repo.run_git_command(
                     ['merge-base', org_rev, other_rev]
                 )
                 ancestors = [re.findall(r'[0-9a-fA-F]{40}', so)[0]]
             org_changesets = []
 
         else:
             raise Exception('Bad alias only git and hg is allowed')
 
         return other_changesets, org_changesets, ancestors
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def index(self, repo_name):
         c.compare_home = True
         c.a_ref_name = c.cs_ref_name = None
         return render('compare/compare_diff.html')
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def compare(self, repo_name, org_ref_type, org_ref_name, other_ref_type, other_ref_name):
         org_ref_name = org_ref_name.strip()
         other_ref_name = other_ref_name.strip()
 
         # If merge is True:
         #   Show what org would get if merged with other:
         #   List changesets that are ancestors of other but not of org.
         #   New changesets in org is thus ignored.
         #   Diff will be from common ancestor, and merges of org to other will thus be ignored.
         # If merge is False:
         #   Make a raw diff from org to other, no matter if related or not.
         #   Changesets in one and not in the other will be ignored
         merge = bool(request.GET.get('merge'))
         # fulldiff disables cut_off_limit
         fulldiff = request.GET.get('fulldiff')
         # partial uses compare_cs.html template directly
         partial = request.environ.get('HTTP_X_PARTIAL_XHR')
         # is_ajax_preview puts hidden input field with changeset revisions
         c.is_ajax_preview = partial and request.GET.get('is_ajax_preview')
         # swap url for compare_diff page - never partial and never is_ajax_preview
         c.swap_url = h.url('compare_url',
             repo_name=c.cs_repo.repo_name,
             org_ref_type=other_ref_type, org_ref_name=other_ref_name,
             other_repo=c.a_repo.repo_name,
             other_ref_type=org_ref_type, other_ref_name=org_ref_name,
             merge=merge or '')
 
         # set callbacks for generating markup for icons
         c.ignorews_url = _ignorews_url
         c.context_url = _context_url
         ignore_whitespace = request.GET.get('ignorews') == '1'
         line_context = safe_int(request.GET.get('context'), 3)
 
         c.a_rev = self._get_ref_rev(c.a_repo, org_ref_type, org_ref_name,
             returnempty=True)
         c.cs_rev = self._get_ref_rev(c.cs_repo, other_ref_type, other_ref_name)
 
         c.compare_home = False
         c.a_ref_name = org_ref_name
         c.a_ref_type = org_ref_type
         c.cs_ref_name = other_ref_name
         c.cs_ref_type = other_ref_type
 
         c.cs_ranges, c.cs_ranges_org, c.ancestors = self._get_changesets(
             c.a_repo.scm_instance.alias, c.a_repo.scm_instance, c.a_rev,
             c.cs_repo.scm_instance, c.cs_rev)
         raw_ids = [x.raw_id for x in c.cs_ranges]
         c.cs_comments = c.cs_repo.get_comments(raw_ids)
         c.cs_statuses = c.cs_repo.statuses(raw_ids)
 
         revs = [ctx.revision for ctx in reversed(c.cs_ranges)]
         c.jsdata = graph_data(c.cs_repo.scm_instance, revs)
 
         if partial:
             return render('compare/compare_cs.html')
 
         org_repo = c.a_repo
         other_repo = c.cs_repo
 
         if merge:
             rev1 = msg = None
             if not c.cs_ranges:
                 msg = _('Cannot show empty diff')
             elif not c.ancestors:
                 msg = _('No ancestor found for merge diff')
             elif len(c.ancestors) == 1:
                 rev1 = c.ancestors[0]
             else:
                 msg = _('Multiple merge ancestors found for merge compare')
             if rev1 is None:
                 h.flash(msg, category='error')
                 log.error(msg)
                 raise HTTPNotFound
 
             # case we want a simple diff without incoming changesets,
             # previewing what will be merged.
             # Make the diff on the other repo (which is known to have other_rev)
             log.debug('Using ancestor %s as rev1 instead of %s',
                       rev1, c.a_rev)
             org_repo = other_repo
         else: # comparing tips, not necessarily linearly related
             if org_repo != other_repo:
                 # TODO: we could do this by using hg unionrepo
                 log.error('cannot compare across repos %s and %s', org_repo, other_repo)
                 h.flash(_('Cannot compare repositories without using common ancestor'), category='error')
                 raise HTTPBadRequest
             rev1 = c.a_rev
 
         diff_limit = None if fulldiff else self.cut_off_limit
 
         log.debug('running diff between %s and %s in %s',
                   rev1, c.cs_rev, org_repo.scm_instance.path)
         raw_diff = diffs.get_diff(org_repo.scm_instance, rev1=rev1, rev2=c.cs_rev,
                                       ignore_whitespace=ignore_whitespace,
                                       context=line_context)
 
-        diff_processor = diffs.DiffProcessor(raw_diff or '', diff_limit=diff_limit)
+        diff_processor = diffs.DiffProcessor(raw_diff, diff_limit=diff_limit)
         c.limited_diff = diff_processor.limited_diff
         c.file_diff_data = []
         c.lines_added = 0
         c.lines_deleted = 0
         for f in diff_processor.parsed:
             st = f['stats']
             c.lines_added += st['added']
             c.lines_deleted += st['deleted']
             filename = f['filename']
             fid = h.FID('', filename)
             html_diff = diffs.as_html(enable_comments=False, parsed_lines=[f])
             c.file_diff_data.append((fid, None, f['operation'], f['old_filename'], filename, html_diff, st))
 
         return render('compare/compare_diff.html')

kallithea/controllers/feed.py

@@ -1,134 +1,134 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.controllers.feed
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Feed controller for Kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 23, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 
 import logging
 
 from beaker.cache import cache_region
 from tg import response
 from tg import tmpl_context as c
 from tg.i18n import ugettext as _
 
 from kallithea import CONFIG
 from kallithea.lib import feeds
 from kallithea.lib import helpers as h
 from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired
 from kallithea.lib.base import BaseRepoController
 from kallithea.lib.diffs import DiffProcessor
 from kallithea.lib.utils2 import safe_int, safe_unicode, str2bool
 
 
 log = logging.getLogger(__name__)
 
 
 class FeedController(BaseRepoController):
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def _before(self, *args, **kwargs):
         super(FeedController, self)._before(*args, **kwargs)
 
     def _get_title(self, cs):
         return h.shorter(cs.message, 160)
 
     def __get_desc(self, cs):
         desc_msg = [(_('%s committed on %s')
                      % (h.person(cs.author), h.fmt_date(cs.date))) + '<br/>']
         # branches, tags, bookmarks
         for branch in cs.branches:
             desc_msg.append('branch: %s<br/>' % branch)
         for book in cs.bookmarks:
             desc_msg.append('bookmark: %s<br/>' % book)
         for tag in cs.tags:
             desc_msg.append('tag: %s<br/>' % tag)
 
         changes = []
         diff_limit = safe_int(CONFIG.get('rss_cut_off_limit', 32 * 1024))
         raw_diff = cs.diff()
         diff_processor = DiffProcessor(raw_diff,
                                        diff_limit=diff_limit,
                                        inline_diff=False)
 
         for st in diff_processor.parsed:
             st.update({'added': st['stats']['added'],
                        'removed': st['stats']['deleted']})
             changes.append('\n %(operation)s %(filename)s '
                            '(%(added)s lines added, %(removed)s lines removed)'
                             % st)
         if diff_processor.limited_diff:
             changes = changes + ['\n ' +
                                  _('Changeset was too big and was cut off...')]
 
         # rev link
         _url = h.canonical_url('changeset_home', repo_name=c.db_repo.repo_name,
                    revision=cs.raw_id)
         desc_msg.append('changeset: <a href="%s">%s</a>' % (_url, cs.raw_id[:8]))
 
         desc_msg.append('<pre>')
         desc_msg.append(h.urlify_text(cs.message))
         desc_msg.append('\n')
         desc_msg.extend(changes)
         if str2bool(CONFIG.get('rss_include_diff', False)):
             desc_msg.append('\n\n')
-            desc_msg.append(raw_diff)
+            desc_msg.append(safe_unicode(raw_diff))
         desc_msg.append('</pre>')
         return [safe_unicode(chunk) for chunk in desc_msg]
 
     def _feed(self, repo_name, feeder):
         """Produce a simple feed"""
 
         @cache_region('long_term', '_get_feed_from_cache')
         def _get_feed_from_cache(*_cache_keys):  # parameters are not really used - only as caching key
             header = dict(
                 title=_('%s %s feed') % (c.site_name, repo_name),
                 link=h.canonical_url('summary_home', repo_name=repo_name),
                 description=_('Changes on %s repository') % repo_name,
             )
 
             rss_items_per_page = safe_int(CONFIG.get('rss_items_per_page', 20))
             entries=[]
             for cs in reversed(list(c.db_repo_scm_instance[-rss_items_per_page:])):
                 entries.append(dict(
                     title=self._get_title(cs),
                     link=h.canonical_url('changeset_home', repo_name=repo_name, revision=cs.raw_id),
                     author_email=cs.author_email,
                     author_name=cs.author_name,
                     description=''.join(self.__get_desc(cs)),
                     pubdate=cs.date,
                 ))
             return feeder.render(header, entries)
 
         response.content_type = feeder.content_type
         return _get_feed_from_cache(repo_name, feeder.__name__)
 
     def atom(self, repo_name):
         """Produce a simple atom-1.0 feed"""
         return self._feed(repo_name, feeds.AtomFeed)
 
     def rss(self, repo_name):
         """Produce a simple rss2 feed"""
         return self._feed(repo_name, feeds.RssFeed)

kallithea/controllers/files.py

@@ -1,757 +1,756 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.controllers.files
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Files controller for Kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 21, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 import logging
 import os
 import posixpath
 import shutil
 import tempfile
 import traceback
 from collections import OrderedDict
 
 from tg import request, response
 from tg import tmpl_context as c
 from tg.i18n import ugettext as _
 from webob.exc import HTTPFound, HTTPNotFound
 
 from kallithea.config.routing import url
 from kallithea.controllers.changeset import _context_url, _ignorews_url, anchor_url, get_ignore_ws, get_line_ctx
 from kallithea.lib import diffs
 from kallithea.lib import helpers as h
 from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired
 from kallithea.lib.base import BaseRepoController, jsonify, render
 from kallithea.lib.exceptions import NonRelativePathError
 from kallithea.lib.utils import action_logger
-from kallithea.lib.utils2 import convert_line_endings, detect_mode, safe_int, safe_str, str2bool
+from kallithea.lib.utils2 import convert_line_endings, detect_mode, safe_int, safe_str, safe_unicode, str2bool
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.lib.vcs.conf import settings
 from kallithea.lib.vcs.exceptions import (
     ChangesetDoesNotExistError, ChangesetError, EmptyRepositoryError, ImproperArchiveTypeError, NodeAlreadyExistsError, NodeDoesNotExistError, NodeError, RepositoryError, VCSError)
 from kallithea.lib.vcs.nodes import FileNode
 from kallithea.model.db import Repository
 from kallithea.model.repo import RepoModel
 from kallithea.model.scm import ScmModel
 
 
 log = logging.getLogger(__name__)
 
 
 class FilesController(BaseRepoController):
 
     def _before(self, *args, **kwargs):
         super(FilesController, self)._before(*args, **kwargs)
 
     def __get_cs(self, rev, silent_empty=False):
         """
         Safe way to get changeset if error occur it redirects to tip with
         proper message
 
         :param rev: revision to fetch
         :silent_empty: return None if repository is empty
         """
 
         try:
             return c.db_repo_scm_instance.get_changeset(rev)
         except EmptyRepositoryError as e:
             if silent_empty:
                 return None
             url_ = url('files_add_home',
                        repo_name=c.repo_name,
                        revision=0, f_path='', anchor='edit')
             add_new = h.link_to(_('Click here to add new file'), url_, class_="alert-link")
             h.flash(_('There are no files yet.') + ' ' + add_new, category='warning')
             raise HTTPNotFound()
         except (ChangesetDoesNotExistError, LookupError):
             msg = _('Such revision does not exist for this repository')
             h.flash(msg, category='error')
             raise HTTPNotFound()
         except RepositoryError as e:
             h.flash(unicode(e), category='error')
             raise HTTPNotFound()
 
     def __get_filenode(self, cs, path):
         """
         Returns file_node or raise HTTP error.
 
         :param cs: given changeset
         :param path: path to lookup
         """
 
         try:
             file_node = cs.get_node(path)
             if file_node.is_dir():
                 raise RepositoryError('given path is a directory')
         except ChangesetDoesNotExistError:
             msg = _('Such revision does not exist for this repository')
             h.flash(msg, category='error')
             raise HTTPNotFound()
         except RepositoryError as e:
             h.flash(unicode(e), category='error')
             raise HTTPNotFound()
 
         return file_node
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def index(self, repo_name, revision, f_path, annotate=False):
         # redirect to given revision from form if given
         post_revision = request.POST.get('at_rev', None)
         if post_revision:
             cs = self.__get_cs(post_revision) # FIXME - unused!
 
         c.revision = revision
         c.changeset = self.__get_cs(revision)
         c.branch = request.GET.get('branch', None)
         c.f_path = f_path
         c.annotate = annotate
         cur_rev = c.changeset.revision
         # used in files_source.html:
         c.cut_off_limit = self.cut_off_limit
         c.fulldiff = request.GET.get('fulldiff')
 
         # prev link
         try:
             prev_rev = c.db_repo_scm_instance.get_changeset(cur_rev).prev(c.branch)
             c.url_prev = url('files_home', repo_name=c.repo_name,
                          revision=prev_rev.raw_id, f_path=f_path)
             if c.branch:
                 c.url_prev += '?branch=%s' % c.branch
         except (ChangesetDoesNotExistError, VCSError):
             c.url_prev = '#'
 
         # next link
         try:
             next_rev = c.db_repo_scm_instance.get_changeset(cur_rev).next(c.branch)
             c.url_next = url('files_home', repo_name=c.repo_name,
                      revision=next_rev.raw_id, f_path=f_path)
             if c.branch:
                 c.url_next += '?branch=%s' % c.branch
         except (ChangesetDoesNotExistError, VCSError):
             c.url_next = '#'
 
         # files or dirs
         try:
             c.file = c.changeset.get_node(f_path)
 
             if c.file.is_submodule():
                 raise HTTPFound(location=c.file.url)
             elif c.file.is_file():
                 c.load_full_history = False
                 # determine if we're on branch head
                 _branches = c.db_repo_scm_instance.branches
                 c.on_branch_head = revision in _branches or revision in _branches.values()
                 _hist = []
                 c.file_history = []
                 if c.load_full_history:
                     c.file_history, _hist = self._get_node_history(c.changeset, f_path)
 
                 c.authors = []
                 for a in set([x.author for x in _hist]):
                     c.authors.append((h.email(a), h.person(a)))
             else:
                 c.authors = c.file_history = []
         except RepositoryError as e:
             h.flash(unicode(e), category='error')
             raise HTTPNotFound()
 
         if request.environ.get('HTTP_X_PARTIAL_XHR'):
             return render('files/files_ypjax.html')
 
         # TODO: tags and bookmarks?
         c.revision_options = [(c.changeset.raw_id,
                               _('%s at %s') % (b, h.short_id(c.changeset.raw_id))) for b in c.changeset.branches] + \
             [(n, b) for b, n in c.db_repo_scm_instance.branches.items()]
         if c.db_repo_scm_instance.closed_branches:
             prefix = _('(closed)') + ' '
             c.revision_options += [('-', '-')] + \
                 [(n, prefix + b) for b, n in c.db_repo_scm_instance.closed_branches.items()]
 
         return render('files/files.html')
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     @jsonify
     def history(self, repo_name, revision, f_path):
         changeset = self.__get_cs(revision)
         _file = changeset.get_node(f_path)
         if _file.is_file():
             file_history, _hist = self._get_node_history(changeset, f_path)
 
             res = []
             for obj in file_history:
                 res.append({
                     'text': obj[1],
                     'children': [{'id': o[0], 'text': o[1]} for o in obj[0]]
                 })
 
             data = {
                 'more': False,
                 'results': res
             }
             return data
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def authors(self, repo_name, revision, f_path):
         changeset = self.__get_cs(revision)
         _file = changeset.get_node(f_path)
         if _file.is_file():
             file_history, _hist = self._get_node_history(changeset, f_path)
             c.authors = []
             for a in set([x.author for x in _hist]):
                 c.authors.append((h.email(a), h.person(a)))
             return render('files/files_history_box.html')
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def rawfile(self, repo_name, revision, f_path):
         cs = self.__get_cs(revision)
         file_node = self.__get_filenode(cs, f_path)
 
         response.content_disposition = 'attachment; filename=%s' % \
             safe_str(f_path.split(Repository.url_sep())[-1])
 
         response.content_type = file_node.mimetype
         return file_node.content
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def raw(self, repo_name, revision, f_path):
         cs = self.__get_cs(revision)
         file_node = self.__get_filenode(cs, f_path)
 
         raw_mimetype_mapping = {
             # map original mimetype to a mimetype used for "show as raw"
             # you can also provide a content-disposition to override the
             # default "attachment" disposition.
             # orig_type: (new_type, new_dispo)
 
             # show images inline:
             'image/x-icon': ('image/x-icon', 'inline'),
             'image/png': ('image/png', 'inline'),
             'image/gif': ('image/gif', 'inline'),
             'image/jpeg': ('image/jpeg', 'inline'),
             'image/svg+xml': ('image/svg+xml', 'inline'),
         }
 
         mimetype = file_node.mimetype
         try:
             mimetype, dispo = raw_mimetype_mapping[mimetype]
         except KeyError:
             # we don't know anything special about this, handle it safely
             if file_node.is_binary:
                 # do same as download raw for binary files
                 mimetype, dispo = 'application/octet-stream', 'attachment'
             else:
                 # do not just use the original mimetype, but force text/plain,
                 # otherwise it would serve text/html and that might be unsafe.
                 # Note: underlying vcs library fakes text/plain mimetype if the
                 # mimetype can not be determined and it thinks it is not
                 # binary.This might lead to erroneous text display in some
                 # cases, but helps in other cases, like with text files
                 # without extension.
                 mimetype, dispo = 'text/plain', 'inline'
 
         if dispo == 'attachment':
             dispo = 'attachment; filename=%s' % \
                         safe_str(f_path.split(os.sep)[-1])
 
         response.content_disposition = dispo
         response.content_type = mimetype
         return file_node.content
 
     @LoginRequired()
     @HasRepoPermissionLevelDecorator('write')
     def delete(self, repo_name, revision, f_path):
         repo = c.db_repo
         # check if revision is a branch identifier- basically we cannot
         # create multiple heads via file editing
         _branches = repo.scm_instance.branches
         # check if revision is a branch name or branch hash
         if revision not in _branches and revision not in _branches.values():
             h.flash(_('You can only delete files with revision '
                       'being a valid branch'), category='warning')
             raise HTTPFound(location=h.url('files_home',
                                   repo_name=repo_name, revision='tip',
                                   f_path=f_path))
 
         r_post = request.POST
 
         c.cs = self.__get_cs(revision)
         c.file = self.__get_filenode(c.cs, f_path)
 
         c.default_message = _('Deleted file %s via Kallithea') % (f_path)
         c.f_path = f_path
         node_path = f_path
         author = request.authuser.full_contact
 
         if r_post:
             message = r_post.get('message') or c.default_message
 
             try:
                 nodes = {
                     node_path: {
                         'content': ''
                     }
                 }
                 self.scm_model.delete_nodes(
                     user=request.authuser.user_id,
                     ip_addr=request.ip_addr,
                     repo=c.db_repo,
                     message=message,
                     nodes=nodes,
                     parent_cs=c.cs,
                     author=author,
                 )
 
                 h.flash(_('Successfully deleted file %s') % f_path,
                         category='success')
             except Exception:
                 log.error(traceback.format_exc())
                 h.flash(_('Error occurred during commit'), category='error')
             raise HTTPFound(location=url('changeset_home',
                                 repo_name=c.repo_name, revision='tip'))
 
         return render('files/files_delete.html')
 
     @LoginRequired()
     @HasRepoPermissionLevelDecorator('write')
     def edit(self, repo_name, revision, f_path):
         repo = c.db_repo
         # check if revision is a branch identifier- basically we cannot
         # create multiple heads via file editing
         _branches = repo.scm_instance.branches
         # check if revision is a branch name or branch hash
         if revision not in _branches and revision not in _branches.values():
             h.flash(_('You can only edit files with revision '
                       'being a valid branch'), category='warning')
             raise HTTPFound(location=h.url('files_home',
                                   repo_name=repo_name, revision='tip',
                                   f_path=f_path))
 
         r_post = request.POST
 
         c.cs = self.__get_cs(revision)
         c.file = self.__get_filenode(c.cs, f_path)
 
         if c.file.is_binary:
             raise HTTPFound(location=url('files_home', repo_name=c.repo_name,
                             revision=c.cs.raw_id, f_path=f_path))
         c.default_message = _('Edited file %s via Kallithea') % (f_path)
         c.f_path = f_path
 
         if r_post:
-
-            old_content = c.file.content
+            old_content = safe_unicode(c.file.content)
             sl = old_content.splitlines(1)
             first_line = sl[0] if sl else ''
             # modes:  0 - Unix, 1 - Mac, 2 - DOS
             mode = detect_mode(first_line, 0)
             content = convert_line_endings(r_post.get('content', ''), mode)
 
             message = r_post.get('message') or c.default_message
             author = request.authuser.full_contact
 
             if content == old_content:
                 h.flash(_('No changes'), category='warning')
                 raise HTTPFound(location=url('changeset_home', repo_name=c.repo_name,
                                     revision='tip'))
             try:
                 self.scm_model.commit_change(repo=c.db_repo_scm_instance,
                                              repo_name=repo_name, cs=c.cs,
                                              user=request.authuser.user_id,
                                              ip_addr=request.ip_addr,
                                              author=author, message=message,
                                              content=content, f_path=f_path)
                 h.flash(_('Successfully committed to %s') % f_path,
                         category='success')
             except Exception:
                 log.error(traceback.format_exc())
                 h.flash(_('Error occurred during commit'), category='error')
             raise HTTPFound(location=url('changeset_home',
                                 repo_name=c.repo_name, revision='tip'))
 
         return render('files/files_edit.html')
 
     @LoginRequired()
     @HasRepoPermissionLevelDecorator('write')
     def add(self, repo_name, revision, f_path):
 
         repo = c.db_repo
         r_post = request.POST
         c.cs = self.__get_cs(revision, silent_empty=True)
         if c.cs is None:
             c.cs = EmptyChangeset(alias=c.db_repo_scm_instance.alias)
         c.default_message = (_('Added file via Kallithea'))
         c.f_path = f_path
 
         if r_post:
             unix_mode = 0
             content = convert_line_endings(r_post.get('content', ''), unix_mode)
 
             message = r_post.get('message') or c.default_message
             filename = r_post.get('filename')
             location = r_post.get('location', '')
             file_obj = r_post.get('upload_file', None)
 
             if file_obj is not None and hasattr(file_obj, 'filename'):
                 filename = file_obj.filename
                 content = file_obj.file
 
                 if hasattr(content, 'file'):
                     # non posix systems store real file under file attr
                     content = content.file
 
             if not content:
                 h.flash(_('No content'), category='warning')
                 raise HTTPFound(location=url('changeset_home', repo_name=c.repo_name,
                                     revision='tip'))
             if not filename:
                 h.flash(_('No filename'), category='warning')
                 raise HTTPFound(location=url('changeset_home', repo_name=c.repo_name,
                                     revision='tip'))
             # strip all crap out of file, just leave the basename
             filename = os.path.basename(filename)
             node_path = posixpath.join(location, filename)
             author = request.authuser.full_contact
 
             try:
                 nodes = {
                     node_path: {
                         'content': content
                     }
                 }
                 self.scm_model.create_nodes(
                     user=request.authuser.user_id,
                     ip_addr=request.ip_addr,
                     repo=c.db_repo,
                     message=message,
                     nodes=nodes,
                     parent_cs=c.cs,
                     author=author,
                 )
 
                 h.flash(_('Successfully committed to %s') % node_path,
                         category='success')
             except NonRelativePathError as e:
                 h.flash(_('Location must be relative path and must not '
                           'contain .. in path'), category='warning')
                 raise HTTPFound(location=url('changeset_home', repo_name=c.repo_name,
                                     revision='tip'))
             except (NodeError, NodeAlreadyExistsError) as e:
                 h.flash(_(e), category='error')
             except Exception:
                 log.error(traceback.format_exc())
                 h.flash(_('Error occurred during commit'), category='error')
             raise HTTPFound(location=url('changeset_home',
                                 repo_name=c.repo_name, revision='tip'))
 
         return render('files/files_add.html')
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def archivefile(self, repo_name, fname):
         fileformat = None
         revision = None
         ext = None
         subrepos = request.GET.get('subrepos') == 'true'
 
         for a_type, ext_data in settings.ARCHIVE_SPECS.items():
             archive_spec = fname.split(ext_data[1])
             if len(archive_spec) == 2 and archive_spec[1] == '':
                 fileformat = a_type or ext_data[1]
                 revision = archive_spec[0]
                 ext = ext_data[1]
 
         try:
             dbrepo = RepoModel().get_by_repo_name(repo_name)
             if not dbrepo.enable_downloads:
                 return _('Downloads disabled') # TODO: do something else?
 
             if c.db_repo_scm_instance.alias == 'hg':
                 # patch and reset hooks section of UI config to not run any
                 # hooks on fetching archives with subrepos
                 for k, v in c.db_repo_scm_instance._repo.ui.configitems('hooks'):
                     c.db_repo_scm_instance._repo.ui.setconfig('hooks', k, None)
 
             cs = c.db_repo_scm_instance.get_changeset(revision)
             content_type = settings.ARCHIVE_SPECS[fileformat][0]
         except ChangesetDoesNotExistError:
             return _('Unknown revision %s') % revision
         except EmptyRepositoryError:
             return _('Empty repository')
         except (ImproperArchiveTypeError, KeyError):
             return _('Unknown archive type')
 
         from kallithea import CONFIG
         rev_name = cs.raw_id[:12]
         archive_name = '%s-%s%s' % (safe_str(repo_name.replace('/', '_')),
                                     safe_str(rev_name), ext)
 
         archive_path = None
         cached_archive_path = None
         archive_cache_dir = CONFIG.get('archive_cache_dir')
         if archive_cache_dir and not subrepos: # TODO: subrepo caching?
             if not os.path.isdir(archive_cache_dir):
                 os.makedirs(archive_cache_dir)
             cached_archive_path = os.path.join(archive_cache_dir, archive_name)
             if os.path.isfile(cached_archive_path):
                 log.debug('Found cached archive in %s', cached_archive_path)
                 archive_path = cached_archive_path
             else:
                 log.debug('Archive %s is not yet cached', archive_name)
 
         if archive_path is None:
             # generate new archive
             fd, archive_path = tempfile.mkstemp()
             log.debug('Creating new temp archive in %s', archive_path)
             with os.fdopen(fd, 'wb') as stream:
                 cs.fill_archive(stream=stream, kind=fileformat, subrepos=subrepos)
                 # stream (and thus fd) has been closed by cs.fill_archive
             if cached_archive_path is not None:
                 # we generated the archive - move it to cache
                 log.debug('Storing new archive in %s', cached_archive_path)
                 shutil.move(archive_path, cached_archive_path)
                 archive_path = cached_archive_path
 
         def get_chunked_archive(archive_path):
             stream = open(archive_path, 'rb')
             while True:
                 data = stream.read(16 * 1024)
                 if not data:
                     break
                 yield data
             stream.close()
             if archive_path != cached_archive_path:
                 log.debug('Destroying temp archive %s', archive_path)
                 os.remove(archive_path)
 
         action_logger(user=request.authuser,
                       action='user_downloaded_archive:%s' % (archive_name),
                       repo=repo_name, ipaddr=request.ip_addr, commit=True)
 
         response.content_disposition = str('attachment; filename=%s' % (archive_name))
         response.content_type = str(content_type)
         return get_chunked_archive(archive_path)
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def diff(self, repo_name, f_path):
         ignore_whitespace = request.GET.get('ignorews') == '1'
         line_context = safe_int(request.GET.get('context'), 3)
         diff2 = request.GET.get('diff2', '')
         diff1 = request.GET.get('diff1', '') or diff2
         c.action = request.GET.get('diff')
         c.no_changes = diff1 == diff2
         c.f_path = f_path
         c.big_diff = False
         fulldiff = request.GET.get('fulldiff')
         c.anchor_url = anchor_url
         c.ignorews_url = _ignorews_url
         c.context_url = _context_url
         c.changes = OrderedDict()
         c.changes[diff2] = []
 
         # special case if we want a show rev only, it's impl here
         # to reduce JS and callbacks
 
         if request.GET.get('show_rev'):
             if str2bool(request.GET.get('annotate', 'False')):
                 _url = url('files_annotate_home', repo_name=c.repo_name,
                            revision=diff1, f_path=c.f_path)
             else:
                 _url = url('files_home', repo_name=c.repo_name,
                            revision=diff1, f_path=c.f_path)
 
             raise HTTPFound(location=_url)
         try:
             if diff1 not in ['', None, 'None', '0' * 12, '0' * 40]:
                 c.changeset_1 = c.db_repo_scm_instance.get_changeset(diff1)
                 try:
                     node1 = c.changeset_1.get_node(f_path)
                     if node1.is_dir():
                         raise NodeError('%s path is a %s not a file'
                                         % (node1, type(node1)))
                 except NodeDoesNotExistError:
                     c.changeset_1 = EmptyChangeset(cs=diff1,
                                                    revision=c.changeset_1.revision,
                                                    repo=c.db_repo_scm_instance)
                     node1 = FileNode(f_path, '', changeset=c.changeset_1)
             else:
                 c.changeset_1 = EmptyChangeset(repo=c.db_repo_scm_instance)
                 node1 = FileNode(f_path, '', changeset=c.changeset_1)
 
             if diff2 not in ['', None, 'None', '0' * 12, '0' * 40]:
                 c.changeset_2 = c.db_repo_scm_instance.get_changeset(diff2)
                 try:
                     node2 = c.changeset_2.get_node(f_path)
                     if node2.is_dir():
                         raise NodeError('%s path is a %s not a file'
                                         % (node2, type(node2)))
                 except NodeDoesNotExistError:
                     c.changeset_2 = EmptyChangeset(cs=diff2,
                                                    revision=c.changeset_2.revision,
                                                    repo=c.db_repo_scm_instance)
                     node2 = FileNode(f_path, '', changeset=c.changeset_2)
             else:
                 c.changeset_2 = EmptyChangeset(repo=c.db_repo_scm_instance)
                 node2 = FileNode(f_path, '', changeset=c.changeset_2)
         except (RepositoryError, NodeError):
             log.error(traceback.format_exc())
             raise HTTPFound(location=url('files_home', repo_name=c.repo_name,
                                 f_path=f_path))
 
         if c.action == 'download':
             raw_diff = diffs.get_gitdiff(node1, node2,
                                       ignore_whitespace=ignore_whitespace,
                                       context=line_context)
             diff_name = '%s_vs_%s.diff' % (diff1, diff2)
             response.content_type = 'text/plain'
             response.content_disposition = (
                 'attachment; filename=%s' % diff_name
             )
             return raw_diff
 
         elif c.action == 'raw':
             raw_diff = diffs.get_gitdiff(node1, node2,
                                       ignore_whitespace=ignore_whitespace,
                                       context=line_context)
             response.content_type = 'text/plain'
             return raw_diff
 
         else:
             fid = h.FID(diff2, node2.path)
             line_context_lcl = get_line_ctx(fid, request.GET)
             ign_whitespace_lcl = get_ignore_ws(fid, request.GET)
 
             diff_limit = None if fulldiff else self.cut_off_limit
             c.a_rev, c.cs_rev, a_path, diff, st, op = diffs.wrapped_diff(filenode_old=node1,
                                          filenode_new=node2,
                                          diff_limit=diff_limit,
                                          ignore_whitespace=ign_whitespace_lcl,
                                          line_context=line_context_lcl,
                                          enable_comments=False)
             c.file_diff_data = [(fid, fid, op, a_path, node2.path, diff, st)]
 
             return render('files/file_diff.html')
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def diff_2way(self, repo_name, f_path):
         diff1 = request.GET.get('diff1', '')
         diff2 = request.GET.get('diff2', '')
         try:
             if diff1 not in ['', None, 'None', '0' * 12, '0' * 40]:
                 c.changeset_1 = c.db_repo_scm_instance.get_changeset(diff1)
                 try:
                     node1 = c.changeset_1.get_node(f_path)
                     if node1.is_dir():
                         raise NodeError('%s path is a %s not a file'
                                         % (node1, type(node1)))
                 except NodeDoesNotExistError:
                     c.changeset_1 = EmptyChangeset(cs=diff1,
                                                    revision=c.changeset_1.revision,
                                                    repo=c.db_repo_scm_instance)
                     node1 = FileNode(f_path, '', changeset=c.changeset_1)
             else:
                 c.changeset_1 = EmptyChangeset(repo=c.db_repo_scm_instance)
                 node1 = FileNode(f_path, '', changeset=c.changeset_1)
 
             if diff2 not in ['', None, 'None', '0' * 12, '0' * 40]:
                 c.changeset_2 = c.db_repo_scm_instance.get_changeset(diff2)
                 try:
                     node2 = c.changeset_2.get_node(f_path)
                     if node2.is_dir():
                         raise NodeError('%s path is a %s not a file'
                                         % (node2, type(node2)))
                 except NodeDoesNotExistError:
                     c.changeset_2 = EmptyChangeset(cs=diff2,
                                                    revision=c.changeset_2.revision,
                                                    repo=c.db_repo_scm_instance)
                     node2 = FileNode(f_path, '', changeset=c.changeset_2)
             else:
                 c.changeset_2 = EmptyChangeset(repo=c.db_repo_scm_instance)
                 node2 = FileNode(f_path, '', changeset=c.changeset_2)
         except ChangesetDoesNotExistError as e:
             msg = _('Such revision does not exist for this repository')
             h.flash(msg, category='error')
             raise HTTPNotFound()
         c.node1 = node1
         c.node2 = node2
         c.cs1 = c.changeset_1
         c.cs2 = c.changeset_2
 
         return render('files/diff_2way.html')
 
     def _get_node_history(self, cs, f_path, changesets=None):
         """
         get changesets history for given node
 
         :param cs: changeset to calculate history
         :param f_path: path for node to calculate history for
         :param changesets: if passed don't calculate history and take
             changesets defined in this list
         """
         # calculate history based on tip
         tip_cs = c.db_repo_scm_instance.get_changeset()
         if changesets is None:
             try:
                 changesets = tip_cs.get_file_history(f_path)
             except (NodeDoesNotExistError, ChangesetError):
                 # this node is not present at tip !
                 changesets = cs.get_file_history(f_path)
         hist_l = []
 
         changesets_group = ([], _("Changesets"))
         branches_group = ([], _("Branches"))
         tags_group = ([], _("Tags"))
         for chs in changesets:
             # TODO: loop over chs.branches ... but that will not give all the bogus None branches for Git ...
             _branch = chs.branch
             n_desc = '%s (%s)' % (h.show_id(chs), _branch)
             changesets_group[0].append((chs.raw_id, n_desc,))
         hist_l.append(changesets_group)
 
         for name, chs in c.db_repo_scm_instance.branches.items():
             branches_group[0].append((chs, name),)
         hist_l.append(branches_group)
 
         for name, chs in c.db_repo_scm_instance.tags.items():
             tags_group[0].append((chs, name),)
         hist_l.append(tags_group)
 
         return hist_l, changesets
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     @jsonify
     def nodelist(self, repo_name, revision, f_path):
         if request.environ.get('HTTP_X_PARTIAL_XHR'):
             cs = self.__get_cs(revision)
             _d, _f = ScmModel().get_nodes(repo_name, cs.raw_id, f_path,
                                           flat=False)
             return {'nodes': _d + _f}

kallithea/controllers/pullrequests.py

@@ -1,645 +1,645 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.controllers.pullrequests
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 pull requests controller for Kallithea for initializing pull requests
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: May 7, 2012
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 import logging
 import traceback
 
 import formencode
 from tg import request
 from tg import tmpl_context as c
 from tg.i18n import ugettext as _
 from webob.exc import HTTPBadRequest, HTTPForbidden, HTTPFound, HTTPNotFound
 
 from kallithea.config.routing import url
 from kallithea.controllers.changeset import _context_url, _ignorews_url, create_cs_pr_comment, delete_cs_pr_comment
 from kallithea.lib import diffs
 from kallithea.lib import helpers as h
 from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired
 from kallithea.lib.base import BaseRepoController, jsonify, render
 from kallithea.lib.graphmod import graph_data
 from kallithea.lib.page import Page
 from kallithea.lib.utils2 import safe_int
 from kallithea.lib.vcs.exceptions import ChangesetDoesNotExistError, EmptyRepositoryError
 from kallithea.lib.vcs.utils import safe_str
 from kallithea.lib.vcs.utils.hgcompat import unionrepo
 from kallithea.model.changeset_status import ChangesetStatusModel
 from kallithea.model.comment import ChangesetCommentsModel
 from kallithea.model.db import ChangesetStatus, PullRequest, PullRequestReviewer, Repository, User
 from kallithea.model.forms import PullRequestForm, PullRequestPostForm
 from kallithea.model.meta import Session
 from kallithea.model.pull_request import CreatePullRequestAction, CreatePullRequestIterationAction, PullRequestModel
 
 
 log = logging.getLogger(__name__)
 
 
 def _get_reviewer(user_id):
     """Look up user by ID and validate it as a potential reviewer."""
     try:
         user = User.get(int(user_id))
     except ValueError:
         user = None
 
     if user is None or user.is_default_user:
         h.flash(_('Invalid reviewer "%s" specified') % user_id, category='error')
         raise HTTPBadRequest()
 
     return user
 
 
 class PullrequestsController(BaseRepoController):
 
     def _get_repo_refs(self, repo, rev=None, branch=None, branch_rev=None):
         """return a structure with repo's interesting changesets, suitable for
         the selectors in pullrequest.html
 
         rev: a revision that must be in the list somehow and selected by default
         branch: a branch that must be in the list and selected by default - even if closed
         branch_rev: a revision of which peers should be preferred and available."""
         # list named branches that has been merged to this named branch - it should probably merge back
         peers = []
 
         if rev:
             rev = safe_str(rev)
 
         if branch:
             branch = safe_str(branch)
 
         if branch_rev:
             branch_rev = safe_str(branch_rev)
             # a revset not restricting to merge() would be better
             # (especially because it would get the branch point)
             # ... but is currently too expensive
             # including branches of children could be nice too
             peerbranches = set()
             for i in repo._repo.revs(
                 "sort(parents(branch(id(%s)) and merge()) - branch(id(%s)), -rev)",
                 branch_rev, branch_rev
             ):
                 for abranch in repo.get_changeset(i).branches:
                     if abranch not in peerbranches:
                         n = 'branch:%s:%s' % (abranch, repo.get_changeset(abranch).raw_id)
                         peers.append((n, abranch))
                         peerbranches.add(abranch)
 
         selected = None
         tiprev = repo.tags.get('tip')
         tipbranch = None
 
         branches = []
         for abranch, branchrev in repo.branches.iteritems():
             n = 'branch:%s:%s' % (abranch, branchrev)
             desc = abranch
             if branchrev == tiprev:
                 tipbranch = abranch
                 desc = '%s (current tip)' % desc
             branches.append((n, desc))
             if rev == branchrev:
                 selected = n
             if branch == abranch:
                 if not rev:
                     selected = n
                 branch = None
         if branch:  # branch not in list - it is probably closed
             branchrev = repo.closed_branches.get(branch)
             if branchrev:
                 n = 'branch:%s:%s' % (branch, branchrev)
                 branches.append((n, _('%s (closed)') % branch))
                 selected = n
                 branch = None
             if branch:
                 log.debug('branch %r not found in %s', branch, repo)
 
         bookmarks = []
         for bookmark, bookmarkrev in repo.bookmarks.iteritems():
             n = 'book:%s:%s' % (bookmark, bookmarkrev)
             bookmarks.append((n, bookmark))
             if rev == bookmarkrev:
                 selected = n
 
         tags = []
         for tag, tagrev in repo.tags.iteritems():
             if tag == 'tip':
                 continue
             n = 'tag:%s:%s' % (tag, tagrev)
             tags.append((n, tag))
             # note: even if rev == tagrev, don't select the static tag - it must be chosen explicitly
 
         # prio 1: rev was selected as existing entry above
 
         # prio 2: create special entry for rev; rev _must_ be used
         specials = []
         if rev and selected is None:
             selected = 'rev:%s:%s' % (rev, rev)
             specials = [(selected, '%s: %s' % (_("Changeset"), rev[:12]))]
 
         # prio 3: most recent peer branch
         if peers and not selected:
             selected = peers[0][0]
 
         # prio 4: tip revision
         if not selected:
             if h.is_hg(repo):
                 if tipbranch:
                     selected = 'branch:%s:%s' % (tipbranch, tiprev)
                 else:
                     selected = 'tag:null:' + repo.EMPTY_CHANGESET
                     tags.append((selected, 'null'))
             else:
                 if 'master' in repo.branches:
                     selected = 'branch:master:%s' % repo.branches['master']
                 else:
                     k, v = list(repo.branches.items())[0]
                     selected = 'branch:%s:%s' % (k, v)
 
         groups = [(specials, _("Special")),
                   (peers, _("Peer branches")),
                   (bookmarks, _("Bookmarks")),
                   (branches, _("Branches")),
                   (tags, _("Tags")),
                   ]
         return [g for g in groups if g[0]], selected
 
     def _is_allowed_to_change_status(self, pull_request):
         if pull_request.is_closed():
             return False
 
         owner = request.authuser.user_id == pull_request.owner_id
         reviewer = PullRequestReviewer.query() \
             .filter(PullRequestReviewer.pull_request == pull_request) \
             .filter(PullRequestReviewer.user_id == request.authuser.user_id) \
             .count() != 0
 
         return request.authuser.admin or owner or reviewer
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def show_all(self, repo_name):
         c.from_ = request.GET.get('from_') or ''
         c.closed = request.GET.get('closed') or ''
         url_params = {}
         if c.from_:
             url_params['from_'] = 1
         if c.closed:
             url_params['closed'] = 1
         p = safe_int(request.GET.get('page'), 1)
 
         q = PullRequest.query(include_closed=c.closed, sorted=True)
         if c.from_:
             q = q.filter_by(org_repo=c.db_repo)
         else:
             q = q.filter_by(other_repo=c.db_repo)
         c.pull_requests = q.all()
 
         c.pullrequests_pager = Page(c.pull_requests, page=p, items_per_page=100, **url_params)
 
         return render('/pullrequests/pullrequest_show_all.html')
 
     @LoginRequired()
     def show_my(self):
         c.closed = request.GET.get('closed') or ''
 
         c.my_pull_requests = PullRequest.query(
             include_closed=c.closed,
             sorted=True,
         ).filter_by(owner_id=request.authuser.user_id).all()
 
         c.participate_in_pull_requests = []
         c.participate_in_pull_requests_todo = []
         done_status = set([ChangesetStatus.STATUS_APPROVED, ChangesetStatus.STATUS_REJECTED])
         for pr in PullRequest.query(
             include_closed=c.closed,
             reviewer_id=request.authuser.user_id,
             sorted=True,
         ):
             status = pr.user_review_status(request.authuser.user_id) # very inefficient!!!
             if status in done_status:
                 c.participate_in_pull_requests.append(pr)
             else:
                 c.participate_in_pull_requests_todo.append(pr)
 
         return render('/pullrequests/pullrequest_show_my.html')
 
     @LoginRequired()
     @HasRepoPermissionLevelDecorator('read')
     def index(self):
         org_repo = c.db_repo
         org_scm_instance = org_repo.scm_instance
         try:
             org_scm_instance.get_changeset()
         except EmptyRepositoryError as e:
             h.flash(_('There are no changesets yet'),
                     category='warning')
             raise HTTPFound(location=url('summary_home', repo_name=org_repo.repo_name))
 
         org_rev = request.GET.get('rev_end')
         # rev_start is not directly useful - its parent could however be used
         # as default for other and thus give a simple compare view
         rev_start = request.GET.get('rev_start')
         other_rev = None
         if rev_start:
             starters = org_repo.get_changeset(rev_start).parents
             if starters:
                 other_rev = starters[0].raw_id
             else:
                 other_rev = org_repo.scm_instance.EMPTY_CHANGESET
         branch = request.GET.get('branch')
 
         c.cs_repos = [(org_repo.repo_name, org_repo.repo_name)]
         c.default_cs_repo = org_repo.repo_name
         c.cs_refs, c.default_cs_ref = self._get_repo_refs(org_scm_instance, rev=org_rev, branch=branch)
 
         default_cs_ref_type, default_cs_branch, default_cs_rev = c.default_cs_ref.split(':')
         if default_cs_ref_type != 'branch':
             default_cs_branch = org_repo.get_changeset(default_cs_rev).branch
 
         # add org repo to other so we can open pull request against peer branches on itself
         c.a_repos = [(org_repo.repo_name, '%s (self)' % org_repo.repo_name)]
 
         if org_repo.parent:
             # add parent of this fork also and select it.
             # use the same branch on destination as on source, if available.
             c.a_repos.append((org_repo.parent.repo_name, '%s (parent)' % org_repo.parent.repo_name))
             c.a_repo = org_repo.parent
             c.a_refs, c.default_a_ref = self._get_repo_refs(
                     org_repo.parent.scm_instance, branch=default_cs_branch, rev=other_rev)
 
         else:
             c.a_repo = org_repo
             c.a_refs, c.default_a_ref = self._get_repo_refs(org_scm_instance, rev=other_rev)
 
         # gather forks and add to this list ... even though it is rare to
         # request forks to pull from their parent
         for fork in org_repo.forks:
             c.a_repos.append((fork.repo_name, fork.repo_name))
 
         return render('/pullrequests/pullrequest.html')
 
     @LoginRequired()
     @HasRepoPermissionLevelDecorator('read')
     @jsonify
     def repo_info(self, repo_name):
         repo = c.db_repo
         refs, selected_ref = self._get_repo_refs(repo.scm_instance)
         return {
             'description': repo.description.split('\n', 1)[0],
             'selected_ref': selected_ref,
             'refs': refs,
             }
 
     @LoginRequired()
     @HasRepoPermissionLevelDecorator('read')
     def create(self, repo_name):
         repo = c.db_repo
         try:
             _form = PullRequestForm(repo.repo_id)().to_python(request.POST)
         except formencode.Invalid as errors:
             log.error(traceback.format_exc())
             log.error(str(errors))
             msg = _('Error creating pull request: %s') % errors.msg
             h.flash(msg, 'error')
             raise HTTPBadRequest
 
         # heads up: org and other might seem backward here ...
         org_ref = _form['org_ref'] # will have merge_rev as rev but symbolic name
         org_repo = Repository.guess_instance(_form['org_repo'])
 
         other_ref = _form['other_ref'] # will have symbolic name and head revision
         other_repo = Repository.guess_instance(_form['other_repo'])
 
         reviewers = []
 
         title = _form['pullrequest_title']
         description = _form['pullrequest_desc'].strip()
         owner = User.get(request.authuser.user_id)
 
         try:
             cmd = CreatePullRequestAction(org_repo, other_repo, org_ref, other_ref, title, description, owner, reviewers)
         except CreatePullRequestAction.ValidationError as e:
             h.flash(str(e), category='error', logf=log.error)
             raise HTTPNotFound
 
         try:
             pull_request = cmd.execute()
             Session().commit()
         except Exception:
             h.flash(_('Error occurred while creating pull request'),
                     category='error')
             log.error(traceback.format_exc())
             raise HTTPFound(location=url('pullrequest_home', repo_name=repo_name))
 
         h.flash(_('Successfully opened new pull request'),
                 category='success')
         raise HTTPFound(location=pull_request.url())
 
     def create_new_iteration(self, old_pull_request, new_rev, title, description, reviewers):
         owner = User.get(request.authuser.user_id)
         new_org_rev = self._get_ref_rev(old_pull_request.org_repo, 'rev', new_rev)
         new_other_rev = self._get_ref_rev(old_pull_request.other_repo, old_pull_request.other_ref_parts[0], old_pull_request.other_ref_parts[1])
         try:
             cmd = CreatePullRequestIterationAction(old_pull_request, new_org_rev, new_other_rev, title, description, owner, reviewers)
         except CreatePullRequestAction.ValidationError as e:
             h.flash(str(e), category='error', logf=log.error)
             raise HTTPNotFound
 
         try:
             pull_request = cmd.execute()
             Session().commit()
         except Exception:
             h.flash(_('Error occurred while creating pull request'),
                     category='error')
             log.error(traceback.format_exc())
             raise HTTPFound(location=old_pull_request.url())
 
         h.flash(_('New pull request iteration created'),
                 category='success')
         raise HTTPFound(location=pull_request.url())
 
     # pullrequest_post for PR editing
     @LoginRequired()
     @HasRepoPermissionLevelDecorator('read')
     def post(self, repo_name, pull_request_id):
         pull_request = PullRequest.get_or_404(pull_request_id)
         if pull_request.is_closed():
             raise HTTPForbidden()
         assert pull_request.other_repo.repo_name == repo_name
         # only owner or admin can update it
         owner = pull_request.owner_id == request.authuser.user_id
         repo_admin = h.HasRepoPermissionLevel('admin')(c.repo_name)
         if not (h.HasPermissionAny('hg.admin')() or repo_admin or owner):
             raise HTTPForbidden()
 
         _form = PullRequestPostForm()().to_python(request.POST)
 
         cur_reviewers = set(pull_request.get_reviewer_users())
         new_reviewers = set(_get_reviewer(s) for s in _form['review_members'])
         old_reviewers = set(_get_reviewer(s) for s in _form['org_review_members'])
 
         other_added = cur_reviewers - old_reviewers
         other_removed = old_reviewers - cur_reviewers
 
         if other_added:
             h.flash(_('Meanwhile, the following reviewers have been added: %s') %
                     (', '.join(u.username for u in other_added)),
                     category='warning')
         if other_removed:
             h.flash(_('Meanwhile, the following reviewers have been removed: %s') %
                     (', '.join(u.username for u in other_removed)),
                     category='warning')
 
         if _form['updaterev']:
             return self.create_new_iteration(pull_request,
                                       _form['updaterev'],
                                       _form['pullrequest_title'],
                                       _form['pullrequest_desc'],
                                       new_reviewers)
 
         added_reviewers = new_reviewers - old_reviewers - cur_reviewers
         removed_reviewers = (old_reviewers - new_reviewers) & cur_reviewers
 
         old_description = pull_request.description
         pull_request.title = _form['pullrequest_title']
         pull_request.description = _form['pullrequest_desc'].strip() or _('No description')
         pull_request.owner = User.get_by_username(_form['owner'])
         user = User.get(request.authuser.user_id)
 
         PullRequestModel().mention_from_description(user, pull_request, old_description)
         PullRequestModel().add_reviewers(user, pull_request, added_reviewers)
         PullRequestModel().remove_reviewers(user, pull_request, removed_reviewers)
 
         Session().commit()
         h.flash(_('Pull request updated'), category='success')
 
         raise HTTPFound(location=pull_request.url())
 
     @LoginRequired()
     @HasRepoPermissionLevelDecorator('read')
     @jsonify
     def delete(self, repo_name, pull_request_id):
         pull_request = PullRequest.get_or_404(pull_request_id)
         # only owner can delete it !
         if pull_request.owner_id == request.authuser.user_id:
             PullRequestModel().delete(pull_request)
             Session().commit()
             h.flash(_('Successfully deleted pull request'),
                     category='success')
             raise HTTPFound(location=url('my_pullrequests'))
         raise HTTPForbidden()
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def show(self, repo_name, pull_request_id, extra=None):
         c.pull_request = PullRequest.get_or_404(pull_request_id)
         c.allowed_to_change_status = self._is_allowed_to_change_status(c.pull_request)
         cc_model = ChangesetCommentsModel()
         cs_model = ChangesetStatusModel()
 
         # pull_requests repo_name we opened it against
         # ie. other_repo must match
         if repo_name != c.pull_request.other_repo.repo_name:
             raise HTTPNotFound
 
         # load compare data into template context
         c.cs_repo = c.pull_request.org_repo
         (c.cs_ref_type,
          c.cs_ref_name,
          c.cs_rev) = c.pull_request.org_ref.split(':')
 
         c.a_repo = c.pull_request.other_repo
         (c.a_ref_type,
          c.a_ref_name,
          c.a_rev) = c.pull_request.other_ref.split(':') # a_rev is ancestor
 
         org_scm_instance = c.cs_repo.scm_instance # property with expensive cache invalidation check!!!
         try:
             c.cs_ranges = []
             for x in c.pull_request.revisions:
                 c.cs_ranges.append(org_scm_instance.get_changeset(x))
         except ChangesetDoesNotExistError:
             c.cs_ranges = []
             h.flash(_('Revision %s not found in %s') % (x, c.cs_repo.repo_name),
                 'error')
         c.cs_ranges_org = None # not stored and not important and moving target - could be calculated ...
         revs = [ctx.revision for ctx in reversed(c.cs_ranges)]
         c.jsdata = graph_data(org_scm_instance, revs)
 
         c.is_range = False
         try:
             if c.a_ref_type == 'rev': # this looks like a free range where target is ancestor
                 cs_a = org_scm_instance.get_changeset(c.a_rev)
                 root_parents = c.cs_ranges[0].parents
                 c.is_range = cs_a in root_parents
                 #c.merge_root = len(root_parents) > 1 # a range starting with a merge might deserve a warning
         except ChangesetDoesNotExistError: # probably because c.a_rev not found
             pass
         except IndexError: # probably because c.cs_ranges is empty, probably because revisions are missing
             pass
 
         avail_revs = set()
         avail_show = []
         c.cs_branch_name = c.cs_ref_name
         c.a_branch_name = None
         other_scm_instance = c.a_repo.scm_instance
         c.update_msg = ""
         c.update_msg_other = ""
         try:
             if not c.cs_ranges:
                 c.update_msg = _('Error: changesets not found when displaying pull request from %s.') % c.cs_rev
             elif org_scm_instance.alias == 'hg' and c.a_ref_name != 'ancestor':
                 if c.cs_ref_type != 'branch':
                     c.cs_branch_name = org_scm_instance.get_changeset(c.cs_ref_name).branch # use ref_type ?
                 c.a_branch_name = c.a_ref_name
                 if c.a_ref_type != 'branch':
                     try:
                         c.a_branch_name = other_scm_instance.get_changeset(c.a_ref_name).branch # use ref_type ?
                     except EmptyRepositoryError:
                         c.a_branch_name = 'null' # not a branch name ... but close enough
                 # candidates: descendants of old head that are on the right branch
                 #             and not are the old head itself ...
                 #             and nothing at all if old head is a descendant of target ref name
                 if not c.is_range and other_scm_instance._repo.revs('present(%s)::&%s', c.cs_ranges[-1].raw_id, c.a_branch_name):
                     c.update_msg = _('This pull request has already been merged to %s.') % c.a_branch_name
                 elif c.pull_request.is_closed():
                     c.update_msg = _('This pull request has been closed and can not be updated.')
                 else: # look for descendants of PR head on source branch in org repo
                     avail_revs = org_scm_instance._repo.revs('%s:: & branch(%s)',
                                                              revs[0], c.cs_branch_name)
                     if len(avail_revs) > 1: # more than just revs[0]
                         # also show changesets that not are descendants but would be merged in
                         targethead = other_scm_instance.get_changeset(c.a_branch_name).raw_id
                         if org_scm_instance.path != other_scm_instance.path:
                             # Note: org_scm_instance.path must come first so all
                             # valid revision numbers are 100% org_scm compatible
                             # - both for avail_revs and for revset results
                             hgrepo = unionrepo.makeunionrepository(org_scm_instance.baseui,
                                                                    org_scm_instance.path,
                                                                    other_scm_instance.path)
                         else:
                             hgrepo = org_scm_instance._repo
                         show = set(hgrepo.revs('::%ld & !::parents(%s) & !::%s',
                                                avail_revs, revs[0], targethead))
                         if show:
                             c.update_msg = _('The following additional changes are available on %s:') % c.cs_branch_name
                         else:
                             c.update_msg = _('No additional changesets found for iterating on this pull request.')
                     else:
                         show = set()
                         avail_revs = set() # drop revs[0]
                         c.update_msg = _('No additional changesets found for iterating on this pull request.')
 
                     # TODO: handle branch heads that not are tip-most
                     brevs = org_scm_instance._repo.revs('%s - %ld - %s', c.cs_branch_name, avail_revs, revs[0])
                     if brevs:
                         # also show changesets that are on branch but neither ancestors nor descendants
                         show.update(org_scm_instance._repo.revs('::%ld - ::%ld - ::%s', brevs, avail_revs, c.a_branch_name))
                         show.add(revs[0]) # make sure graph shows this so we can see how they relate
                         c.update_msg_other = _('Note: Branch %s has another head: %s.') % (c.cs_branch_name,
                             h.short_id(org_scm_instance.get_changeset((max(brevs))).raw_id))
 
                     avail_show = sorted(show, reverse=True)
 
             elif org_scm_instance.alias == 'git':
                 c.cs_repo.scm_instance.get_changeset(c.cs_rev) # check it exists - raise ChangesetDoesNotExistError if not
                 c.update_msg = _("Git pull requests don't support iterating yet.")
         except ChangesetDoesNotExistError:
             c.update_msg = _('Error: some changesets not found when displaying pull request from %s.') % c.cs_rev
 
         c.avail_revs = avail_revs
         c.avail_cs = [org_scm_instance.get_changeset(r) for r in avail_show]
         c.avail_jsdata = graph_data(org_scm_instance, avail_show)
 
         raw_ids = [x.raw_id for x in c.cs_ranges]
         c.cs_comments = c.cs_repo.get_comments(raw_ids)
         c.cs_statuses = c.cs_repo.statuses(raw_ids)
 
         ignore_whitespace = request.GET.get('ignorews') == '1'
         line_context = safe_int(request.GET.get('context'), 3)
         c.ignorews_url = _ignorews_url
         c.context_url = _context_url
         fulldiff = request.GET.get('fulldiff')
         diff_limit = None if fulldiff else self.cut_off_limit
 
         # we swap org/other ref since we run a simple diff on one repo
         log.debug('running diff between %s and %s in %s',
                   c.a_rev, c.cs_rev, org_scm_instance.path)
         try:
             raw_diff = diffs.get_diff(org_scm_instance, rev1=safe_str(c.a_rev), rev2=safe_str(c.cs_rev),
                                       ignore_whitespace=ignore_whitespace, context=line_context)
         except ChangesetDoesNotExistError:
             raw_diff = _("The diff can't be shown - the PR revisions could not be found.")
-        diff_processor = diffs.DiffProcessor(raw_diff or '', diff_limit=diff_limit)
+        diff_processor = diffs.DiffProcessor(raw_diff, diff_limit=diff_limit)
         c.limited_diff = diff_processor.limited_diff
         c.file_diff_data = []
         c.lines_added = 0
         c.lines_deleted = 0
 
         for f in diff_processor.parsed:
             st = f['stats']
             c.lines_added += st['added']
             c.lines_deleted += st['deleted']
             filename = f['filename']
             fid = h.FID('', filename)
             html_diff = diffs.as_html(enable_comments=True, parsed_lines=[f])
             c.file_diff_data.append((fid, None, f['operation'], f['old_filename'], filename, html_diff, st))
 
         # inline comments
         c.inline_cnt = 0
         c.inline_comments = cc_model.get_inline_comments(
                                 c.db_repo.repo_id,
                                 pull_request=pull_request_id)
         # count inline comments
         for __, lines in c.inline_comments:
             for comments in lines.values():
                 c.inline_cnt += len(comments)
         # comments
         c.comments = cc_model.get_comments(c.db_repo.repo_id, pull_request=pull_request_id)
 
         # (badly named) pull-request status calculation based on reviewer votes
         (c.pull_request_reviewers,
          c.pull_request_pending_reviewers,
          c.current_voting_result,
          ) = cs_model.calculate_pull_request_result(c.pull_request)
         c.changeset_statuses = ChangesetStatus.STATUSES
 
         c.is_ajax_preview = False
         c.ancestors = None # [c.a_rev] ... but that is shown in an other way
         return render('/pullrequests/pullrequest_show.html')
 
     @LoginRequired()
     @HasRepoPermissionLevelDecorator('read')
     @jsonify
     def comment(self, repo_name, pull_request_id):
         pull_request = PullRequest.get_or_404(pull_request_id)
         allowed_to_change_status = self._is_allowed_to_change_status(pull_request)
         return create_cs_pr_comment(repo_name, pull_request=pull_request,
                 allowed_to_change_status=allowed_to_change_status)
 
     @LoginRequired()
     @HasRepoPermissionLevelDecorator('read')
     @jsonify
     def delete_comment(self, repo_name, comment_id):
         return delete_cs_pr_comment(repo_name, comment_id)

kallithea/controllers/summary.py

@@ -1,221 +1,221 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.controllers.summary
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Summary controller for Kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 18, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 import calendar
 import itertools
 import logging
 import traceback
 from datetime import date, timedelta
 from time import mktime
 
 from beaker.cache import cache_region
 from tg import request
 from tg import tmpl_context as c
 from tg.i18n import ugettext as _
 from webob.exc import HTTPBadRequest
 
 import kallithea.lib.helpers as h
 from kallithea.config.conf import ALL_EXTS, ALL_READMES, LANGUAGES_EXTENSIONS_MAP
 from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired
 from kallithea.lib.base import BaseRepoController, jsonify, render
 from kallithea.lib.celerylib.tasks import get_commits_stats
 from kallithea.lib.compat import json
 from kallithea.lib.markup_renderer import MarkupRenderer
 from kallithea.lib.page import Page
-from kallithea.lib.utils2 import safe_int
+from kallithea.lib.utils2 import safe_int, safe_unicode
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.lib.vcs.exceptions import ChangesetError, EmptyRepositoryError, NodeDoesNotExistError
 from kallithea.lib.vcs.nodes import FileNode
 from kallithea.model.db import Statistics
 
 
 log = logging.getLogger(__name__)
 
 README_FILES = [''.join([x[0][0], x[1][0]]) for x in
                     sorted(list(itertools.product(ALL_READMES, ALL_EXTS)),
                            key=lambda y:y[0][1] + y[1][1])]
 
 
 class SummaryController(BaseRepoController):
 
     def __get_readme_data(self, db_repo):
         repo_name = db_repo.repo_name
         log.debug('Looking for README file')
 
         @cache_region('long_term', '_get_readme_from_cache')
         def _get_readme_from_cache(*_cache_keys):  # parameters are not really used - only as caching key
             readme_data = None
             readme_file = None
             try:
                 # gets the landing revision! or tip if fails
                 cs = db_repo.get_landing_changeset()
                 if isinstance(cs, EmptyChangeset):
                     raise EmptyRepositoryError()
                 renderer = MarkupRenderer()
                 for f in README_FILES:
                     try:
                         readme = cs.get_node(f)
                         if not isinstance(readme, FileNode):
                             continue
                         readme_file = f
                         log.debug('Found README file `%s` rendering...',
                                   readme_file)
-                        readme_data = renderer.render(readme.content,
+                        readme_data = renderer.render(safe_unicode(readme.content),
                                                       filename=f)
                         break
                     except NodeDoesNotExistError:
                         continue
             except ChangesetError:
                 log.error(traceback.format_exc())
                 pass
             except EmptyRepositoryError:
                 pass
 
             return readme_data, readme_file
 
         kind = 'README'
         return _get_readme_from_cache(repo_name, kind, c.db_repo.changeset_cache.get('raw_id'))
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def index(self, repo_name):
         p = safe_int(request.GET.get('page'), 1)
         size = safe_int(request.GET.get('size'), 10)
         try:
             collection = c.db_repo_scm_instance.get_changesets(reverse=True)
         except EmptyRepositoryError as e:
             h.flash(unicode(e), category='warning')
             collection = []
         c.cs_pagination = Page(collection, page=p, items_per_page=size)
         page_revisions = [x.raw_id for x in list(c.cs_pagination)]
         c.cs_comments = c.db_repo.get_comments(page_revisions)
         c.cs_statuses = c.db_repo.statuses(page_revisions)
 
         c.ssh_repo_url = None
         if request.authuser.is_default_user:
             username = None
         else:
             username = request.authuser.username
             if c.ssh_enabled:
                 c.ssh_repo_url = c.db_repo.clone_url(clone_uri_tmpl=c.clone_ssh_tmpl)
 
         c.clone_repo_url = c.db_repo.clone_url(clone_uri_tmpl=c.clone_uri_tmpl, with_id=False, username=username)
         c.clone_repo_url_id = c.db_repo.clone_url(clone_uri_tmpl=c.clone_uri_tmpl, with_id=True, username=username)
 
         if c.db_repo.enable_statistics:
             c.show_stats = True
         else:
             c.show_stats = False
 
         stats = Statistics.query() \
             .filter(Statistics.repository == c.db_repo) \
             .scalar()
 
         c.stats_percentage = 0
 
         if stats and stats.languages:
             c.no_data = False is c.db_repo.enable_statistics
             lang_stats_d = json.loads(stats.languages)
 
             lang_stats = [(x, {"count": y,
                                "desc": LANGUAGES_EXTENSIONS_MAP.get(x, '?')})
                           for x, y in lang_stats_d.items()]
             lang_stats.sort(key=lambda k: (-k[1]['count'], k[0]))
 
             c.trending_languages = lang_stats[:10]
         else:
             c.no_data = True
             c.trending_languages = []
 
         c.enable_downloads = c.db_repo.enable_downloads
         c.readme_data, c.readme_file = \
             self.__get_readme_data(c.db_repo)
         return render('summary/summary.html')
 
     @LoginRequired()
     @HasRepoPermissionLevelDecorator('read')
     @jsonify
     def repo_size(self, repo_name):
         if request.is_xhr:
             return c.db_repo._repo_size()
         else:
             raise HTTPBadRequest()
 
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def statistics(self, repo_name):
         if c.db_repo.enable_statistics:
             c.show_stats = True
             c.no_data_msg = _('No data ready yet')
         else:
             c.show_stats = False
             c.no_data_msg = _('Statistics are disabled for this repository')
 
         td = date.today() + timedelta(days=1)
         td_1m = td - timedelta(days=calendar.mdays[td.month])
         td_1y = td - timedelta(days=365)
 
         ts_min_m = mktime(td_1m.timetuple())
         ts_min_y = mktime(td_1y.timetuple())
         ts_max_y = mktime(td.timetuple())
         c.ts_min = ts_min_m
         c.ts_max = ts_max_y
 
         stats = Statistics.query() \
             .filter(Statistics.repository == c.db_repo) \
             .scalar()
         c.stats_percentage = 0
         if stats and stats.languages:
             c.no_data = False is c.db_repo.enable_statistics
             lang_stats_d = json.loads(stats.languages)
             c.commit_data = json.loads(stats.commit_activity)
             c.overview_data = json.loads(stats.commit_activity_combined)
 
             lang_stats = ((x, {"count": y,
                                "desc": LANGUAGES_EXTENSIONS_MAP.get(x)})
                           for x, y in lang_stats_d.items())
 
             c.trending_languages = (
                 sorted(lang_stats, reverse=True, key=lambda k: k[1])[:10]
             )
             last_rev = stats.stat_on_revision + 1
             c.repo_last_rev = c.db_repo_scm_instance.count() \
                 if c.db_repo_scm_instance.revisions else 0
             if last_rev == 0 or c.repo_last_rev == 0:
                 pass
             else:
                 c.stats_percentage = '%.2f' % ((float((last_rev)) /
                                                 c.repo_last_rev) * 100)
         else:
             c.commit_data = {}
             c.overview_data = ([[ts_min_y, 0], [ts_max_y, 10]])
             c.trending_languages = {}
             c.no_data = True
 
         recurse_limit = 500  # don't recurse more than 500 times when parsing
         get_commits_stats(c.db_repo.repo_name, ts_min_y, ts_max_y, recurse_limit)
         return render('summary/statistics.html')

kallithea/lib/annotate.py

@@ -1,206 +1,207 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.annotate
 ~~~~~~~~~~~~~~~~~~~~~~
 
 Annotation library for usage in Kallithea, previously part of vcs
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Dec 4, 2011
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 from pygments import highlight
 from pygments.formatters import HtmlFormatter
 
 from kallithea.lib.vcs.exceptions import VCSError
 from kallithea.lib.vcs.nodes import FileNode
+from kallithea.lib.vcs.utils import safe_unicode
 
 
 def annotate_highlight(filenode, annotate_from_changeset_func=None,
         order=None, headers=None, **options):
     """
     Returns html portion containing annotated table with 3 columns: line
     numbers, changeset information and pygmentized line of code.
 
     :param filenode: FileNode object
     :param annotate_from_changeset_func: function taking changeset and
       returning single annotate cell; needs break line at the end
     :param order: ordered sequence of ``ls`` (line numbers column),
       ``annotate`` (annotate column), ``code`` (code column); Default is
       ``['ls', 'annotate', 'code']``
     :param headers: dictionary with headers (keys are whats in ``order``
       parameter)
     """
     from kallithea.lib.pygmentsutils import get_custom_lexer
     options['linenos'] = True
     formatter = AnnotateHtmlFormatter(filenode=filenode, order=order,
         headers=headers,
         annotate_from_changeset_func=annotate_from_changeset_func, **options)
     lexer = get_custom_lexer(filenode.extension) or filenode.lexer
-    highlighted = highlight(filenode.content, lexer, formatter)
+    highlighted = highlight(safe_unicode(filenode.content), lexer, formatter)
     return highlighted
 
 
 class AnnotateHtmlFormatter(HtmlFormatter):
 
     def __init__(self, filenode, annotate_from_changeset_func=None,
             order=None, **options):
         """
         If ``annotate_from_changeset_func`` is passed it should be a function
         which returns string from the given changeset. For example, we may pass
         following function as ``annotate_from_changeset_func``::
 
             def changeset_to_anchor(changeset):
                 return '<a href="/changesets/%s/">%s</a>\n' % \
                        (changeset.id, changeset.id)
 
         :param annotate_from_changeset_func: see above
         :param order: (default: ``['ls', 'annotate', 'code']``); order of
           columns;
         :param options: standard pygment's HtmlFormatter options, there is
           extra option tough, ``headers``. For instance we can pass::
 
              formatter = AnnotateHtmlFormatter(filenode, headers={
                 'ls': '#',
                 'annotate': 'Annotate',
                 'code': 'Code',
              })
 
         """
         super(AnnotateHtmlFormatter, self).__init__(**options)
         self.annotate_from_changeset_func = annotate_from_changeset_func
         self.order = order or ('ls', 'annotate', 'code')
         headers = options.pop('headers', None)
         if headers and not ('ls' in headers and 'annotate' in headers and
             'code' in headers
         ):
             raise ValueError("If headers option dict is specified it must "
                 "all 'ls', 'annotate' and 'code' keys")
         self.headers = headers
         if isinstance(filenode, FileNode):
             self.filenode = filenode
         else:
             raise VCSError("This formatter expect FileNode parameter, not %r"
                 % type(filenode))
 
     def annotate_from_changeset(self, changeset):
         """
         Returns full html line for single changeset per annotated line.
         """
         if self.annotate_from_changeset_func:
             return self.annotate_from_changeset_func(changeset)
         else:
             return ''.join((changeset.id, '\n'))
 
     def _wrap_tablelinenos(self, inner):
         inner_lines = []
         lncount = 0
         for t, line in inner:
             if t:
                 lncount += 1
             inner_lines.append(line)
 
         fl = self.linenostart
         mw = len(str(lncount + fl - 1))
         sp = self.linenospecial
         st = self.linenostep
         la = self.lineanchors
         aln = self.anchorlinenos
         if sp:
             lines = []
 
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if i % sp == 0:
                         if aln:
                             lines.append('<a href="#%s-%d" class="special">'
                                          '%*d</a>' %
                                          (la, i, mw, i))
                         else:
                             lines.append('<span class="special">'
                                          '%*d</span>' % (mw, i))
                     else:
                         if aln:
                             lines.append('<a href="#%s-%d">'
                                          '%*d</a>' % (la, i, mw, i))
                         else:
                             lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
         else:
             lines = []
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if aln:
                         lines.append('<a href="#%s-%d">%*d</a>'
                                      % (la, i, mw, i))
                     else:
                         lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
 
 #        annotate_changesets = [tup[1] for tup in self.filenode.annotate]
 #        # TODO: not sure what that fixes
 #        # If pygments cropped last lines break we need do that too
 #        ln_cs = len(annotate_changesets)
 #        ln_ = len(ls.splitlines())
 #        if  ln_cs > ln_:
 #            annotate_changesets = annotate_changesets[:ln_ - ln_cs]
         annotate = ''.join((self.annotate_from_changeset(el[2]())
                             for el in self.filenode.annotate))
         # in case you wonder about the seemingly redundant <div> here:
         # since the content in the other cell also is wrapped in a div,
         # some browsers in some configurations seem to mess up the formatting.
         '''
         yield 0, ('<table class="%stable">' % self.cssclass +
                   '<tr><td class="linenos"><div class="linenodiv"><pre>' +
                   ls + '</pre></div></td>' +
                   '<td class="code">')
         yield 0, ''.join(inner_lines)
         yield 0, '</td></tr></table>'
 
         '''
         headers_row = []
         if self.headers:
             headers_row = ['<tr class="annotate-header">']
             for key in self.order:
                 td = ''.join(('<td>', self.headers[key], '</td>'))
                 headers_row.append(td)
             headers_row.append('</tr>')
 
         body_row_start = ['<tr>']
         for key in self.order:
             if key == 'ls':
                 body_row_start.append(
                     '<td class="linenos"><div class="linenodiv"><pre>' +
                     ls + '</pre></div></td>')
             elif key == 'annotate':
                 body_row_start.append(
                     '<td class="annotate"><div class="annotatediv"><pre>' +
                     annotate + '</pre></div></td>')
             elif key == 'code':
                 body_row_start.append('<td class="code">')
         yield 0, ('<table class="%stable">' % self.cssclass +
                   ''.join(headers_row) +
                   ''.join(body_row_start)
                   )
         yield 0, ''.join(inner_lines)
         yield 0, '</td></tr></table>'

kallithea/lib/diffs.py

@@ -1,676 +1,679 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.diffs
 ~~~~~~~~~~~~~~~~~~~
 
 Set of diffing helpers, previously part of vcs
 
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Dec 4, 2011
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import difflib
 import logging
 import re
 
 from tg.i18n import ugettext as _
 
 from kallithea.lib import helpers as h
 from kallithea.lib.utils2 import safe_unicode
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.lib.vcs.exceptions import VCSError
 from kallithea.lib.vcs.nodes import FileNode, SubModuleNode
 
 
 log = logging.getLogger(__name__)
 
 
 def _safe_id(idstring):
     r"""Make a string safe for including in an id attribute.
 
     The HTML spec says that id attributes 'must begin with
     a letter ([A-Za-z]) and may be followed by any number
     of letters, digits ([0-9]), hyphens ("-"), underscores
     ("_"), colons (":"), and periods (".")'. These regexps
     are slightly over-zealous, in that they remove colons
     and periods unnecessarily.
 
     Whitespace is transformed into underscores, and then
     anything which is not a hyphen or a character that
     matches \w (alphanumerics and underscore) is removed.
 
     """
     # Transform all whitespace to underscore
     idstring = re.sub(r'\s', "_", idstring)
     # Remove everything that is not a hyphen or a member of \w
     idstring = re.sub(r'(?!-)\W', "", idstring).lower()
     return idstring
 
 
 def as_html(table_class='code-difftable', line_class='line',
             old_lineno_class='lineno old', new_lineno_class='lineno new',
             no_lineno_class='lineno',
             code_class='code', enable_comments=False, parsed_lines=None):
     """
     Return given diff as html table with customized css classes
     """
     def _link_to_if(condition, label, url):
         """
         Generates a link if condition is meet or just the label if not.
         """
 
         if condition:
             return '''<a href="%(url)s" data-pseudo-content="%(label)s"></a>''' % {
                 'url': url,
                 'label': label
             }
         else:
             return label
 
     _html_empty = True
     _html = []
     _html.append('''<table class="%(table_class)s">\n''' % {
         'table_class': table_class
     })
 
     for diff in parsed_lines:
         for line in diff['chunks']:
             _html_empty = False
             for change in line:
                 _html.append('''<tr class="%(lc)s %(action)s">\n''' % {
                     'lc': line_class,
                     'action': change['action']
                 })
                 anchor_old_id = ''
                 anchor_new_id = ''
                 anchor_old = "%(filename)s_o%(oldline_no)s" % {
                     'filename': _safe_id(diff['filename']),
                     'oldline_no': change['old_lineno']
                 }
                 anchor_new = "%(filename)s_n%(oldline_no)s" % {
                     'filename': _safe_id(diff['filename']),
                     'oldline_no': change['new_lineno']
                 }
                 cond_old = (change['old_lineno'] != '...' and
                             change['old_lineno'])
                 cond_new = (change['new_lineno'] != '...' and
                             change['new_lineno'])
                 no_lineno = (change['old_lineno'] == '...' and
                              change['new_lineno'] == '...')
                 if cond_old:
                     anchor_old_id = 'id="%s"' % anchor_old
                 if cond_new:
                     anchor_new_id = 'id="%s"' % anchor_new
                 ###########################################################
                 # OLD LINE NUMBER
                 ###########################################################
                 _html.append('''\t<td %(a_id)s class="%(olc)s" %(colspan)s>''' % {
                     'a_id': anchor_old_id,
                     'olc': no_lineno_class if no_lineno else old_lineno_class,
                     'colspan': 'colspan="2"' if no_lineno else ''
                 })
 
                 _html.append('''%(link)s''' % {
                     'link': _link_to_if(not no_lineno, change['old_lineno'],
                                         '#%s' % anchor_old)
                 })
                 _html.append('''</td>\n''')
                 ###########################################################
                 # NEW LINE NUMBER
                 ###########################################################
 
                 if not no_lineno:
                     _html.append('''\t<td %(a_id)s class="%(nlc)s">''' % {
                         'a_id': anchor_new_id,
                         'nlc': new_lineno_class
                     })
 
                     _html.append('''%(link)s''' % {
                         'link': _link_to_if(True, change['new_lineno'],
                                             '#%s' % anchor_new)
                     })
                     _html.append('''</td>\n''')
                 ###########################################################
                 # CODE
                 ###########################################################
                 comments = '' if enable_comments else 'no-comment'
                 _html.append('''\t<td class="%(cc)s %(inc)s">''' % {
                     'cc': code_class,
                     'inc': comments
                 })
                 _html.append('''\n\t\t<div class="add-bubble"><div>&nbsp;</div></div><pre>%(code)s</pre>\n''' % {
                     'code': change['line']
                 })
 
                 _html.append('''\t</td>''')
                 _html.append('''\n</tr>\n''')
     _html.append('''</table>''')
     if _html_empty:
         return None
     return ''.join(_html)
 
 
 def wrap_to_table(html):
     """Given a string with html, return it wrapped in a table, similar to what
     DiffProcessor returns."""
     return '''\
               <table class="code-difftable">
                 <tr class="line no-comment">
                 <td class="lineno new"></td>
                 <td class="code no-comment"><pre>%s</pre></td>
                 </tr>
               </table>''' % html
 
 
 def wrapped_diff(filenode_old, filenode_new, diff_limit=None,
                 ignore_whitespace=True, line_context=3,
                 enable_comments=False):
     """
     Returns a file diff wrapped into a table.
     Checks for diff_limit and presents a message if the diff is too big.
     """
     if filenode_old is None:
         filenode_old = FileNode(filenode_new.path, '', EmptyChangeset())
 
     op = None
     a_path = filenode_old.path # default, might be overriden by actual rename in diff
     if filenode_old.is_binary or filenode_new.is_binary:
         html_diff = wrap_to_table(_('Binary file'))
         stats = (0, 0)
 
     elif diff_limit != -1 and (
             diff_limit is None or
             (filenode_old.size < diff_limit and filenode_new.size < diff_limit)):
 
         raw_diff = get_gitdiff(filenode_old, filenode_new,
                                 ignore_whitespace=ignore_whitespace,
                                 context=line_context)
         diff_processor = DiffProcessor(raw_diff)
         if diff_processor.parsed: # there should be exactly one element, for the specified file
             f = diff_processor.parsed[0]
             op = f['operation']
             a_path = f['old_filename']
 
         html_diff = as_html(parsed_lines=diff_processor.parsed, enable_comments=enable_comments)
         stats = diff_processor.stat()
 
     else:
         html_diff = wrap_to_table(_('Changeset was too big and was cut off, use '
                                'diff menu to display this diff'))
         stats = (0, 0)
 
     if not html_diff:
         submodules = [o for o in [filenode_new, filenode_old] if isinstance(o, SubModuleNode)]
         if submodules:
             html_diff = wrap_to_table(h.escape('Submodule %r' % submodules[0]))
         else:
             html_diff = wrap_to_table(_('No changes detected'))
 
     cs1 = filenode_old.changeset.raw_id
     cs2 = filenode_new.changeset.raw_id
 
     return cs1, cs2, a_path, html_diff, stats, op
 
 
 def get_gitdiff(filenode_old, filenode_new, ignore_whitespace=True, context=3):
     """
     Returns git style diff between given ``filenode_old`` and ``filenode_new``.
     """
     # make sure we pass in default context
     context = context or 3
     submodules = [o for o in [filenode_new, filenode_old] if isinstance(o, SubModuleNode)]
     if submodules:
         return ''
 
     for filenode in (filenode_old, filenode_new):
         if not isinstance(filenode, FileNode):
             raise VCSError("Given object should be FileNode object, not %s"
                 % filenode.__class__)
 
     repo = filenode_new.changeset.repository
     old_raw_id = getattr(filenode_old.changeset, 'raw_id', repo.EMPTY_CHANGESET)
     new_raw_id = getattr(filenode_new.changeset, 'raw_id', repo.EMPTY_CHANGESET)
 
     vcs_gitdiff = get_diff(repo, old_raw_id, new_raw_id, filenode_new.path,
                            ignore_whitespace, context)
     return vcs_gitdiff
 
 
 def get_diff(scm_instance, rev1, rev2, path=None, ignore_whitespace=False, context=3):
     """
     A thin wrapper around vcs lib get_diff.
     """
     try:
         return scm_instance.get_diff(rev1, rev2, path=path,
                                      ignore_whitespace=ignore_whitespace, context=context)
     except MemoryError:
         h.flash('MemoryError: Diff is too big', category='error')
         return ''
 
 
 NEW_FILENODE = 1
 DEL_FILENODE = 2
 MOD_FILENODE = 3
 RENAMED_FILENODE = 4
 COPIED_FILENODE = 5
 CHMOD_FILENODE = 6
 BIN_FILENODE = 7
 
 
 class DiffProcessor(object):
     """
     Give it a unified or git diff and it returns a list of the files that were
     mentioned in the diff together with a dict of meta information that
     can be used to render it in a HTML template.
     """
     _diff_git_re = re.compile('^diff --git', re.MULTILINE)
 
     def __init__(self, diff, vcs='hg', diff_limit=None, inline_diff=True):
         """
         :param diff:   a text in diff format
         :param vcs: type of version control hg or git
         :param diff_limit: define the size of diff that is considered "big"
             based on that parameter cut off will be triggered, set to None
             to show full diff
         """
-        if not isinstance(diff, basestring):
-            raise Exception('Diff must be a basestring got %s instead' % type(diff))
+        if not isinstance(diff, bytes):
+            raise Exception('Diff must be bytes - got %s' % type(diff))
 
         self._diff = diff
         self.adds = 0
         self.removes = 0
         self.diff_limit = diff_limit
         self.limited_diff = False
         self.vcs = vcs
         self.parsed = self._parse_gitdiff(inline_diff=inline_diff)
 
     def _parse_gitdiff(self, inline_diff):
         """Parse self._diff and return a list of dicts with meta info and chunks for each file.
         Might set limited_diff.
         Optionally, do an extra pass and to extra markup of one-liner changes.
         """
         _files = [] # list of dicts with meta info and chunks
 
         starts = [m.start() for m in self._diff_git_re.finditer(self._diff)]
         starts.append(len(self._diff))
 
         for start, end in zip(starts, starts[1:]):
             if self.diff_limit and end > self.diff_limit:
                 self.limited_diff = True
                 continue
 
             head, diff_lines = _get_header(self.vcs, buffer(self._diff, start, end - start))
 
             op = None
             stats = {
                 'added': 0,
                 'deleted': 0,
                 'binary': False,
                 'ops': {},
             }
 
             if head['deleted_file_mode']:
                 op = 'removed'
                 stats['binary'] = True
                 stats['ops'][DEL_FILENODE] = 'deleted file'
 
             elif head['new_file_mode']:
                 op = 'added'
                 stats['binary'] = True
                 stats['ops'][NEW_FILENODE] = 'new file %s' % head['new_file_mode']
             else:  # modify operation, can be cp, rename, chmod
                 # CHMOD
                 if head['new_mode'] and head['old_mode']:
                     op = 'modified'
                     stats['binary'] = True
                     stats['ops'][CHMOD_FILENODE] = ('modified file chmod %s => %s'
                                         % (head['old_mode'], head['new_mode']))
                 # RENAME
                 if (head['rename_from'] and head['rename_to']
                       and head['rename_from'] != head['rename_to']):
                     op = 'renamed'
                     stats['binary'] = True
                     stats['ops'][RENAMED_FILENODE] = ('file renamed from %s to %s'
                                     % (head['rename_from'], head['rename_to']))
                 # COPY
                 if head.get('copy_from') and head.get('copy_to'):
                     op = 'modified'
                     stats['binary'] = True
                     stats['ops'][COPIED_FILENODE] = ('file copied from %s to %s'
                                         % (head['copy_from'], head['copy_to']))
                 # FALL BACK: detect missed old style add or remove
                 if op is None:
                     if not head['a_file'] and head['b_file']:
                         op = 'added'
                         stats['binary'] = True
                         stats['ops'][NEW_FILENODE] = 'new file'
 
                     elif head['a_file'] and not head['b_file']:
                         op = 'removed'
                         stats['binary'] = True
                         stats['ops'][DEL_FILENODE] = 'deleted file'
 
                 # it's not ADD not DELETE
                 if op is None:
                     op = 'modified'
                     stats['binary'] = True
                     stats['ops'][MOD_FILENODE] = 'modified file'
 
             # a real non-binary diff
             if head['a_file'] or head['b_file']:
                 chunks, added, deleted = _parse_lines(diff_lines)
                 stats['binary'] = False
                 stats['added'] = added
                 stats['deleted'] = deleted
                 # explicit mark that it's a modified file
                 if op == 'modified':
                     stats['ops'][MOD_FILENODE] = 'modified file'
             else:  # Git binary patch (or empty diff)
                 # Git binary patch
                 if head['bin_patch']:
                     stats['ops'][BIN_FILENODE] = 'binary diff not shown'
                 chunks = []
 
             if op == 'removed' and chunks:
                 # a way of seeing deleted content could perhaps be nice - but
                 # not with the current UI
                 chunks = []
 
             chunks.insert(0, [{
                 'old_lineno': '',
                 'new_lineno': '',
                 'action':     'context',
                 'line':       msg,
                 } for _op, msg in stats['ops'].iteritems()
                   if _op not in [MOD_FILENODE]])
 
             _files.append({
                 'old_filename':     head['a_path'],
                 'filename':         head['b_path'],
                 'old_revision':     head['a_blob_id'],
                 'new_revision':     head['b_blob_id'],
                 'chunks':           chunks,
                 'operation':        op,
                 'stats':            stats,
             })
 
         if not inline_diff:
             return _files
 
         # highlight inline changes when one del is followed by one add
         for diff_data in _files:
             for chunk in diff_data['chunks']:
                 lineiter = iter(chunk)
                 try:
                     peekline = lineiter.next()
                     while True:
                         # find a first del line
                         while peekline['action'] != 'del':
                             peekline = lineiter.next()
                         delline = peekline
                         peekline = lineiter.next()
                         # if not followed by add, eat all following del lines
                         if peekline['action'] != 'add':
                             while peekline['action'] == 'del':
                                 peekline = lineiter.next()
                             continue
                         # found an add - make sure it is the only one
                         addline = peekline
                         try:
                             peekline = lineiter.next()
                         except StopIteration:
                             # add was last line - ok
                             _highlight_inline_diff(delline, addline)
                             raise
                         if peekline['action'] != 'add':
                             # there was only one add line - ok
                             _highlight_inline_diff(delline, addline)
                 except StopIteration:
                     pass
 
         return _files
 
     def stat(self):
         """
         Returns tuple of added, and removed lines for this instance
         """
         return self.adds, self.removes
 
 
 _escape_re = re.compile(r'(&)|(<)|(>)|(\t)|(\r)|(?<=.)( \n| $)')
 
 
 def _escaper(string):
     """
     Do HTML escaping/markup
     """
 
     def substitute(m):
         groups = m.groups()
         if groups[0]:
             return '&amp;'
         if groups[1]:
             return '&lt;'
         if groups[2]:
             return '&gt;'
         if groups[3]:
             return '<u>\t</u>'
         if groups[4]:
             return '<u class="cr"></u>'
         if groups[5]:
             return ' <i></i>'
         assert False
 
     return _escape_re.sub(substitute, safe_unicode(string))
 
 
 _git_header_re = re.compile(r"""
     ^diff[ ]--git[ ]a/(?P<a_path>.+?)[ ]b/(?P<b_path>.+?)\n
     (?:^old[ ]mode[ ](?P<old_mode>\d+)\n
        ^new[ ]mode[ ](?P<new_mode>\d+)(?:\n|$))?
     (?:^similarity[ ]index[ ](?P<similarity_index>\d+)%\n
        ^rename[ ]from[ ](?P<rename_from>.+)\n
        ^rename[ ]to[ ](?P<rename_to>.+)(?:\n|$))?
     (?:^new[ ]file[ ]mode[ ](?P<new_file_mode>.+)(?:\n|$))?
     (?:^deleted[ ]file[ ]mode[ ](?P<deleted_file_mode>.+)(?:\n|$))?
     (?:^index[ ](?P<a_blob_id>[0-9A-Fa-f]+)
         \.\.(?P<b_blob_id>[0-9A-Fa-f]+)[ ]?(?P<b_mode>.+)?(?:\n|$))?
     (?:^(?P<bin_patch>GIT[ ]binary[ ]patch)(?:\n|$))?
     (?:^---[ ](a/(?P<a_file>.+?)|/dev/null)\t?(?:\n|$))?
     (?:^\+\+\+[ ](b/(?P<b_file>.+?)|/dev/null)\t?(?:\n|$))?
 """, re.VERBOSE | re.MULTILINE)
 
 
 _hg_header_re = re.compile(r"""
     ^diff[ ]--git[ ]a/(?P<a_path>.+?)[ ]b/(?P<b_path>.+?)\n
     (?:^old[ ]mode[ ](?P<old_mode>\d+)\n
        ^new[ ]mode[ ](?P<new_mode>\d+)(?:\n|$))?
     (?:^similarity[ ]index[ ](?P<similarity_index>\d+)%(?:\n|$))?
     (?:^rename[ ]from[ ](?P<rename_from>.+)\n
        ^rename[ ]to[ ](?P<rename_to>.+)(?:\n|$))?
     (?:^copy[ ]from[ ](?P<copy_from>.+)\n
        ^copy[ ]to[ ](?P<copy_to>.+)(?:\n|$))?
     (?:^new[ ]file[ ]mode[ ](?P<new_file_mode>.+)(?:\n|$))?
     (?:^deleted[ ]file[ ]mode[ ](?P<deleted_file_mode>.+)(?:\n|$))?
     (?:^index[ ](?P<a_blob_id>[0-9A-Fa-f]+)
         \.\.(?P<b_blob_id>[0-9A-Fa-f]+)[ ]?(?P<b_mode>.+)?(?:\n|$))?
     (?:^(?P<bin_patch>GIT[ ]binary[ ]patch)(?:\n|$))?
     (?:^---[ ](a/(?P<a_file>.+?)|/dev/null)\t?(?:\n|$))?
     (?:^\+\+\+[ ](b/(?P<b_file>.+?)|/dev/null)\t?(?:\n|$))?
 """, re.VERBOSE | re.MULTILINE)
 
 
+_header_next_check = re.compile(br'''(?!@)(?!literal )(?!delta )''')
+
+
 def _get_header(vcs, diff_chunk):
     """
     Parses a Git diff for a single file (header and chunks) and returns a tuple with:
 
     1. A dict with meta info:
 
         a_path, b_path, similarity_index, rename_from, rename_to,
         old_mode, new_mode, new_file_mode, deleted_file_mode,
         a_blob_id, b_blob_id, b_mode, a_file, b_file
 
     2. An iterator yielding lines with simple HTML markup.
     """
     match = None
     if vcs == 'git':
         match = _git_header_re.match(diff_chunk)
     elif vcs == 'hg':
         match = _hg_header_re.match(diff_chunk)
     if match is None:
         raise Exception('diff not recognized as valid %s diff' % vcs)
     meta_info = match.groupdict()
     rest = diff_chunk[match.end():]
-    if rest and not rest.startswith('@') and not rest.startswith('literal ') and not rest.startswith('delta '):
+    if rest and _header_next_check.match(rest):
         raise Exception('cannot parse %s diff header: %r followed by %r' % (vcs, diff_chunk[:match.end()], rest[:1000]))
     diff_lines = (_escaper(m.group(0)) for m in re.finditer(r'.*\n|.+$', rest)) # don't split on \r as str.splitlines do
     return meta_info, diff_lines
 
 
 _chunk_re = re.compile(r'^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)')
 _newline_marker = re.compile(r'^\\ No newline at end of file')
 
 
 def _parse_lines(diff_lines):
     """
     Given an iterator of diff body lines, parse them and return a dict per
     line and added/removed totals.
     """
     added = deleted = 0
     old_line = old_end = new_line = new_end = None
 
     try:
         chunks = []
         line = diff_lines.next()
 
         while True:
             lines = []
             chunks.append(lines)
 
             match = _chunk_re.match(line)
 
             if not match:
                 raise Exception('error parsing diff @@ line %r' % line)
 
             gr = match.groups()
             (old_line, old_end,
              new_line, new_end) = [int(x or 1) for x in gr[:-1]]
             old_line -= 1
             new_line -= 1
 
             context = len(gr) == 5
             old_end += old_line
             new_end += new_line
 
             if context:
                 # skip context only if it's first line
                 if int(gr[0]) > 1:
                     lines.append({
                         'old_lineno': '...',
                         'new_lineno': '...',
                         'action':     'context',
                         'line':       line,
                     })
 
             line = diff_lines.next()
 
             while old_line < old_end or new_line < new_end:
                 if not line:
                     raise Exception('error parsing diff - empty line at -%s+%s' % (old_line, new_line))
 
                 affects_old = affects_new = False
 
                 command = line[0]
                 if command == '+':
                     affects_new = True
                     action = 'add'
                     added += 1
                 elif command == '-':
                     affects_old = True
                     action = 'del'
                     deleted += 1
                 elif command == ' ':
                     affects_old = affects_new = True
                     action = 'unmod'
                 else:
                     raise Exception('error parsing diff - unknown command in line %r at -%s+%s' % (line, old_line, new_line))
 
                 if not _newline_marker.match(line):
                     old_line += affects_old
                     new_line += affects_new
                     lines.append({
                         'old_lineno':   affects_old and old_line or '',
                         'new_lineno':   affects_new and new_line or '',
                         'action':       action,
                         'line':         line[1:],
                     })
 
                 line = diff_lines.next()
 
                 if _newline_marker.match(line):
                     # we need to append to lines, since this is not
                     # counted in the line specs of diff
                     lines.append({
                         'old_lineno':   '...',
                         'new_lineno':   '...',
                         'action':       'context',
                         'line':         line,
                     })
                     line = diff_lines.next()
             if old_line > old_end:
                 raise Exception('error parsing diff - more than %s "-" lines at -%s+%s' % (old_end, old_line, new_line))
             if new_line > new_end:
                 raise Exception('error parsing diff - more than %s "+" lines at -%s+%s' % (new_end, old_line, new_line))
     except StopIteration:
         pass
     if old_line != old_end or new_line != new_end:
         raise Exception('diff processing broken when old %s<>%s or new %s<>%s line %r' % (old_line, old_end, new_line, new_end, line))
 
     return chunks, added, deleted
 
 # Used for inline highlighter word split, must match the substitutions in _escaper
 _token_re = re.compile(r'()(&amp;|&lt;|&gt;|<u>\t</u>|<u class="cr"></u>| <i></i>|\W+?)')
 
 
 def _highlight_inline_diff(old, new):
     """
     Highlight simple add/remove in two lines given as info dicts. They are
     modified in place and given markup with <del>/<ins>.
     """
     assert old['action'] == 'del'
     assert new['action'] == 'add'
 
     oldwords = _token_re.split(old['line'])
     newwords = _token_re.split(new['line'])
     sequence = difflib.SequenceMatcher(None, oldwords, newwords)
 
     oldfragments, newfragments = [], []
     for tag, i1, i2, j1, j2 in sequence.get_opcodes():
         oldfrag = ''.join(oldwords[i1:i2])
         newfrag = ''.join(newwords[j1:j2])
         if tag != 'equal':
             if oldfrag:
                 oldfrag = '<del>%s</del>' % oldfrag
             if newfrag:
                 newfrag = '<ins>%s</ins>' % newfrag
         oldfragments.append(oldfrag)
         newfragments.append(newfrag)
 
     old['line'] = "".join(oldfragments)
     new['line'] = "".join(newfragments)

kallithea/lib/helpers.py

@@ -1,1325 +1,1325 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 Helper functions
 
 Consists of functions to typically be used within templates, but also
 available to Controllers. This module is available to both as 'h'.
 """
 import hashlib
 import json
 import logging
 import random
 import re
 import textwrap
 import urlparse
 
 from beaker.cache import cache_region
 from pygments import highlight as code_highlight
 from pygments.formatters.html import HtmlFormatter
 from tg.i18n import ugettext as _
 from webhelpers2.html import HTML, escape, literal
 from webhelpers2.html.tags import NotGiven, Option, Options, _input, _make_safe_id_component, checkbox, end_form
 from webhelpers2.html.tags import form as insecure_form
 from webhelpers2.html.tags import hidden, link_to, password, radio
 from webhelpers2.html.tags import select as webhelpers2_select
 from webhelpers2.html.tags import submit, text, textarea
 from webhelpers2.number import format_byte_size
 from webhelpers2.text import chop_at, truncate, wrap_paragraphs
 
 from kallithea.config.routing import url
 from kallithea.lib.annotate import annotate_highlight
 #==============================================================================
 # PERMS
 #==============================================================================
 from kallithea.lib.auth import HasPermissionAny, HasRepoGroupPermissionLevel, HasRepoPermissionLevel
 from kallithea.lib.markup_renderer import url_re
 from kallithea.lib.pygmentsutils import get_custom_lexer
 from kallithea.lib.utils2 import MENTIONS_REGEX, AttributeDict
 from kallithea.lib.utils2 import age as _age
 from kallithea.lib.utils2 import credentials_filter, safe_int, safe_str, safe_unicode, str2bool, time_to_datetime
 from kallithea.lib.vcs.backends.base import BaseChangeset, EmptyChangeset
 from kallithea.lib.vcs.exceptions import ChangesetDoesNotExistError
 #==============================================================================
 # SCM FILTERS available via h.
 #==============================================================================
 from kallithea.lib.vcs.utils import author_email, author_name
 
 
 log = logging.getLogger(__name__)
 
 
 def canonical_url(*args, **kargs):
     '''Like url(x, qualified=True), but returns url that not only is qualified
     but also canonical, as configured in canonical_url'''
     from kallithea import CONFIG
     try:
         parts = CONFIG.get('canonical_url', '').split('://', 1)
         kargs['host'] = parts[1]
         kargs['protocol'] = parts[0]
     except IndexError:
         kargs['qualified'] = True
     return url(*args, **kargs)
 
 
 def canonical_hostname():
     '''Return canonical hostname of system'''
     from kallithea import CONFIG
     try:
         parts = CONFIG.get('canonical_url', '').split('://', 1)
         return parts[1].split('/', 1)[0]
     except IndexError:
         parts = url('home', qualified=True).split('://', 1)
         return parts[1].split('/', 1)[0]
 
 
 def html_escape(s):
     """Return string with all html escaped.
     This is also safe for javascript in html but not necessarily correct.
     """
     return (s
         .replace('&', '&amp;')
         .replace(">", "&gt;")
         .replace("<", "&lt;")
         .replace('"', "&quot;")
         .replace("'", "&apos;") # Note: this is HTML5 not HTML4 and might not work in mails
         )
 
 def js(value):
     """Convert Python value to the corresponding JavaScript representation.
 
     This is necessary to safely insert arbitrary values into HTML <script>
     sections e.g. using Mako template expression substitution.
 
     Note: Rather than using this function, it's preferable to avoid the
     insertion of values into HTML <script> sections altogether. Instead,
     data should (to the extent possible) be passed to JavaScript using
     data attributes or AJAX calls, eliminating the need for JS specific
     escaping.
 
     Note: This is not safe for use in attributes (e.g. onclick), because
     quotes are not escaped.
 
     Because the rules for parsing <script> varies between XHTML (where
     normal rules apply for any special characters) and HTML (where
     entities are not interpreted, but the literal string "</script>"
     is forbidden), the function ensures that the result never contains
     '&', '<' and '>', thus making it safe in both those contexts (but
     not in attributes).
     """
     return literal(
         ('(' + json.dumps(value) + ')')
         # In JSON, the following can only appear in string literals.
         .replace('&', r'\x26')
         .replace('<', r'\x3c')
         .replace('>', r'\x3e')
     )
 
 
 def jshtml(val):
     """HTML escapes a string value, then converts the resulting string
     to its corresponding JavaScript representation (see `js`).
 
     This is used when a plain-text string (possibly containing special
     HTML characters) will be used by a script in an HTML context (e.g.
     element.innerHTML or jQuery's 'html' method).
 
     If in doubt, err on the side of using `jshtml` over `js`, since it's
     better to escape too much than too little.
     """
     return js(escape(val))
 
 
 def shorter(s, size=20, firstline=False, postfix='...'):
     """Truncate s to size, including the postfix string if truncating.
     If firstline, truncate at newline.
     """
     if firstline:
         s = s.split('\n', 1)[0].rstrip()
     if len(s) > size:
         return s[:size - len(postfix)] + postfix
     return s
 
 
 def reset(name, value, id=NotGiven, **attrs):
     """Create a reset button, similar to webhelpers2.html.tags.submit ."""
     return _input("reset", name, value, id, attrs)
 
 
 def select(name, selected_values, options, id=NotGiven, **attrs):
     """Convenient wrapper of webhelpers2 to let it accept options as a tuple list"""
     if isinstance(options, list):
         option_list = options
         # Handle old value,label lists ... where value also can be value,label lists
         options = Options()
         for x in option_list:
             if isinstance(x, tuple) and len(x) == 2:
                 value, label = x
             elif isinstance(x, basestring):
                 value = label = x
             else:
                 log.error('invalid select option %r', x)
                 raise
             if isinstance(value, list):
                 og = options.add_optgroup(label)
                 for x in value:
                     if isinstance(x, tuple) and len(x) == 2:
                         group_value, group_label = x
                     elif isinstance(x, basestring):
                         group_value = group_label = x
                     else:
                         log.error('invalid select option %r', x)
                         raise
                     og.add_option(group_label, group_value)
             else:
                 options.add_option(label, value)
     return webhelpers2_select(name, selected_values, options, id=id, **attrs)
 
 
 safeid = _make_safe_id_component
 
 
 def FID(raw_id, path):
     """
     Creates a unique ID for filenode based on it's hash of path and revision
     it's safe to use in urls
 
     :param raw_id:
     :param path:
     """
 
     return 'C-%s-%s' % (short_id(raw_id), hashlib.md5(safe_str(path)).hexdigest()[:12])
 
 
 class _FilesBreadCrumbs(object):
 
     def __call__(self, repo_name, rev, paths):
         if isinstance(paths, str):
             paths = safe_unicode(paths)
         url_l = [link_to(repo_name, url('files_home',
                                         repo_name=repo_name,
                                         revision=rev, f_path=''),
                          class_='ypjax-link')]
         paths_l = paths.split('/')
         for cnt, p in enumerate(paths_l):
             if p != '':
                 url_l.append(link_to(p,
                                      url('files_home',
                                          repo_name=repo_name,
                                          revision=rev,
                                          f_path='/'.join(paths_l[:cnt + 1])
                                          ),
                                      class_='ypjax-link'
                                      )
                              )
 
         return literal('/'.join(url_l))
 
 
 files_breadcrumbs = _FilesBreadCrumbs()
 
 
 class CodeHtmlFormatter(HtmlFormatter):
     """
     My code Html Formatter for source codes
     """
 
     def wrap(self, source, outfile):
         return self._wrap_div(self._wrap_pre(self._wrap_code(source)))
 
     def _wrap_code(self, source):
         for cnt, it in enumerate(source):
             i, t = it
             t = '<span id="L%s">%s</span>' % (cnt + 1, t)
             yield i, t
 
     def _wrap_tablelinenos(self, inner):
         inner_lines = []
         lncount = 0
         for t, line in inner:
             if t:
                 lncount += 1
             inner_lines.append(line)
 
         fl = self.linenostart
         mw = len(str(lncount + fl - 1))
         sp = self.linenospecial
         st = self.linenostep
         la = self.lineanchors
         aln = self.anchorlinenos
         nocls = self.noclasses
         if sp:
             lines = []
 
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if i % sp == 0:
                         if aln:
                             lines.append('<a href="#%s%d" class="special">%*d</a>' %
                                          (la, i, mw, i))
                         else:
                             lines.append('<span class="special">%*d</span>' % (mw, i))
                     else:
                         if aln:
                             lines.append('<a href="#%s%d">%*d</a>' % (la, i, mw, i))
                         else:
                             lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
         else:
             lines = []
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if aln:
                         lines.append('<a href="#%s%d">%*d</a>' % (la, i, mw, i))
                     else:
                         lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
 
         # in case you wonder about the seemingly redundant <div> here: since the
         # content in the other cell also is wrapped in a div, some browsers in
         # some configurations seem to mess up the formatting...
         if nocls:
             yield 0, ('<table class="%stable">' % self.cssclass +
                       '<tr><td><div class="linenodiv">'
                       '<pre>' + ls + '</pre></div></td>'
                       '<td id="hlcode" class="code">')
         else:
             yield 0, ('<table class="%stable">' % self.cssclass +
                       '<tr><td class="linenos"><div class="linenodiv">'
                       '<pre>' + ls + '</pre></div></td>'
                       '<td id="hlcode" class="code">')
         yield 0, ''.join(inner_lines)
         yield 0, '</td></tr></table>'
 
 
 _whitespace_re = re.compile(r'(\t)|( )(?=\n|</div>)')
 
 
 def _markup_whitespace(m):
     groups = m.groups()
     if groups[0]:
         return '<u>\t</u>'
     if groups[1]:
         return ' <i></i>'
 
 
 def markup_whitespace(s):
     return _whitespace_re.sub(_markup_whitespace, s)
 
 
 def pygmentize(filenode, **kwargs):
     """
     pygmentize function using pygments
 
     :param filenode:
     """
     lexer = get_custom_lexer(filenode.extension) or filenode.lexer
     return literal(markup_whitespace(
-        code_highlight(filenode.content, lexer, CodeHtmlFormatter(**kwargs))))
+        code_highlight(safe_unicode(filenode.content), lexer, CodeHtmlFormatter(**kwargs))))
 
 
 def pygmentize_annotation(repo_name, filenode, **kwargs):
     """
     pygmentize function for annotation
 
     :param filenode:
     """
 
     color_dict = {}
 
     def gen_color(n=10000):
         """generator for getting n of evenly distributed colors using
         hsv color and golden ratio. It always return same order of colors
 
         :returns: RGB tuple
         """
 
         def hsv_to_rgb(h, s, v):
             if s == 0.0:
                 return v, v, v
             i = int(h * 6.0)  # XXX assume int() truncates!
             f = (h * 6.0) - i
             p = v * (1.0 - s)
             q = v * (1.0 - s * f)
             t = v * (1.0 - s * (1.0 - f))
             i = i % 6
             if i == 0:
                 return v, t, p
             if i == 1:
                 return q, v, p
             if i == 2:
                 return p, v, t
             if i == 3:
                 return p, q, v
             if i == 4:
                 return t, p, v
             if i == 5:
                 return v, p, q
 
         golden_ratio = 0.618033988749895
         h = 0.22717784590367374
 
         for _unused in xrange(n):
             h += golden_ratio
             h %= 1
             HSV_tuple = [h, 0.95, 0.95]
             RGB_tuple = hsv_to_rgb(*HSV_tuple)
             yield [str(int(x * 256)) for x in RGB_tuple]
 
     cgenerator = gen_color()
 
     def get_color_string(cs):
         if cs in color_dict:
             col = color_dict[cs]
         else:
             col = color_dict[cs] = cgenerator.next()
         return "color: rgb(%s)! important;" % (', '.join(col))
 
     def url_func(repo_name):
 
         def _url_func(changeset):
             author = escape(changeset.author)
             date = changeset.date
             message = escape(changeset.message)
             tooltip_html = ("<b>Author:</b> %s<br/>"
                             "<b>Date:</b> %s</b><br/>"
                             "<b>Message:</b> %s") % (author, date, message)
 
             lnk_format = show_id(changeset)
             uri = link_to(
                     lnk_format,
                     url('changeset_home', repo_name=repo_name,
                         revision=changeset.raw_id),
                     style=get_color_string(changeset.raw_id),
                     **{'data-toggle': 'popover',
                        'data-content': tooltip_html}
                   )
 
             uri += '\n'
             return uri
         return _url_func
 
     return literal(markup_whitespace(annotate_highlight(filenode, url_func(repo_name), **kwargs)))
 
 
 class _Message(object):
     """A message returned by ``pop_flash_messages()``.
 
     Converting the message to a string returns the message text. Instances
     also have the following attributes:
 
     * ``message``: the message text.
     * ``category``: the category specified when the message was created.
     """
 
     def __init__(self, category, message):
         self.category = category
         self.message = message
 
     def __str__(self):
         return self.message
 
     __unicode__ = __str__
 
     def __html__(self):
         return escape(safe_unicode(self.message))
 
 
 def _session_flash_messages(append=None, clear=False):
     """Manage a message queue in tg.session: return the current message queue
     after appending the given message, and possibly clearing the queue."""
     key = 'flash'
     from tg import session
     if key in session:
         flash_messages = session[key]
     else:
         if append is None:  # common fast path - also used for clearing empty queue
             return []  # don't bother saving
         flash_messages = []
         session[key] = flash_messages
     if append is not None and append not in flash_messages:
         flash_messages.append(append)
     if clear:
         session.pop(key, None)
     session.save()
     return flash_messages
 
 
 def flash(message, category=None, logf=None):
     """
     Show a message to the user _and_ log it through the specified function
 
     category: notice (default), warning, error, success
     logf: a custom log function - such as log.debug
 
     logf defaults to log.info, unless category equals 'success', in which
     case logf defaults to log.debug.
     """
     if logf is None:
         logf = log.info
         if category == 'success':
             logf = log.debug
 
     logf('Flash %s: %s', category, message)
 
     _session_flash_messages(append=(category, message))
 
 
 def pop_flash_messages():
     """Return all accumulated messages and delete them from the session.
 
     The return value is a list of ``Message`` objects.
     """
     return [_Message(*m) for m in _session_flash_messages(clear=True)]
 
 
 age = lambda x, y=False: _age(x, y)
 capitalize = lambda x: x.capitalize()
 email = author_email
 short_id = lambda x: x[:12]
 hide_credentials = lambda x: ''.join(credentials_filter(x))
 
 
 def show_id(cs):
     """
     Configurable function that shows ID
     by default it's r123:fffeeefffeee
 
     :param cs: changeset instance
     """
     from kallithea import CONFIG
     def_len = safe_int(CONFIG.get('show_sha_length', 12))
     show_rev = str2bool(CONFIG.get('show_revision_number', False))
 
     raw_id = cs.raw_id[:def_len]
     if show_rev:
         return 'r%s:%s' % (cs.revision, raw_id)
     else:
         return raw_id
 
 
 def fmt_date(date):
     if date:
         return date.strftime("%Y-%m-%d %H:%M:%S")
     return ""
 
 
 def is_git(repository):
     if hasattr(repository, 'alias'):
         _type = repository.alias
     elif hasattr(repository, 'repo_type'):
         _type = repository.repo_type
     else:
         _type = repository
     return _type == 'git'
 
 
 def is_hg(repository):
     if hasattr(repository, 'alias'):
         _type = repository.alias
     elif hasattr(repository, 'repo_type'):
         _type = repository.repo_type
     else:
         _type = repository
     return _type == 'hg'
 
 
 @cache_region('long_term', 'user_attr_or_none')
 def user_attr_or_none(author, show_attr):
     """Try to match email part of VCS committer string with a local user and return show_attr
     - or return None if user not found"""
     email = author_email(author)
     if email:
         from kallithea.model.db import User
         user = User.get_by_email(email, cache=True) # cache will only use sql_cache_short
         if user is not None:
             return getattr(user, show_attr)
     return None
 
 
 def email_or_none(author):
     """Try to match email part of VCS committer string with a local user.
     Return primary email of user, email part of the specified author name, or None."""
     if not author:
         return None
     email = user_attr_or_none(author, 'email')
     if email is not None:
         return email # always use user's main email address - not necessarily the one used to find user
 
     # extract email from the commit string
     email = author_email(author)
     if email:
         return email
 
     # No valid email, not a valid user in the system, none!
     return None
 
 
 def person(author, show_attr="username"):
     """Find the user identified by 'author', return one of the users attributes,
     default to the username attribute, None if there is no user"""
     from kallithea.model.db import User
     # if author is already an instance use it for extraction
     if isinstance(author, User):
         return getattr(author, show_attr)
 
     value = user_attr_or_none(author, show_attr)
     if value is not None:
         return value
 
     # Still nothing?  Just pass back the author name if any, else the email
     return author_name(author) or email(author)
 
 
 def person_by_id(id_, show_attr="username"):
     from kallithea.model.db import User
     # attr to return from fetched user
     person_getter = lambda usr: getattr(usr, show_attr)
 
     # maybe it's an ID ?
     if str(id_).isdigit() or isinstance(id_, int):
         id_ = int(id_)
         user = User.get(id_)
         if user is not None:
             return person_getter(user)
     return id_
 
 
 def boolicon(value):
     """Returns boolean value of a value, represented as small html image of true/false
     icons
 
     :param value: value
     """
 
     if value:
         return HTML.tag('i', class_="icon-ok")
     else:
         return HTML.tag('i', class_="icon-minus-circled")
 
 
 def action_parser(user_log, feed=False, parse_cs=False):
     """
     This helper will action_map the specified string action into translated
     fancy names with icons and links
 
     :param user_log: user log instance
     :param feed: use output for feeds (no html and fancy icons)
     :param parse_cs: parse Changesets into VCS instances
     """
 
     action = user_log.action
     action_params = ' '
 
     x = action.split(':')
 
     if len(x) > 1:
         action, action_params = x
 
     def get_cs_links():
         revs_limit = 3  # display this amount always
         revs_top_limit = 50  # show upto this amount of changesets hidden
         revs_ids = action_params.split(',')
         deleted = user_log.repository is None
         if deleted:
             return ','.join(revs_ids)
 
         repo_name = user_log.repository.repo_name
 
         def lnk(rev, repo_name):
             lazy_cs = False
             title_ = None
             url_ = '#'
             if isinstance(rev, BaseChangeset) or isinstance(rev, AttributeDict):
                 if rev.op and rev.ref_name:
                     if rev.op == 'delete_branch':
                         lbl = _('Deleted branch: %s') % rev.ref_name
                     elif rev.op == 'tag':
                         lbl = _('Created tag: %s') % rev.ref_name
                     else:
                         lbl = 'Unknown operation %s' % rev.op
                 else:
                     lazy_cs = True
                     lbl = rev.short_id[:8]
                     url_ = url('changeset_home', repo_name=repo_name,
                                revision=rev.raw_id)
             else:
                 # changeset cannot be found - it might have been stripped or removed
                 lbl = rev[:12]
                 title_ = _('Changeset %s not found') % lbl
             if parse_cs:
                 return link_to(lbl, url_, title=title_, **{'data-toggle': 'tooltip'})
             return link_to(lbl, url_, class_='lazy-cs' if lazy_cs else '',
                            **{'data-raw_id': rev.raw_id, 'data-repo_name': repo_name})
 
         def _get_op(rev_txt):
             _op = None
             _name = rev_txt
             if len(rev_txt.split('=>')) == 2:
                 _op, _name = rev_txt.split('=>')
             return _op, _name
 
         revs = []
         if len([v for v in revs_ids if v != '']) > 0:
             repo = None
             for rev in revs_ids[:revs_top_limit]:
                 _op, _name = _get_op(rev)
 
                 # we want parsed changesets, or new log store format is bad
                 if parse_cs:
                     try:
                         if repo is None:
                             repo = user_log.repository.scm_instance
                         _rev = repo.get_changeset(rev)
                         revs.append(_rev)
                     except ChangesetDoesNotExistError:
                         log.error('cannot find revision %s in this repo', rev)
                         revs.append(rev)
                 else:
                     _rev = AttributeDict({
                         'short_id': rev[:12],
                         'raw_id': rev,
                         'message': '',
                         'op': _op,
                         'ref_name': _name
                     })
                     revs.append(_rev)
         cs_links = [" " + ', '.join(
             [lnk(rev, repo_name) for rev in revs[:revs_limit]]
         )]
         _op1, _name1 = _get_op(revs_ids[0])
         _op2, _name2 = _get_op(revs_ids[-1])
 
         _rev = '%s...%s' % (_name1, _name2)
 
         compare_view = (
             ' <div class="compare_view" data-toggle="tooltip" title="%s">'
             '<a href="%s">%s</a> </div>' % (
                 _('Show all combined changesets %s->%s') % (
                     revs_ids[0][:12], revs_ids[-1][:12]
                 ),
                 url('changeset_home', repo_name=repo_name,
                     revision=_rev
                 ),
                 _('Compare view')
             )
         )
 
         # if we have exactly one more than normally displayed
         # just display it, takes less space than displaying
         # "and 1 more revisions"
         if len(revs_ids) == revs_limit + 1:
             cs_links.append(", " + lnk(revs[revs_limit], repo_name))
 
         # hidden-by-default ones
         if len(revs_ids) > revs_limit + 1:
             uniq_id = revs_ids[0]
             html_tmpl = (
                 '<span> %s <a class="show_more" id="_%s" '
                 'href="#more">%s</a> %s</span>'
             )
             if not feed:
                 cs_links.append(html_tmpl % (
                       _('and'),
                       uniq_id, _('%s more') % (len(revs_ids) - revs_limit),
                       _('revisions')
                     )
                 )
 
             if not feed:
                 html_tmpl = '<span id="%s" style="display:none">, %s </span>'
             else:
                 html_tmpl = '<span id="%s"> %s </span>'
 
             morelinks = ', '.join(
               [lnk(rev, repo_name) for rev in revs[revs_limit:]]
             )
 
             if len(revs_ids) > revs_top_limit:
                 morelinks += ', ...'
 
             cs_links.append(html_tmpl % (uniq_id, morelinks))
         if len(revs) > 1:
             cs_links.append(compare_view)
         return ''.join(cs_links)
 
     def get_fork_name():
         repo_name = action_params
         url_ = url('summary_home', repo_name=repo_name)
         return _('Fork name %s') % link_to(action_params, url_)
 
     def get_user_name():
         user_name = action_params
         return user_name
 
     def get_users_group():
         group_name = action_params
         return group_name
 
     def get_pull_request():
         from kallithea.model.db import PullRequest
         pull_request_id = action_params
         nice_id = PullRequest.make_nice_id(pull_request_id)
 
         deleted = user_log.repository is None
         if deleted:
             repo_name = user_log.repository_name
         else:
             repo_name = user_log.repository.repo_name
 
         return link_to(_('Pull request %s') % nice_id,
                     url('pullrequest_show', repo_name=repo_name,
                     pull_request_id=pull_request_id))
 
     def get_archive_name():
         archive_name = action_params
         return archive_name
 
     # action : translated str, callback(extractor), icon
     action_map = {
         'user_deleted_repo':           (_('[deleted] repository'),
                                         None, 'icon-trashcan'),
         'user_created_repo':           (_('[created] repository'),
                                         None, 'icon-plus'),
         'user_created_fork':           (_('[created] repository as fork'),
                                         None, 'icon-fork'),
         'user_forked_repo':            (_('[forked] repository'),
                                         get_fork_name, 'icon-fork'),
         'user_updated_repo':           (_('[updated] repository'),
                                         None, 'icon-pencil'),
         'user_downloaded_archive':      (_('[downloaded] archive from repository'),
                                         get_archive_name, 'icon-download-cloud'),
         'admin_deleted_repo':          (_('[delete] repository'),
                                         None, 'icon-trashcan'),
         'admin_created_repo':          (_('[created] repository'),
                                         None, 'icon-plus'),
         'admin_forked_repo':           (_('[forked] repository'),
                                         None, 'icon-fork'),
         'admin_updated_repo':          (_('[updated] repository'),
                                         None, 'icon-pencil'),
         'admin_created_user':          (_('[created] user'),
                                         get_user_name, 'icon-user'),
         'admin_updated_user':          (_('[updated] user'),
                                         get_user_name, 'icon-user'),
         'admin_created_users_group':   (_('[created] user group'),
                                         get_users_group, 'icon-pencil'),
         'admin_updated_users_group':   (_('[updated] user group'),
                                         get_users_group, 'icon-pencil'),
         'user_commented_revision':     (_('[commented] on revision in repository'),
                                         get_cs_links, 'icon-comment'),
         'user_commented_pull_request': (_('[commented] on pull request for'),
                                         get_pull_request, 'icon-comment'),
         'user_closed_pull_request':    (_('[closed] pull request for'),
                                         get_pull_request, 'icon-ok'),
         'push':                        (_('[pushed] into'),
                                         get_cs_links, 'icon-move-up'),
         'push_local':                  (_('[committed via Kallithea] into repository'),
                                         get_cs_links, 'icon-pencil'),
         'push_remote':                 (_('[pulled from remote] into repository'),
                                         get_cs_links, 'icon-move-up'),
         'pull':                        (_('[pulled] from'),
                                         None, 'icon-move-down'),
         'started_following_repo':      (_('[started following] repository'),
                                         None, 'icon-heart'),
         'stopped_following_repo':      (_('[stopped following] repository'),
                                         None, 'icon-heart-empty'),
     }
 
     action_str = action_map.get(action, action)
     if feed:
         action = action_str[0].replace('[', '').replace(']', '')
     else:
         action = action_str[0] \
             .replace('[', '<b>') \
             .replace(']', '</b>')
 
     action_params_func = lambda: ""
 
     if callable(action_str[1]):
         action_params_func = action_str[1]
 
     def action_parser_icon():
         action = user_log.action
         action_params = None
         x = action.split(':')
 
         if len(x) > 1:
             action, action_params = x
 
         ico = action_map.get(action, ['', '', ''])[2]
         html = """<i class="%s"></i>""" % ico
         return literal(html)
 
     # returned callbacks we need to call to get
     return [lambda: literal(action), action_params_func, action_parser_icon]
 
 
 #==============================================================================
 # GRAVATAR URL
 #==============================================================================
 def gravatar_div(email_address, cls='', size=30, **div_attributes):
     """Return an html literal with a span around a gravatar if they are enabled.
     Extra keyword parameters starting with 'div_' will get the prefix removed
     and '_' changed to '-' and be used as attributes on the div. The default
     class is 'gravatar'.
     """
     from tg import tmpl_context as c
     if not c.visual.use_gravatar:
         return ''
     if 'div_class' not in div_attributes:
         div_attributes['div_class'] = "gravatar"
     attributes = []
     for k, v in sorted(div_attributes.items()):
         assert k.startswith('div_'), k
         attributes.append(' %s="%s"' % (k[4:].replace('_', '-'), escape(v)))
     return literal("""<span%s>%s</span>""" %
                    (''.join(attributes),
                     gravatar(email_address, cls=cls, size=size)))
 
 
 def gravatar(email_address, cls='', size=30):
     """return html element of the gravatar
 
     This method will return an <img> with the resolution double the size (for
     retina screens) of the image. If the url returned from gravatar_url is
     empty then we fallback to using an icon.
 
     """
     from tg import tmpl_context as c
     if not c.visual.use_gravatar:
         return ''
 
     src = gravatar_url(email_address, size * 2)
 
     if src:
         # here it makes sense to use style="width: ..." (instead of, say, a
         # stylesheet) because we using this to generate a high-res (retina) size
         html = ('<i class="icon-gravatar {cls}"'
                 ' style="font-size: {size}px;background-size: {size}px;background-image: url(\'{src}\')"'
                 '></i>').format(cls=cls, size=size, src=src)
 
     else:
         # if src is empty then there was no gravatar, so we use a font icon
         html = ("""<i class="icon-user {cls}" style="font-size: {size}px;"></i>"""
             .format(cls=cls, size=size, src=src))
 
     return literal(html)
 
 
 def gravatar_url(email_address, size=30, default=''):
     # doh, we need to re-import those to mock it later
     from kallithea.config.routing import url
     from kallithea.model.db import User
     from tg import tmpl_context as c
     if not c.visual.use_gravatar:
         return ""
 
     _def = '[e-mail]'  # default gravatar
     email_address = email_address or _def
 
     if email_address == _def:
         return default
 
     parsed_url = urlparse.urlparse(url.current(qualified=True))
     url = (c.visual.gravatar_url or User.DEFAULT_GRAVATAR_URL) \
                .replace('{email}', email_address) \
                .replace('{md5email}', hashlib.md5(safe_str(email_address).lower()).hexdigest()) \
                .replace('{netloc}', parsed_url.netloc) \
                .replace('{scheme}', parsed_url.scheme) \
                .replace('{size}', str(size))
     return url
 
 
 def changed_tooltip(nodes):
     """
     Generates a html string for changed nodes in changeset page.
     It limits the output to 30 entries
 
     :param nodes: LazyNodesGenerator
     """
     if nodes:
         pref = ': <br/> '
         suf = ''
         if len(nodes) > 30:
             suf = '<br/>' + _(' and %s more') % (len(nodes) - 30)
         return literal(pref + '<br/> '.join([safe_unicode(x.path)
                                              for x in nodes[:30]]) + suf)
     else:
         return ': ' + _('No files')
 
 
 def fancy_file_stats(stats):
     """
     Displays a fancy two colored bar for number of added/deleted
     lines of code on file
 
     :param stats: two element list of added/deleted lines of code
     """
     from kallithea.lib.diffs import NEW_FILENODE, DEL_FILENODE, \
         MOD_FILENODE, RENAMED_FILENODE, CHMOD_FILENODE, BIN_FILENODE
 
     a, d = stats['added'], stats['deleted']
     width = 100
 
     if stats['binary']:
         # binary mode
         lbl = ''
         bin_op = 1
 
         if BIN_FILENODE in stats['ops']:
             lbl = 'bin+'
 
         if NEW_FILENODE in stats['ops']:
             lbl += _('new file')
             bin_op = NEW_FILENODE
         elif MOD_FILENODE in stats['ops']:
             lbl += _('mod')
             bin_op = MOD_FILENODE
         elif DEL_FILENODE in stats['ops']:
             lbl += _('del')
             bin_op = DEL_FILENODE
         elif RENAMED_FILENODE in stats['ops']:
             lbl += _('rename')
             bin_op = RENAMED_FILENODE
 
         # chmod can go with other operations
         if CHMOD_FILENODE in stats['ops']:
             _org_lbl = _('chmod')
             lbl += _org_lbl if lbl.endswith('+') else '+%s' % _org_lbl
 
         #import ipdb;ipdb.set_trace()
         b_d = '<div class="bin bin%s progress-bar" style="width:100%%">%s</div>' % (bin_op, lbl)
         b_a = '<div class="bin bin1" style="width:0%"></div>'
         return literal('<div style="width:%spx" class="progress">%s%s</div>' % (width, b_a, b_d))
 
     t = stats['added'] + stats['deleted']
     unit = float(width) / (t or 1)
 
     # needs > 9% of width to be visible or 0 to be hidden
     a_p = max(9, unit * a) if a > 0 else 0
     d_p = max(9, unit * d) if d > 0 else 0
     p_sum = a_p + d_p
 
     if p_sum > width:
         # adjust the percentage to be == 100% since we adjusted to 9
         if a_p > d_p:
             a_p = a_p - (p_sum - width)
         else:
             d_p = d_p - (p_sum - width)
 
     a_v = a if a > 0 else ''
     d_v = d if d > 0 else ''
 
     d_a = '<div class="added progress-bar" style="width:%s%%">%s</div>' % (
         a_p, a_v
     )
     d_d = '<div class="deleted progress-bar" style="width:%s%%">%s</div>' % (
         d_p, d_v
     )
     return literal('<div class="progress" style="width:%spx">%s%s</div>' % (width, d_a, d_d))
 
 
 _URLIFY_RE = re.compile(r'''
 # URL markup
 (?P<url>%s) |
 # @mention markup
 (?P<mention>%s) |
 # Changeset hash markup
 (?<!\w|[-_])
   (?P<hash>[0-9a-f]{12,40})
 (?!\w|[-_]) |
 # Markup of *bold text*
 (?:
   (?:^|(?<=\s))
   (?P<bold> [*] (?!\s) [^*\n]* (?<!\s) [*] )
   (?![*\w])
 ) |
 # "Stylize" markup
 \[see\ \=&gt;\ *(?P<seen>[a-zA-Z0-9\/\=\?\&\ \:\/\.\-]*)\] |
 \[license\ \=&gt;\ *(?P<license>[a-zA-Z0-9\/\=\?\&\ \:\/\.\-]*)\] |
 \[(?P<tagtype>requires|recommends|conflicts|base)\ \=&gt;\ *(?P<tagvalue>[a-zA-Z0-9\-\/]*)\] |
 \[(?:lang|language)\ \=&gt;\ *(?P<lang>[a-zA-Z\-\/\#\+]*)\] |
 \[(?P<tag>[a-z]+)\]
 ''' % (url_re.pattern, MENTIONS_REGEX.pattern),
     re.VERBOSE | re.MULTILINE | re.IGNORECASE)
 
 
 def urlify_text(s, repo_name=None, link_=None, truncate=None, stylize=False, truncatef=truncate):
     """
     Parses given text message and make literal html with markup.
     The text will be truncated to the specified length.
     Hashes are turned into changeset links to specified repository.
     URLs links to what they say.
     Issues are linked to given issue-server.
     If link_ is provided, all text not already linking somewhere will link there.
     """
 
     def _replace(match_obj):
         url = match_obj.group('url')
         if url is not None:
             return '<a href="%(url)s">%(url)s</a>' % {'url': url}
         mention = match_obj.group('mention')
         if mention is not None:
             return '<b>%s</b>' % mention
         hash_ = match_obj.group('hash')
         if hash_ is not None and repo_name is not None:
             from kallithea.config.routing import url  # doh, we need to re-import url to mock it later
             return '<a class="changeset_hash" href="%(url)s">%(hash)s</a>' % {
                  'url': url('changeset_home', repo_name=repo_name, revision=hash_),
                  'hash': hash_,
                 }
         bold = match_obj.group('bold')
         if bold is not None:
             return '<b>*%s*</b>' % _urlify(bold[1:-1])
         if stylize:
             seen = match_obj.group('seen')
             if seen:
                 return '<div class="label label-meta" data-tag="see">see =&gt; %s</div>' % seen
             license = match_obj.group('license')
             if license:
                 return '<div class="label label-meta" data-tag="license"><a href="http://www.opensource.org/licenses/%s">%s</a></div>' % (license, license)
             tagtype = match_obj.group('tagtype')
             if tagtype:
                 tagvalue = match_obj.group('tagvalue')
                 return '<div class="label label-meta" data-tag="%s">%s =&gt; <a href="/%s">%s</a></div>' % (tagtype, tagtype, tagvalue, tagvalue)
             lang = match_obj.group('lang')
             if lang:
                 return '<div class="label label-meta" data-tag="lang">%s</div>' % lang
             tag = match_obj.group('tag')
             if tag:
                 return '<div class="label label-meta" data-tag="%s">%s</div>' % (tag, tag)
         return match_obj.group(0)
 
     def _urlify(s):
         """
         Extract urls from text and make html links out of them
         """
         return _URLIFY_RE.sub(_replace, s)
 
     if truncate is None:
         s = s.rstrip()
     else:
         s = truncatef(s, truncate, whole_word=True)
     s = html_escape(s)
     s = _urlify(s)
     if repo_name is not None:
         s = urlify_issues(s, repo_name)
     if link_ is not None:
         # make href around everything that isn't a href already
         s = linkify_others(s, link_)
     s = s.replace('\r\n', '<br/>').replace('\n', '<br/>')
     # Turn HTML5 into more valid HTML4 as required by some mail readers.
     # (This is not done in one step in html_escape, because character codes like
     # &#123; risk to be seen as an issue reference due to the presence of '#'.)
     s = s.replace("&apos;", "&#39;")
     return literal(s)
 
 
 def linkify_others(t, l):
     """Add a default link to html with links.
     HTML doesn't allow nesting of links, so the outer link must be broken up
     in pieces and give space for other links.
     """
     urls = re.compile(r'(\<a.*?\<\/a\>)',)
     links = []
     for e in urls.split(t):
         if e.strip() and not urls.match(e):
             links.append('<a class="message-link" href="%s">%s</a>' % (l, e))
         else:
             links.append(e)
 
     return ''.join(links)
 
 
 # Global variable that will hold the actual urlify_issues function body.
 # Will be set on first use when the global configuration has been read.
 _urlify_issues_f = None
 
 
 def urlify_issues(newtext, repo_name):
     """Urlify issue references according to .ini configuration"""
     global _urlify_issues_f
     if _urlify_issues_f is None:
         from kallithea import CONFIG
         from kallithea.model.db import URL_SEP
         assert CONFIG['sqlalchemy.url'] # make sure config has been loaded
 
         # Build chain of urlify functions, starting with not doing any transformation
         tmp_urlify_issues_f = lambda s: s
 
         issue_pat_re = re.compile(r'issue_pat(.*)')
         for k in CONFIG:
             # Find all issue_pat* settings that also have corresponding server_link and prefix configuration
             m = issue_pat_re.match(k)
             if m is None:
                 continue
             suffix = m.group(1)
             issue_pat = CONFIG.get(k)
             issue_server_link = CONFIG.get('issue_server_link%s' % suffix)
             issue_sub = CONFIG.get('issue_sub%s' % suffix)
             if not issue_pat or not issue_server_link or issue_sub is None: # issue_sub can be empty but should be present
                 log.error('skipping incomplete issue pattern %r: %r -> %r %r', suffix, issue_pat, issue_server_link, issue_sub)
                 continue
 
             # Wrap tmp_urlify_issues_f with substitution of this pattern, while making sure all loop variables (and compiled regexpes) are bound
             try:
                 issue_re = re.compile(issue_pat)
             except re.error as e:
                 log.error('skipping invalid issue pattern %r: %r -> %r %r. Error: %s', suffix, issue_pat, issue_server_link, issue_sub, str(e))
                 continue
 
             log.debug('issue pattern %r: %r -> %r %r', suffix, issue_pat, issue_server_link, issue_sub)
 
             def issues_replace(match_obj,
                                issue_server_link=issue_server_link, issue_sub=issue_sub):
                 try:
                     issue_url = match_obj.expand(issue_server_link)
                 except (IndexError, re.error) as e:
                     log.error('invalid issue_url setting %r -> %r %r. Error: %s', issue_pat, issue_server_link, issue_sub, str(e))
                     issue_url = issue_server_link
                 issue_url = issue_url.replace('{repo}', repo_name)
                 issue_url = issue_url.replace('{repo_name}', repo_name.split(URL_SEP)[-1])
                 # if issue_sub is empty use the matched issue reference verbatim
                 if not issue_sub:
                     issue_text = match_obj.group()
                 else:
                     try:
                         issue_text = match_obj.expand(issue_sub)
                     except (IndexError, re.error) as e:
                         log.error('invalid issue_sub setting %r -> %r %r. Error: %s', issue_pat, issue_server_link, issue_sub, str(e))
                         issue_text = match_obj.group()
 
                 return (
                     '<a class="issue-tracker-link" href="%(url)s">'
                     '%(text)s'
                     '</a>'
                     ) % {
                      'url': issue_url,
                      'text': issue_text,
                     }
             tmp_urlify_issues_f = (lambda s,
                                           issue_re=issue_re, issues_replace=issues_replace, chain_f=tmp_urlify_issues_f:
                                    issue_re.sub(issues_replace, chain_f(s)))
 
         # Set tmp function globally - atomically
         _urlify_issues_f = tmp_urlify_issues_f
 
     return _urlify_issues_f(newtext)
 
 
 def render_w_mentions(source, repo_name=None):
     """
     Render plain text with revision hashes and issue references urlified
     and with @mention highlighting.
     """
     s = safe_unicode(source)
     s = urlify_text(s, repo_name=repo_name)
     return literal('<div class="formatted-fixed">%s</div>' % s)
 
 
 def short_ref(ref_type, ref_name):
     if ref_type == 'rev':
         return short_id(ref_name)
     return ref_name
 
 
 def link_to_ref(repo_name, ref_type, ref_name, rev=None):
     """
     Return full markup for a href to changeset_home for a changeset.
     If ref_type is branch it will link to changelog.
     ref_name is shortened if ref_type is 'rev'.
     if rev is specified show it too, explicitly linking to that revision.
     """
     txt = short_ref(ref_type, ref_name)
     if ref_type == 'branch':
         u = url('changelog_home', repo_name=repo_name, branch=ref_name)
     else:
         u = url('changeset_home', repo_name=repo_name, revision=ref_name)
     l = link_to(repo_name + '#' + txt, u)
     if rev and ref_type != 'rev':
         l = literal('%s (%s)' % (l, link_to(short_id(rev), url('changeset_home', repo_name=repo_name, revision=rev))))
     return l
 
 
 def changeset_status(repo, revision):
     from kallithea.model.changeset_status import ChangesetStatusModel
     return ChangesetStatusModel().get_status(repo, revision)
 
 
 def changeset_status_lbl(changeset_status):
     from kallithea.model.db import ChangesetStatus
     return ChangesetStatus.get_status_lbl(changeset_status)
 
 
 def get_permission_name(key):
     from kallithea.model.db import Permission
     return dict(Permission.PERMS).get(key)
 
 
 def journal_filter_help():
     return _(textwrap.dedent('''
         Example filter terms:
             repository:vcs
             username:developer
             action:*push*
             ip:127.0.0.1
             date:20120101
             date:[20120101100000 TO 20120102]
 
         Generate wildcards using '*' character:
             "repository:vcs*" - search everything starting with 'vcs'
             "repository:*vcs*" - search for repository containing 'vcs'
 
         Optional AND / OR operators in queries
             "repository:vcs OR repository:test"
             "username:test AND repository:test*"
     '''))
 
 
 def not_mapped_error(repo_name):
     flash(_('%s repository is not mapped to db perhaps'
             ' it was created or renamed from the filesystem'
             ' please run the application again'
             ' in order to rescan repositories') % repo_name, category='error')
 
 
 def ip_range(ip_addr):
     from kallithea.model.db import UserIpMap
     s, e = UserIpMap._get_ip_range(ip_addr)
     return '%s - %s' % (s, e)
 
 
 session_csrf_secret_name = "_session_csrf_secret_token"
 
 def session_csrf_secret_token():
     """Return (and create) the current session's CSRF protection token."""
     from tg import session
     if not session_csrf_secret_name in session:
         session[session_csrf_secret_name] = str(random.getrandbits(128))
         session.save()
     return session[session_csrf_secret_name]
 
 def form(url, method="post", **attrs):
     """Like webhelpers.html.tags.form , but automatically adding
     session_csrf_secret_token for POST. The secret is thus never leaked in GET
     URLs.
     """
     form = insecure_form(url, method, **attrs)
     if method.lower() == 'get':
         return form
     return form + HTML.div(hidden(session_csrf_secret_name, session_csrf_secret_token()), style="display: none;")

kallithea/lib/indexers/daemon.py

@@ -1,459 +1,460 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.indexers.daemon
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 A daemon will read from task table and run tasks
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Jan 26, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 
 import logging
 import os
 import sys
 import traceback
 from os.path import dirname
 from shutil import rmtree
 from time import mktime
 
 from whoosh.index import create_in, exists_in, open_dir
 from whoosh.qparser import QueryParser
 
 from kallithea.config.conf import INDEX_EXTENSIONS, INDEX_FILENAMES
 from kallithea.lib.indexers import CHGSET_IDX_NAME, CHGSETS_SCHEMA, IDX_NAME, SCHEMA
 from kallithea.lib.utils2 import safe_str, safe_unicode
 from kallithea.lib.vcs.exceptions import ChangesetError, NodeDoesNotExistError, RepositoryError
 from kallithea.model.db import Repository
 from kallithea.model.scm import ScmModel
 
 
 # Add location of top level folder to sys.path
 project_path = dirname(dirname(dirname(dirname(os.path.realpath(__file__)))))
 sys.path.append(project_path)
 
 
 
 
 log = logging.getLogger('whoosh_indexer')
 
 
 class WhooshIndexingDaemon(object):
     """
     Daemon for atomic indexing jobs
     """
 
     def __init__(self, indexname=IDX_NAME, index_location=None,
                  repo_location=None, repo_list=None,
                  repo_update_list=None):
         self.indexname = indexname
 
         self.index_location = index_location
         if not index_location:
             raise Exception('You have to provide index location')
 
         self.repo_location = repo_location
         if not repo_location:
             raise Exception('You have to provide repositories location')
 
         self.repo_paths = ScmModel().repo_scan(self.repo_location)
 
         # filter repo list
         if repo_list:
             # Fix non-ascii repo names to unicode
             repo_list = set(safe_unicode(repo_name) for repo_name in repo_list)
             self.filtered_repo_paths = {}
             for repo_name, repo in self.repo_paths.items():
                 if repo_name in repo_list:
                     self.filtered_repo_paths[repo_name] = repo
 
             self.repo_paths = self.filtered_repo_paths
 
         # filter update repo list
         self.filtered_repo_update_paths = {}
         if repo_update_list:
             self.filtered_repo_update_paths = {}
             for repo_name, repo in self.repo_paths.items():
                 if repo_name in repo_update_list:
                     self.filtered_repo_update_paths[repo_name] = repo
             self.repo_paths = self.filtered_repo_update_paths
 
         self.initial = True
         if not os.path.isdir(self.index_location):
             os.makedirs(self.index_location)
             log.info('Cannot run incremental index since it does not '
                      'yet exist - running full build')
         elif not exists_in(self.index_location, IDX_NAME):
             log.info('Running full index build, as the file content '
                      'index does not exist')
         elif not exists_in(self.index_location, CHGSET_IDX_NAME):
             log.info('Running full index build, as the changeset '
                      'index does not exist')
         else:
             self.initial = False
 
     def _get_index_revision(self, repo):
         db_repo = Repository.get_by_repo_name(safe_unicode(repo.name))
         landing_rev = 'tip'
         if db_repo:
             _rev_type, _rev = db_repo.landing_rev
             landing_rev = _rev
         return landing_rev
 
     def _get_index_changeset(self, repo, index_rev=None):
         if not index_rev:
             index_rev = self._get_index_revision(repo)
         cs = repo.get_changeset(index_rev)
         return cs
 
     def get_paths(self, repo):
         """
         recursive walk in root dir and return a set of all path in that dir
         based on repository walk function
         """
         index_paths_ = set()
         try:
             cs = self._get_index_changeset(repo)
             for _topnode, _dirs, files in cs.walk('/'):
                 for f in files:
                     index_paths_.add(os.path.join(safe_str(repo.path), safe_str(f.path)))
 
         except RepositoryError:
             log.debug(traceback.format_exc())
             pass
         return index_paths_
 
     def get_node(self, repo, path, index_rev=None):
         """
         gets a filenode based on given full path. It operates on string for
         hg git compatibility.
 
         :param repo: scm repo instance
         :param path: full path including root location
         :return: FileNode
         """
         # FIXME: paths should be normalized ... or even better: don't include repo.path
         path = safe_str(path)
         repo_path = safe_str(repo.path)
         assert path.startswith(repo_path)
         assert path[len(repo_path)] in (os.path.sep, os.path.altsep)
         node_path = path[len(repo_path) + 1:]
         cs = self._get_index_changeset(repo, index_rev=index_rev)
         node = cs.get_node(node_path)
         return node
 
     def is_indexable_node(self, node):
         """
         Just index the content of chosen files, skipping binary files
         """
         return (node.extension in INDEX_EXTENSIONS or node.name in INDEX_FILENAMES) and \
                not node.is_binary
 
     def get_node_mtime(self, node):
         return mktime(node.last_changeset.date.timetuple())
 
     def add_doc(self, writer, path, repo, repo_name, index_rev=None):
         """
         Adding doc to writer this function itself fetches data from
         the instance of vcs backend
         """
         try:
             node = self.get_node(repo, path, index_rev)
         except (ChangesetError, NodeDoesNotExistError):
             log.debug("    >> %s - not found in %s %s", path, repo, index_rev)
             return 0, 0
 
         indexed = indexed_w_content = 0
         if self.is_indexable_node(node):
-            u_content = node.content
-            if not isinstance(u_content, unicode):
+            bytes_content = node.content
+            if b'\0' in bytes_content:
                 log.warning('    >> %s - no text content', path)
                 u_content = u''
             else:
                 log.debug('    >> %s', path)
+                u_content = safe_unicode(bytes_content)
                 indexed_w_content += 1
 
         else:
             log.debug('    >> %s - not indexable', path)
             # just index file name without it's content
             u_content = u''
             indexed += 1
 
         p = safe_unicode(path)
         writer.add_document(
             fileid=p,
             owner=unicode(repo.contact),
             repository_rawname=safe_unicode(repo_name),
             repository=safe_unicode(repo_name),
             path=p,
             content=u_content,
             modtime=self.get_node_mtime(node),
             extension=node.extension
         )
         return indexed, indexed_w_content
 
     def index_changesets(self, writer, repo_name, repo, start_rev=None):
         """
         Add all changeset in the vcs repo starting at start_rev
         to the index writer
 
         :param writer: the whoosh index writer to add to
         :param repo_name: name of the repository from whence the
           changeset originates including the repository group
         :param repo: the vcs repository instance to index changesets for,
           the presumption is the repo has changesets to index
         :param start_rev=None: the full sha id to start indexing from
           if start_rev is None then index from the first changeset in
           the repo
         """
 
         if start_rev is None:
             start_rev = repo[0].raw_id
 
         log.debug('Indexing changesets in %s, starting at rev %s',
                   repo_name, start_rev)
 
         indexed = 0
         cs_iter = repo.get_changesets(start=start_rev)
         total = len(cs_iter)
         for cs in cs_iter:
             indexed += 1
             log.debug('    >> %s %s/%s', cs, indexed, total)
             writer.add_document(
                 raw_id=unicode(cs.raw_id),
                 owner=unicode(repo.contact),
                 date=cs._timestamp,
                 repository_rawname=safe_unicode(repo_name),
                 repository=safe_unicode(repo_name),
                 author=cs.author,
                 message=cs.message,
                 last=cs.last,
                 added=u' '.join([safe_unicode(node.path) for node in cs.added]).lower(),
                 removed=u' '.join([safe_unicode(node.path) for node in cs.removed]).lower(),
                 changed=u' '.join([safe_unicode(node.path) for node in cs.changed]).lower(),
                 parents=u' '.join([cs.raw_id for cs in cs.parents]),
             )
 
         return indexed
 
     def index_files(self, file_idx_writer, repo_name, repo):
         """
         Index files for given repo_name
 
         :param file_idx_writer: the whoosh index writer to add to
         :param repo_name: name of the repository we're indexing
         :param repo: instance of vcs repo
         """
         i_cnt = iwc_cnt = 0
         log.debug('Building file index for %s @revision:%s', repo_name,
                                                 self._get_index_revision(repo))
         index_rev = self._get_index_revision(repo)
         for idx_path in self.get_paths(repo):
             i, iwc = self.add_doc(file_idx_writer, idx_path, repo, repo_name, index_rev)
             i_cnt += i
             iwc_cnt += iwc
 
         log.debug('added %s files %s with content for repo %s',
                   i_cnt + iwc_cnt, iwc_cnt, repo.path)
         return i_cnt, iwc_cnt
 
     def update_changeset_index(self):
         idx = open_dir(self.index_location, indexname=CHGSET_IDX_NAME)
 
         with idx.searcher() as searcher:
             writer = idx.writer()
             writer_is_dirty = False
             try:
                 indexed_total = 0
                 repo_name = None
                 for repo_name, repo in sorted(self.repo_paths.items()):
                     log.debug('Updating changeset index for repo %s', repo_name)
                     # skip indexing if there aren't any revs in the repo
                     num_of_revs = len(repo)
                     if num_of_revs < 1:
                         continue
 
                     qp = QueryParser('repository', schema=CHGSETS_SCHEMA)
                     q = qp.parse(u"last:t AND %s" % repo_name)
 
                     results = searcher.search(q)
 
                     # default to scanning the entire repo
                     last_rev = 0
                     start_id = None
 
                     if len(results) > 0:
                         # assuming that there is only one result, if not this
                         # may require a full re-index.
                         start_id = results[0]['raw_id']
                         last_rev = repo.get_changeset(revision=start_id).revision
 
                     # there are new changesets to index or a new repo to index
                     if last_rev == 0 or num_of_revs > last_rev + 1:
                         # delete the docs in the index for the previous
                         # last changeset(s)
                         for hit in results:
                             q = qp.parse(u"last:t AND %s AND raw_id:%s" %
                                             (repo_name, hit['raw_id']))
                             writer.delete_by_query(q)
 
                         # index from the previous last changeset + all new ones
                         indexed_total += self.index_changesets(writer,
                                                 repo_name, repo, start_id)
                         writer_is_dirty = True
                 log.debug('indexed %s changesets for repo %s',
                           indexed_total, repo_name
                 )
             finally:
                 if writer_is_dirty:
                     log.debug('>> COMMITING CHANGES TO CHANGESET INDEX<<')
                     writer.commit(merge=True)
                     log.debug('>>> FINISHED REBUILDING CHANGESET INDEX <<<')
                 else:
                     log.debug('>> NOTHING TO COMMIT TO CHANGESET INDEX<<')
 
     def update_file_index(self):
         log.debug((u'STARTING INCREMENTAL INDEXING UPDATE FOR EXTENSIONS %s '
                    'AND REPOS %s') % (INDEX_EXTENSIONS, self.repo_paths.keys()))
 
         idx = open_dir(self.index_location, indexname=self.indexname)
         # The set of all paths in the index
         indexed_paths = set()
         # The set of all paths we need to re-index
         to_index = set()
 
         writer = idx.writer()
         writer_is_dirty = False
         try:
             with idx.reader() as reader:
 
                 # Loop over the stored fields in the index
                 for fields in reader.all_stored_fields():
                     indexed_path = fields['path']
                     indexed_repo_path = fields['repository']
                     indexed_paths.add(indexed_path)
 
                     if indexed_repo_path not in self.filtered_repo_update_paths:
                         continue
 
                     repo = self.repo_paths[indexed_repo_path]
 
                     try:
                         node = self.get_node(repo, indexed_path)
                         # Check if this file was changed since it was indexed
                         indexed_time = fields['modtime']
                         mtime = self.get_node_mtime(node)
                         if mtime > indexed_time:
                             # The file has changed, delete it and add it to
                             # the list of files to reindex
                             log.debug(
                                 'adding to reindex list %s mtime: %s vs %s',
                                     indexed_path, mtime, indexed_time
                             )
                             writer.delete_by_term('fileid', indexed_path)
                             writer_is_dirty = True
 
                             to_index.add(indexed_path)
                     except (ChangesetError, NodeDoesNotExistError):
                         # This file was deleted since it was indexed
                         log.debug('removing from index %s', indexed_path)
                         writer.delete_by_term('path', indexed_path)
                         writer_is_dirty = True
 
             # Loop over the files in the filesystem
             # Assume we have a function that gathers the filenames of the
             # documents to be indexed
             ri_cnt_total = 0  # indexed
             riwc_cnt_total = 0  # indexed with content
             for repo_name, repo in sorted(self.repo_paths.items()):
                 log.debug('Updating file index for repo %s', repo_name)
                 # skip indexing if there aren't any revisions
                 if len(repo) < 1:
                     continue
                 ri_cnt = 0   # indexed
                 riwc_cnt = 0  # indexed with content
                 for path in self.get_paths(repo):
                     path = safe_unicode(path)
                     if path in to_index or path not in indexed_paths:
 
                         # This is either a file that's changed, or a new file
                         # that wasn't indexed before. So index it!
                         i, iwc = self.add_doc(writer, path, repo, repo_name)
                         writer_is_dirty = True
                         ri_cnt += i
                         ri_cnt_total += 1
                         riwc_cnt += iwc
                         riwc_cnt_total += iwc
                 log.debug('added %s files %s with content for repo %s',
                              ri_cnt + riwc_cnt, riwc_cnt, repo.path
                 )
             log.debug('indexed %s files in total and %s with content',
                         ri_cnt_total, riwc_cnt_total
             )
         finally:
             if writer_is_dirty:
                 log.debug('>> COMMITING CHANGES TO FILE INDEX <<')
                 writer.commit(merge=True)
                 log.debug('>>> FINISHED REBUILDING FILE INDEX <<<')
             else:
                 log.debug('>> NOTHING TO COMMIT TO FILE INDEX <<')
                 writer.cancel()
 
     def build_indexes(self):
         if os.path.exists(self.index_location):
             log.debug('removing previous index')
             rmtree(self.index_location)
 
         if not os.path.exists(self.index_location):
             os.mkdir(self.index_location)
 
         chgset_idx = create_in(self.index_location, CHGSETS_SCHEMA,
                                indexname=CHGSET_IDX_NAME)
         chgset_idx_writer = chgset_idx.writer()
 
         file_idx = create_in(self.index_location, SCHEMA, indexname=IDX_NAME)
         file_idx_writer = file_idx.writer()
         log.debug('BUILDING INDEX FOR EXTENSIONS %s '
                   'AND REPOS %s' % (INDEX_EXTENSIONS, self.repo_paths.keys()))
 
         for repo_name, repo in sorted(self.repo_paths.items()):
             log.debug('Updating indices for repo %s', repo_name)
             # skip indexing if there aren't any revisions
             if len(repo) < 1:
                 continue
 
             self.index_files(file_idx_writer, repo_name, repo)
             self.index_changesets(chgset_idx_writer, repo_name, repo)
 
         log.debug('>> COMMITING CHANGES <<')
         file_idx_writer.commit(merge=True)
         chgset_idx_writer.commit(merge=True)
         log.debug('>>> FINISHED BUILDING INDEX <<<')
 
     def update_indexes(self):
         self.update_file_index()
         self.update_changeset_index()
 
     def run(self, full_index=False):
         """Run daemon"""
         if full_index or self.initial:
             self.build_indexes()
         else:
             self.update_indexes()

kallithea/lib/vcs/backends/git/inmemory.py

@@ -1,201 +1,197 b''
 import datetime
 import posixpath
 import stat
 import time
 
 from dulwich import objects
 
 from kallithea.lib.vcs.backends.base import BaseInMemoryChangeset
 from kallithea.lib.vcs.exceptions import RepositoryError
 from kallithea.lib.vcs.utils import safe_bytes, safe_str
 
 
 class GitInMemoryChangeset(BaseInMemoryChangeset):
 
     def commit(self, message, author, parents=None, branch=None, date=None,
                **kwargs):
         """
         Performs in-memory commit (doesn't check workdir in any way) and
         returns newly created ``Changeset``. Updates repository's
         ``revisions``.
 
         :param message: message of the commit
         :param author: full username, i.e. "Joe Doe <[e-mail]>"
         :param parents: single parent or sequence of parents from which commit
           would be derived
         :param date: ``datetime.datetime`` instance. Defaults to
           ``datetime.datetime.now()``.
         :param branch: branch name, as string. If none given, default backend's
           branch would be used.
 
         :raises ``CommitError``: if any error occurs while committing
         """
         self.check_integrity(parents)
 
         from .repository import GitRepository
         if branch is None:
             branch = GitRepository.DEFAULT_BRANCH_NAME
 
         repo = self.repository._repo
         object_store = repo.object_store
 
         ENCODING = "UTF-8"  # TODO: should probably be kept in sync with safe_unicode/safe_bytes and vcs/conf/settings.py DEFAULT_ENCODINGS
 
         # Create tree and populates it with blobs
         commit_tree = self.parents[0] and repo[self.parents[0]._commit.tree] or \
             objects.Tree()
         for node in self.added + self.changed:
             # Compute subdirs if needed
             dirpath, nodename = posixpath.split(node.path)
             dirnames = safe_str(dirpath).split('/') if dirpath else []
             parent = commit_tree
             ancestors = [('', parent)]
 
             # Tries to dig for the deepest existing tree
             while dirnames:
                 curdir = dirnames.pop(0)
                 try:
                     dir_id = parent[curdir][1]
                 except KeyError:
                     # put curdir back into dirnames and stops
                     dirnames.insert(0, curdir)
                     break
                 else:
                     # If found, updates parent
                     parent = self.repository._repo[dir_id]
                     ancestors.append((curdir, parent))
             # Now parent is deepest existing tree and we need to create subtrees
             # for dirnames (in reverse order) [this only applies for nodes from added]
             new_trees = []
 
-            if not node.is_binary:
-                content = node.content.encode(ENCODING)
-            else:
-                content = node.content
-            blob = objects.Blob.from_string(content)
+            blob = objects.Blob.from_string(node.content)
 
             node_path = safe_bytes(node.name)
             if dirnames:
                 # If there are trees which should be created we need to build
                 # them now (in reverse order)
                 reversed_dirnames = list(reversed(dirnames))
                 curtree = objects.Tree()
                 curtree[node_path] = node.mode, blob.id
                 new_trees.append(curtree)
                 for dirname in reversed_dirnames[:-1]:
                     newtree = objects.Tree()
                     #newtree.add(stat.S_IFDIR, dirname, curtree.id)
                     newtree[dirname] = stat.S_IFDIR, curtree.id
                     new_trees.append(newtree)
                     curtree = newtree
                 parent[reversed_dirnames[-1]] = stat.S_IFDIR, curtree.id
             else:
                 parent.add(name=node_path, mode=node.mode, hexsha=blob.id)
 
             new_trees.append(parent)
             # Update ancestors
             for parent, tree, path in reversed([(a[1], b[1], b[0]) for a, b in
                 zip(ancestors, ancestors[1:])]
             ):
                 parent[path] = stat.S_IFDIR, tree.id
                 object_store.add_object(tree)
 
             object_store.add_object(blob)
             for tree in new_trees:
                 object_store.add_object(tree)
         for node in self.removed:
             paths = node.path.split('/')
             tree = commit_tree
             trees = [tree]
             # Traverse deep into the forest...
             for path in paths:
                 try:
                     obj = self.repository._repo[tree[path][1]]
                     if isinstance(obj, objects.Tree):
                         trees.append(obj)
                         tree = obj
                 except KeyError:
                     break
             # Cut down the blob and all rotten trees on the way back...
             for path, tree in reversed(zip(paths, trees)):
                 del tree[path]
                 if tree:
                     # This tree still has elements - don't remove it or any
                     # of it's parents
                     break
 
         object_store.add_object(commit_tree)
 
         # Create commit
         commit = objects.Commit()
         commit.tree = commit_tree.id
         commit.parents = [p._commit.id for p in self.parents if p]
         commit.author = commit.committer = safe_str(author)
         commit.encoding = ENCODING
         commit.message = safe_str(message)
 
         # Compute date
         if date is None:
             date = time.time()
         elif isinstance(date, datetime.datetime):
             date = time.mktime(date.timetuple())
 
         author_time = kwargs.pop('author_time', date)
         commit.commit_time = int(date)
         commit.author_time = int(author_time)
         tz = time.timezone
         author_tz = kwargs.pop('author_timezone', tz)
         commit.commit_timezone = tz
         commit.author_timezone = author_tz
 
         object_store.add_object(commit)
 
         ref = 'refs/heads/%s' % branch
         repo.refs[ref] = commit.id
 
         # Update vcs repository object & recreate dulwich repo
         self.repository.revisions.append(commit.id)
         # invalidate parsed refs after commit
         self.repository._parsed_refs = self.repository._get_parsed_refs()
         tip = self.repository.get_changeset()
         self.reset()
         return tip
 
     def _get_missing_trees(self, path, root_tree):
         """
         Creates missing ``Tree`` objects for the given path.
 
         :param path: path given as a string. It may be a path to a file node
           (i.e. ``foo/bar/baz.txt``) or directory path - in that case it must
           end with slash (i.e. ``foo/bar/``).
         :param root_tree: ``dulwich.objects.Tree`` object from which we start
           traversing (should be commit's root tree)
         """
         dirpath = posixpath.split(path)[0]
         dirs = dirpath.split('/')
         if not dirs or dirs == ['']:
             return []
 
         def get_tree_for_dir(tree, dirname):
             for name, mode, id in tree.iteritems():
                 if name == dirname:
                     obj = self.repository._repo[id]
                     if isinstance(obj, objects.Tree):
                         return obj
                     else:
                         raise RepositoryError("Cannot create directory %s "
                             "at tree %s as path is occupied and is not a "
                             "Tree" % (dirname, tree))
             return None
 
         trees = []
         parent = root_tree
         for dirname in dirs:
             tree = get_tree_for_dir(parent, dirname)
             if tree is None:
                 tree = objects.Tree()
                 parent.add(stat.S_IFDIR, dirname, tree.id)
                 parent = tree
             # Always append tree
             trees.append(tree)
         return trees

kallithea/lib/vcs/backends/hg/inmemory.py

@@ -1,109 +1,107 b''
 import datetime
 
 from kallithea.lib.vcs.backends.base import BaseInMemoryChangeset
 from kallithea.lib.vcs.exceptions import RepositoryError
 from kallithea.lib.vcs.utils import safe_bytes
 from kallithea.lib.vcs.utils.hgcompat import hex, memctx, memfilectx
 
 
 class MercurialInMemoryChangeset(BaseInMemoryChangeset):
 
     def commit(self, message, author, parents=None, branch=None, date=None,
             **kwargs):
         """
         Performs in-memory commit (doesn't check workdir in any way) and
         returns newly created ``Changeset``. Updates repository's
         ``revisions``.
 
         :param message: message of the commit
         :param author: full username, i.e. "Joe Doe <[e-mail]>"
         :param parents: single parent or sequence of parents from which commit
           would be derived
         :param date: ``datetime.datetime`` instance. Defaults to
           ``datetime.datetime.now()``.
         :param branch: branch name, as string. If none given, default backend's
           branch would be used.
 
         :raises ``CommitError``: if any error occurs while committing
         """
         self.check_integrity(parents)
 
         from .repository import MercurialRepository
         if not isinstance(message, unicode) or not isinstance(author, unicode):
             raise RepositoryError('Given message and author needs to be '
                                   'an <unicode> instance got %r & %r instead'
                                   % (type(message), type(author)))
 
         if branch is None:
             branch = MercurialRepository.DEFAULT_BRANCH_NAME
         kwargs['branch'] = branch
 
         def filectxfn(_repo, memctx, path):
             """
             Marks given path as added/changed/removed in a given _repo. This is
             for internal mercurial commit function.
             """
 
             # check if this path is removed
             if path in (node.path for node in self.removed):
                 return None
 
             # check if this path is added
             for node in self.added:
                 if node.path == path:
                     return memfilectx(_repo, memctx, path=node.path,
-                        data=(node.content.encode('utf-8')
-                              if not node.is_binary else node.content),
+                        data=node.content,
                         islink=False,
                         isexec=node.is_executable,
                         copysource=False)
 
             # or changed
             for node in self.changed:
                 if node.path == path:
                     return memfilectx(_repo, memctx, path=node.path,
-                        data=(node.content.encode('utf-8')
-                              if not node.is_binary else node.content),
+                        data=node.content,
                         islink=False,
                         isexec=node.is_executable,
                         copysource=False)
 
             raise RepositoryError("Given path haven't been marked as added,"
                                   "changed or removed (%s)" % path)
 
         parents = [None, None]
         for i, parent in enumerate(self.parents):
             if parent is not None:
                 parents[i] = parent._ctx.node()
 
         if date and isinstance(date, datetime.datetime):
             date = date.strftime('%a, %d %b %Y %H:%M:%S')
 
         commit_ctx = memctx(repo=self.repository._repo,
             parents=parents,
             text='',
             files=self.get_paths(),
             filectxfn=filectxfn,
             user=author,
             date=date,
             extra=kwargs)
 
         # injecting given _repo params
         commit_ctx._text = safe_bytes(message)
         commit_ctx._user = safe_bytes(author)
         commit_ctx._date = date
 
         # TODO: Catch exceptions!
         n = self.repository._repo.commitctx(commit_ctx)
         # Returns mercurial node
         self._commit_ctx = commit_ctx  # For reference
         # Update vcs repository object & recreate mercurial _repo
         # new_ctx = self.repository._repo[node]
         # new_tip = self.repository.get_changeset(new_ctx.hex())
         new_id = hex(n)
         self.repository.revisions.append(new_id)
         self._repo = self.repository._get_repo(create=False)
         self.repository.branches = self.repository._get_branches()
         tip = self.repository.get_changeset()
         self.reset()
         return tip

kallithea/lib/vcs/nodes.py

@@ -1,622 +1,617 b''
 # -*- coding: utf-8 -*-
 """
     vcs.nodes
     ~~~~~~~~~
 
     Module holding everything related to vcs nodes.
 
     :created_on: Apr 8, 2010
     :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
 """
 
 import functools
 import mimetypes
 import posixpath
 import stat
 
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.lib.vcs.exceptions import NodeError, RemovedFileNodeError
-from kallithea.lib.vcs.utils import safe_str, safe_unicode
+from kallithea.lib.vcs.utils import safe_bytes, safe_str, safe_unicode
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 
 
 class NodeKind:
     SUBMODULE = -1
     DIR = 1
     FILE = 2
 
 
 class NodeState:
     ADDED = u'added'
     CHANGED = u'changed'
     NOT_CHANGED = u'not changed'
     REMOVED = u'removed'
 
 
 class NodeGeneratorBase(object):
     """
     Base class for removed added and changed filenodes, it's a lazy generator
     class that will create filenodes only on iteration or call
 
     The len method doesn't need to create filenodes at all
     """
 
     def __init__(self, current_paths, cs):
         self.cs = cs
         self.current_paths = current_paths
 
     def __call__(self):
         return [n for n in self]
 
     def __getitem__(self, key):
         assert isinstance(key, slice), key
         for p in self.current_paths[key]:
             yield self.cs.get_node(p)
 
     def __len__(self):
         return len(self.current_paths)
 
     def __iter__(self):
         for p in self.current_paths:
             yield self.cs.get_node(p)
 
 
 class AddedFileNodesGenerator(NodeGeneratorBase):
     """
     Class holding Added files for current changeset
     """
     pass
 
 
 class ChangedFileNodesGenerator(NodeGeneratorBase):
     """
     Class holding Changed files for current changeset
     """
     pass
 
 
 class RemovedFileNodesGenerator(NodeGeneratorBase):
     """
     Class holding removed files for current changeset
     """
     def __iter__(self):
         for p in self.current_paths:
             yield RemovedFileNode(path=p)
 
     def __getitem__(self, key):
         assert isinstance(key, slice), key
         for p in self.current_paths[key]:
             yield RemovedFileNode(path=p)
 
 
 @functools.total_ordering
 class Node(object):
     """
     Simplest class representing file or directory on repository.  SCM backends
     should use ``FileNode`` and ``DirNode`` subclasses rather than ``Node``
     directly.
 
     Node's ``path`` cannot start with slash as we operate on *relative* paths
     only. Moreover, every single node is identified by the ``path`` attribute,
     so it cannot end with slash, too. Otherwise, path could lead to mistakes.
     """
 
     def __init__(self, path, kind):
         if path.startswith('/'):
             raise NodeError("Cannot initialize Node objects with slash at "
                             "the beginning as only relative paths are supported")
         self.path = safe_str(path.rstrip('/'))  # we store paths as str
         if path == '' and kind != NodeKind.DIR:
             raise NodeError("Only DirNode and its subclasses may be "
                             "initialized with empty path")
         self.kind = kind
         #self.dirs, self.files = [], []
         if self.is_root() and not self.is_dir():
             raise NodeError("Root node cannot be FILE kind")
 
     @LazyProperty
     def parent(self):
         parent_path = self.get_parent_path()
         if parent_path:
             if self.changeset:
                 return self.changeset.get_node(parent_path)
             return DirNode(parent_path)
         return None
 
     @LazyProperty
     def name(self):
         """
         Returns name of the node so if its path
         then only last part is returned.
         """
         return safe_unicode(self.path.rstrip('/').split('/')[-1])
 
     def _get_kind(self):
         return self._kind
 
     def _set_kind(self, kind):
         if hasattr(self, '_kind'):
             raise NodeError("Cannot change node's kind")
         else:
             self._kind = kind
             # Post setter check (path's trailing slash)
             if self.path.endswith('/'):
                 raise NodeError("Node's path cannot end with slash")
 
     kind = property(_get_kind, _set_kind)
 
     def __eq__(self, other):
         if type(self) is not type(other):
             return False
         if self._kind != other._kind:
             return False
         if self.path != other.path:
             return False
         if self.is_file():
             return self.content == other.content
         else:
             # For DirNode's check without entering each dir
             self_nodes_paths = list(sorted(n.path for n in self.nodes))
             other_nodes_paths = list(sorted(n.path for n in self.nodes))
             return self_nodes_paths == other_nodes_paths
 
     def __lt__(self, other):
         if self._kind < other._kind:
             return True
         if self._kind > other._kind:
             return False
         if self.path < other.path:
             return True
         if self.path > other.path:
             return False
         if self.is_file():
             return self.content < other.content
         else:
             # For DirNode's check without entering each dir
             self_nodes_paths = list(sorted(n.path for n in self.nodes))
             other_nodes_paths = list(sorted(n.path for n in self.nodes))
             return self_nodes_paths < other_nodes_paths
 
     def __repr__(self):
         return '<%s %r>' % (self.__class__.__name__, self.path)
 
     def __str__(self):
         return self.__repr__()
 
     def __unicode__(self):
         return self.name
 
     def get_parent_path(self):
         """
         Returns node's parent path or empty string if node is root.
         """
         if self.is_root():
             return ''
         return posixpath.dirname(self.path.rstrip('/')) + '/'
 
     def is_file(self):
         """
         Returns ``True`` if node's kind is ``NodeKind.FILE``, ``False``
         otherwise.
         """
         return self.kind == NodeKind.FILE
 
     def is_dir(self):
         """
         Returns ``True`` if node's kind is ``NodeKind.DIR``, ``False``
         otherwise.
         """
         return self.kind == NodeKind.DIR
 
     def is_root(self):
         """
         Returns ``True`` if node is a root node and ``False`` otherwise.
         """
         return self.kind == NodeKind.DIR and self.path == ''
 
     def is_submodule(self):
         """
         Returns ``True`` if node's kind is ``NodeKind.SUBMODULE``, ``False``
         otherwise.
         """
         return self.kind == NodeKind.SUBMODULE
 
     @LazyProperty
     def added(self):
         return self.state is NodeState.ADDED
 
     @LazyProperty
     def changed(self):
         return self.state is NodeState.CHANGED
 
     @LazyProperty
     def not_changed(self):
         return self.state is NodeState.NOT_CHANGED
 
     @LazyProperty
     def removed(self):
         return self.state is NodeState.REMOVED
 
 
 class FileNode(Node):
     """
     Class representing file nodes.
 
     :attribute: path: path to the node, relative to repository's root
     :attribute: content: if given arbitrary sets content of the file
     :attribute: changeset: if given, first time content is accessed, callback
     :attribute: mode: octal stat mode for a node. Default is 0100644.
     """
 
     def __init__(self, path, content=None, changeset=None, mode=None):
         """
         Only one of ``content`` and ``changeset`` may be given. Passing both
         would raise ``NodeError`` exception.
 
         :param path: relative path to the node
         :param content: content may be passed to constructor
         :param changeset: if given, will use it to lazily fetch content
         :param mode: octal representation of ST_MODE (i.e. 0100644)
         """
 
         if content and changeset:
             raise NodeError("Cannot use both content and changeset")
         super(FileNode, self).__init__(path, kind=NodeKind.FILE)
         self.changeset = changeset
+        if not isinstance(content, bytes) and content is not None:
+            # File content is one thing that inherently must be bytes ... but
+            # VCS module tries to be "user friendly" and support unicode ...
+            content = safe_bytes(content)
         self._content = content
         self._mode = mode or 0o100644
 
     @LazyProperty
     def mode(self):
         """
         Returns lazily mode of the FileNode. If ``changeset`` is not set, would
         use value given at initialization or 0100644 (default).
         """
         if self.changeset:
             mode = self.changeset.get_file_mode(self.path)
         else:
             mode = self._mode
         return mode
 
-    def _get_content(self):
+    @property
+    def content(self):
+        """
+        Returns lazily byte content of the FileNode.
+        """
         if self.changeset:
             content = self.changeset.get_file_content(self.path)
         else:
             content = self._content
         return content
 
-    @property
-    def content(self):
-        """
-        Returns lazily content of the FileNode. If possible, would try to
-        decode content from UTF-8.
-        """
-        content = self._get_content()
-
-        if bool(content and '\0' in content):
-            return content
-        return safe_unicode(content)
-
     @LazyProperty
     def size(self):
         if self.changeset:
             return self.changeset.get_file_size(self.path)
         raise NodeError("Cannot retrieve size of the file without related "
             "changeset attribute")
 
     @LazyProperty
     def message(self):
         if self.changeset:
             return self.last_changeset.message
         raise NodeError("Cannot retrieve message of the file without related "
             "changeset attribute")
 
     @LazyProperty
     def last_changeset(self):
         if self.changeset:
             return self.changeset.get_file_changeset(self.path)
         raise NodeError("Cannot retrieve last changeset of the file without "
             "related changeset attribute")
 
     def get_mimetype(self):
         """
         Mimetype is calculated based on the file's content.
         """
 
         mtype, encoding = mimetypes.guess_type(self.name)
 
         if mtype is None:
             if self.is_binary:
                 mtype = 'application/octet-stream'
                 encoding = None
             else:
                 mtype = 'text/plain'
                 encoding = None
 
                 # try with pygments
                 try:
                     from pygments import lexers
                     mt = lexers.get_lexer_for_filename(self.name).mimetypes
                 except lexers.ClassNotFound:
                     mt = None
 
                 if mt:
                     mtype = mt[0]
 
         return mtype, encoding
 
     @LazyProperty
     def mimetype(self):
         """
         Wrapper around full mimetype info. It returns only type of fetched
         mimetype without the encoding part. use get_mimetype function to fetch
         full set of (type,encoding)
         """
         return self.get_mimetype()[0]
 
     @LazyProperty
     def mimetype_main(self):
         return self.mimetype.split('/')[0]
 
     @LazyProperty
     def lexer(self):
         """
         Returns pygment's lexer class. Would try to guess lexer taking file's
         content, name and mimetype.
         """
         from pygments import lexers
         try:
-            lexer = lexers.guess_lexer_for_filename(self.name, self.content, stripnl=False)
+            lexer = lexers.guess_lexer_for_filename(self.name, safe_unicode(self.content), stripnl=False)
         except lexers.ClassNotFound:
             lexer = lexers.TextLexer(stripnl=False)
         # returns first alias
         return lexer
 
     @LazyProperty
     def lexer_alias(self):
         """
         Returns first alias of the lexer guessed for this file.
         """
         return self.lexer.aliases[0]
 
     @LazyProperty
     def history(self):
         """
         Returns a list of changeset for this file in which the file was changed
         """
         if self.changeset is None:
             raise NodeError('Unable to get changeset for this FileNode')
         return self.changeset.get_file_history(self.path)
 
     @LazyProperty
     def annotate(self):
         """
         Returns a list of three element tuples with lineno,changeset and line
         """
         if self.changeset is None:
             raise NodeError('Unable to get changeset for this FileNode')
         return self.changeset.get_file_annotate(self.path)
 
     @LazyProperty
     def state(self):
         if not self.changeset:
             raise NodeError("Cannot check state of the node if it's not "
                 "linked with changeset")
         elif self.path in (node.path for node in self.changeset.added):
             return NodeState.ADDED
         elif self.path in (node.path for node in self.changeset.changed):
             return NodeState.CHANGED
         else:
             return NodeState.NOT_CHANGED
 
     @property
     def is_binary(self):
         """
         Returns True if file has binary content.
         """
-        _bin = '\0' in self._get_content()
-        return _bin
+        return b'\0' in self.content
 
     def is_browser_compatible_image(self):
         return self.mimetype in [
             "image/gif",
             "image/jpeg",
             "image/png",
             "image/bmp"
         ]
 
     @LazyProperty
     def extension(self):
         """Returns filenode extension"""
         return self.name.split('.')[-1]
 
     @property
     def is_executable(self):
         """
         Returns ``True`` if file has executable flag turned on.
         """
         return bool(self.mode & stat.S_IXUSR)
 
     def __repr__(self):
         return '<%s %r @ %s>' % (self.__class__.__name__, self.path,
                                  getattr(self.changeset, 'short_id', ''))
 
 
 class RemovedFileNode(FileNode):
     """
     Dummy FileNode class - trying to access any public attribute except path,
     name, kind or state (or methods/attributes checking those two) would raise
     RemovedFileNodeError.
     """
     ALLOWED_ATTRIBUTES = [
         'name', 'path', 'state', 'is_root', 'is_file', 'is_dir', 'kind',
         'added', 'changed', 'not_changed', 'removed'
     ]
 
     def __init__(self, path):
         """
         :param path: relative path to the node
         """
         super(RemovedFileNode, self).__init__(path=path)
 
     def __getattribute__(self, attr):
         if attr.startswith('_') or attr in RemovedFileNode.ALLOWED_ATTRIBUTES:
             return super(RemovedFileNode, self).__getattribute__(attr)
         raise RemovedFileNodeError("Cannot access attribute %s on "
             "RemovedFileNode" % attr)
 
     @LazyProperty
     def state(self):
         return NodeState.REMOVED
 
 
 class DirNode(Node):
     """
     DirNode stores list of files and directories within this node.
     Nodes may be used standalone but within repository context they
     lazily fetch data within same repository's changeset.
     """
 
     def __init__(self, path, nodes=(), changeset=None):
         """
         Only one of ``nodes`` and ``changeset`` may be given. Passing both
         would raise ``NodeError`` exception.
 
         :param path: relative path to the node
         :param nodes: content may be passed to constructor
         :param changeset: if given, will use it to lazily fetch content
         :param size: always 0 for ``DirNode``
         """
         if nodes and changeset:
             raise NodeError("Cannot use both nodes and changeset")
         super(DirNode, self).__init__(path, NodeKind.DIR)
         self.changeset = changeset
         self._nodes = nodes
 
     @LazyProperty
     def content(self):
         raise NodeError("%s represents a dir and has no ``content`` attribute"
             % self)
 
     @LazyProperty
     def nodes(self):
         if self.changeset:
             nodes = self.changeset.get_nodes(self.path)
         else:
             nodes = self._nodes
         self._nodes_dict = dict((node.path, node) for node in nodes)
         return sorted(nodes)
 
     @LazyProperty
     def files(self):
         return sorted((node for node in self.nodes if node.is_file()))
 
     @LazyProperty
     def dirs(self):
         return sorted((node for node in self.nodes if node.is_dir()))
 
     def __iter__(self):
         for node in self.nodes:
             yield node
 
     def get_node(self, path):
         """
         Returns node from within this particular ``DirNode``, so it is now
         allowed to fetch, i.e. node located at 'docs/api/index.rst' from node
         'docs'. In order to access deeper nodes one must fetch nodes between
         them first - this would work::
 
            docs = root.get_node('docs')
            docs.get_node('api').get_node('index.rst')
 
         :param: path - relative to the current node
 
         .. note::
            To access lazily (as in example above) node have to be initialized
            with related changeset object - without it node is out of
            context and may know nothing about anything else than nearest
            (located at same level) nodes.
         """
         try:
             path = path.rstrip('/')
             if path == '':
                 raise NodeError("Cannot retrieve node without path")
             self.nodes  # access nodes first in order to set _nodes_dict
             paths = path.split('/')
             if len(paths) == 1:
                 if not self.is_root():
                     path = '/'.join((self.path, paths[0]))
                 else:
                     path = paths[0]
                 return self._nodes_dict[path]
             elif len(paths) > 1:
                 if self.changeset is None:
                     raise NodeError("Cannot access deeper "
                                     "nodes without changeset")
                 else:
                     path1, path2 = paths[0], '/'.join(paths[1:])
                     return self.get_node(path1).get_node(path2)
             else:
                 raise KeyError
         except KeyError:
             raise NodeError("Node does not exist at %s" % path)
 
     @LazyProperty
     def state(self):
         raise NodeError("Cannot access state of DirNode")
 
     @LazyProperty
     def size(self):
         size = 0
         for root, dirs, files in self.changeset.walk(self.path):
             for f in files:
                 size += f.size
 
         return size
 
     def __repr__(self):
         return '<%s %r @ %s>' % (self.__class__.__name__, self.path,
                                  getattr(self.changeset, 'short_id', ''))
 
 
 class RootNode(DirNode):
     """
     DirNode being the root node of the repository.
     """
 
     def __init__(self, nodes=(), changeset=None):
         super(RootNode, self).__init__(path='', nodes=nodes,
             changeset=changeset)
 
     def __repr__(self):
         return '<%s>' % self.__class__.__name__
 
 
 class SubModuleNode(Node):
     """
     represents a SubModule of Git or SubRepo of Mercurial
     """
     is_binary = False
     size = 0
 
     def __init__(self, name, url, changeset=None, alias=None):
         self.path = name
         self.kind = NodeKind.SUBMODULE
         self.alias = alias
         # we have to use emptyChangeset here since this can point to svn/git/hg
         # submodules we cannot get from repository
         self.changeset = EmptyChangeset(str(changeset), alias=alias)
         self.url = url
 
     def __repr__(self):
         return '<%s %r @ %s>' % (self.__class__.__name__, self.path,
                                  getattr(self.changeset, 'short_id', ''))
 
     @LazyProperty
     def name(self):
         """
         Returns name of the node so if its path
         then only last part is returned.
         """
         org = safe_unicode(self.path.rstrip('/').split('/')[-1])
         return u'%s @ %s' % (org, self.changeset.short_id)

kallithea/lib/vcs/utils/annotate.py

@@ -1,177 +1,176 b''
 from pygments import highlight
 from pygments.formatters import HtmlFormatter
 
 from kallithea.lib.vcs.exceptions import VCSError
 from kallithea.lib.vcs.nodes import FileNode
+from kallithea.lib.vcs.utils import safe_unicode
 
 
 def annotate_highlight(filenode, annotate_from_changeset_func=None,
         order=None, headers=None, **options):
     """
     Returns html portion containing annotated table with 3 columns: line
     numbers, changeset information and pygmentized line of code.
 
     :param filenode: FileNode object
     :param annotate_from_changeset_func: function taking changeset and
       returning single annotate cell; needs break line at the end
     :param order: ordered sequence of ``ls`` (line numbers column),
       ``annotate`` (annotate column), ``code`` (code column); Default is
       ``['ls', 'annotate', 'code']``
     :param headers: dictionary with headers (keys are whats in ``order``
       parameter)
     """
     options['linenos'] = True
     formatter = AnnotateHtmlFormatter(filenode=filenode, order=order,
         headers=headers,
         annotate_from_changeset_func=annotate_from_changeset_func, **options)
-    lexer = filenode.lexer
-    highlighted = highlight(filenode.content, lexer, formatter)
-    return highlighted
+    return highlight(safe_unicode(filenode.content), filenode.lexer, formatter)
 
 
 class AnnotateHtmlFormatter(HtmlFormatter):
 
     def __init__(self, filenode, annotate_from_changeset_func=None,
             order=None, **options):
         """
         If ``annotate_from_changeset_func`` is passed it should be a function
         which returns string from the given changeset. For example, we may pass
         following function as ``annotate_from_changeset_func``::
 
             def changeset_to_anchor(changeset):
                 return '<a href="/changesets/%s/">%s</a>\n' % \
                        (changeset.id, changeset.id)
 
         :param annotate_from_changeset_func: see above
         :param order: (default: ``['ls', 'annotate', 'code']``); order of
           columns;
         :param options: standard pygment's HtmlFormatter options, there is
           extra option tough, ``headers``. For instance we can pass::
 
              formatter = AnnotateHtmlFormatter(filenode, headers={
                 'ls': '#',
                 'annotate': 'Annotate',
                 'code': 'Code',
              })
 
         """
         super(AnnotateHtmlFormatter, self).__init__(**options)
         self.annotate_from_changeset_func = annotate_from_changeset_func
         self.order = order or ('ls', 'annotate', 'code')
         headers = options.pop('headers', None)
         if headers and not ('ls' in headers and 'annotate' in headers and
             'code' in headers
         ):
             raise ValueError("If headers option dict is specified it must "
                 "all 'ls', 'annotate' and 'code' keys")
         self.headers = headers
         if isinstance(filenode, FileNode):
             self.filenode = filenode
         else:
             raise VCSError("This formatter expect FileNode parameter, not %r"
                 % type(filenode))
 
     def annotate_from_changeset(self, changeset):
         """
         Returns full html line for single changeset per annotated line.
         """
         if self.annotate_from_changeset_func:
             return self.annotate_from_changeset_func(changeset)
         else:
             return ''.join((changeset.id, '\n'))
 
     def _wrap_tablelinenos(self, inner):
         inner_lines = []
         lncount = 0
         for t, line in inner:
             if t:
                 lncount += 1
             inner_lines.append(line)
 
         fl = self.linenostart
         mw = len(str(lncount + fl - 1))
         sp = self.linenospecial
         st = self.linenostep
         la = self.lineanchors
         aln = self.anchorlinenos
         if sp:
             lines = []
 
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if i % sp == 0:
                         if aln:
                             lines.append('<a href="#%s-%d" class="special">'
                                          '%*d</a>' %
                                          (la, i, mw, i))
                         else:
                             lines.append('<span class="special">'
                                          '%*d</span>' % (mw, i))
                     else:
                         if aln:
                             lines.append('<a href="#%s-%d">'
                                          '%*d</a>' % (la, i, mw, i))
                         else:
                             lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
         else:
             lines = []
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if aln:
                         lines.append('<a href="#%s-%d">%*d</a>'
                                      % (la, i, mw, i))
                     else:
                         lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
 
         annotate_changesets = [tup[1] for tup in self.filenode.annotate]
         # If pygments cropped last lines break we need do that too
         ln_cs = len(annotate_changesets)
         ln_ = len(ls.splitlines())
         if ln_cs > ln_:
             annotate_changesets = annotate_changesets[:ln_ - ln_cs]
         annotate = ''.join((self.annotate_from_changeset(changeset)
             for changeset in annotate_changesets))
         # in case you wonder about the seemingly redundant <div> here:
         # since the content in the other cell also is wrapped in a div,
         # some browsers in some configurations seem to mess up the formatting.
         '''
         yield 0, ('<table class="%stable">' % self.cssclass +
                   '<tr><td class="linenos"><div class="linenodiv"><pre>' +
                   ls + '</pre></div></td>' +
                   '<td class="code">')
         yield 0, ''.join(inner_lines)
         yield 0, '</td></tr></table>'
 
         '''
         headers_row = []
         if self.headers:
             headers_row = ['<tr class="annotate-header">']
             for key in self.order:
                 td = ''.join(('<td>', self.headers[key], '</td>'))
                 headers_row.append(td)
             headers_row.append('</tr>')
 
         body_row_start = ['<tr>']
         for key in self.order:
             if key == 'ls':
                 body_row_start.append(
                     '<td class="linenos"><div class="linenodiv"><pre>' +
                     ls + '</pre></div></td>')
             elif key == 'annotate':
                 body_row_start.append(
                     '<td class="annotate"><div class="annotatediv"><pre>' +
                     annotate + '</pre></div></td>')
             elif key == 'code':
                 body_row_start.append('<td class="code">')
         yield 0, ('<table class="%stable">' % self.cssclass +
                   ''.join(headers_row) +
                   ''.join(body_row_start)
                   )
         yield 0, ''.join(inner_lines)
         yield 0, '</td></tr></table>'

kallithea/templates/admin/gists/edit.html

@@ -1,174 +1,174 b''
 ## -*- coding: utf-8 -*-
 <%inherit file="/base/base.html"/>
 
 <%block name="title">
     ${_('Edit Gist')} &middot; ${c.gist.gist_access_id}
 </%block>
 
 <%block name="js_extra">
   <script type="text/javascript" src="${h.url('/codemirror/lib/codemirror.js')}"></script>
   <script type="text/javascript" src="${h.url('/js/codemirror_loadmode.js')}"></script>
   <script type="text/javascript" src="${h.url('/codemirror/mode/meta.js')}"></script>
 </%block>
 <%block name="css_extra">
   <link rel="stylesheet" type="text/css" href="${h.url('/codemirror/lib/codemirror.css')}"/>
 </%block>
 
 <%def name="breadcrumbs_links()">
     ${_('Edit Gist')} &middot; ${c.gist.gist_access_id}
 </%def>
 
 <%block name="header_menu">
     ${self.menu('gists')}
 </%block>
 
 <%def name="main()">
 <div class="panel panel-primary">
     <div class="panel-heading clearfix">
         ${self.breadcrumbs()}
     </div>
 
     <div class="panel-body">
         <div id="edit_error" style="display: none" class="flash_msg">
             <div class="alert alert-dismissable alert-warning">
               <button type="button" class="close" data-dismiss="alert" aria-hidden="true"><i class="icon-cancel-circled"></i></button>
               ${(h.HTML(_('Gist was updated since you started editing. Copy your changes and click %(here)s to reload new version.'))
                              % {'here': h.link_to(_('here'),h.url('edit_gist', gist_id=c.gist.gist_access_id))})}
             </div>
             <script>
             if (typeof jQuery != 'undefined') {
                 $(".alert").alert();
             }
             </script>
         </div>
 
         <div id="files_data">
           ${h.form(h.url('edit_gist', gist_id=c.gist.gist_access_id), method='post', id='eform')}
             <div>
                 <input type="hidden" value="${c.file_changeset.raw_id}" name="parent_hash">
                 <textarea class="form-control"
                           id="description" name="description"
                           placeholder="${_('Gist description ...')}">${c.gist.gist_description}</textarea>
                 <div>
                     <label>
                         ${_('Gist lifetime')}
                         ${h.select('lifetime', '0', c.lifetime_options)}
                     </label>
                     <span class="text-muted">
                      %if c.gist.gist_expires == -1:
                       ${_('Expires')}: ${_('Never')}
                      %else:
                       ${_('Expires')}: ${h.age(h.time_to_datetime(c.gist.gist_expires))}
                      %endif
                    </span>
                 </div>
             </div>
 
             % for cnt, file in enumerate(c.files):
                 <div id="body" class="panel panel-default form-inline">
                     <div class="panel-heading">
                         <input type="hidden" value="${h.safe_unicode(file.path)}" name="org_files">
                         <input class="form-control" id="filename_${h.FID('f',file.path)}" name="files" size="30" type="text" value="${h.safe_unicode(file.path)}">
                         <select class="form-control" id="mimetype_${h.FID('f',file.path)}" name="mimetypes"></select>
                     </div>
                     <div class="panel-body no-padding">
                         <div id="editor_container">
-                            <textarea id="editor_${h.FID('f',file.path)}" name="contents" style="display:none">${file.content}</textarea>
+                            <textarea id="editor_${h.FID('f',file.path)}" name="contents" style="display:none">${safe_unicode(file.content)}</textarea>
                         </div>
                     </div>
                 </div>
 
                 ## dynamic edit box.
                 <script type="text/javascript">
                     $(document).ready(function(){
                         var myCodeMirror = initCodeMirror(${h.js('editor_' + h.FID('f',file.path))}, ${h.jshtml(request.script_name)}, '');
 
                         //inject new modes
                         var $mimetype_select = $(${h.js('#mimetype_' + h.FID('f',file.path))});
                         $mimetype_select.each(function(){
                             var modes_select = this;
                             var index = 1;
                             for(var i=0;i<CodeMirror.modeInfo.length;i++) {
                                 var m = CodeMirror.modeInfo[i];
                                 var opt = new Option(m.name, m.mime);
                                 $(opt).attr('mode', m.mode);
                                 if (m.mime == 'text/plain') {
                                     // default plain text
                                     $(opt).prop('selected', true);
                                     modes_select.options[0] = opt;
                                 } else {
                                     modes_select.options[index++] = opt;
                                 }
                             }
                         });
 
                         var $filename_input = $(${h.js('#filename_' + h.FID('f',file.path))});
                         // on select change set new mode
                         $mimetype_select.change(function(e){
                             var selected = e.currentTarget;
                             var node = selected.options[selected.selectedIndex];
                             var detected_mode = CodeMirror.findModeByMIME(node.value);
                             setCodeMirrorMode(myCodeMirror, detected_mode);
 
                             var proposed_ext = CodeMirror.findExtensionByMode(detected_mode);
                             var file_data = CodeMirror.getFilenameAndExt($filename_input.val());
                             var filename = file_data['filename'] || 'filename1';
                             $filename_input.val(filename + '.' + proposed_ext);
                         });
 
                         // on type the new filename set mode
                         $filename_input.keyup(function(e){
                             var file_data = CodeMirror.getFilenameAndExt(this.value);
                             if(file_data['ext'] != null){
                                 var detected_mode = CodeMirror.findModeByExtension(file_data['ext']) || CodeMirror.findModeByMIME('text/plain');
 
                                 if (detected_mode){
                                     setCodeMirrorMode(myCodeMirror, detected_mode);
                                     $mimetype_select.val(detected_mode.mime);
                                 }
                             }
                         });
 
                         // set mode on page load
                         var detected_mode = CodeMirror.findModeByExtension(${h.js(file.extension)});
 
                         if (detected_mode){
                             setCodeMirrorMode(myCodeMirror, detected_mode);
                             $mimetype_select.val(detected_mode.mime);
                         }
                     });
                 </script>
 
             %endfor
 
             <div>
             ${h.submit('update',_('Update Gist'),class_="btn btn-success")}
             <a class="btn btn-default" href="${h.url('gist', gist_id=c.gist.gist_access_id)}">${_('Cancel')}</a>
             </div>
           ${h.end_form()}
           <script>
               $('#update').on('click', function(e){
                   e.preventDefault();
 
                   // check for newer version.
                   $.ajax({
                     url: ${h.js(h.url('edit_gist_check_revision', gist_id=c.gist.gist_access_id))},
                     data: {'revision': ${h.js(c.file_changeset.raw_id)}, '_session_csrf_secret_token': _session_csrf_secret_token},
                     dataType: 'json',
                     type: 'POST',
                     success: function(data) {
                       if(data.success == false){
                           $('#edit_error').show();
                       }
                       else{
                         $('#eform').submit();
                       }
                     }
                   });
               });
           </script>
         </div>
     </div>
 
 </div>
 </%def>

kallithea/templates/files/files_edit.html

@@ -1,118 +1,118 b''
 <%inherit file="/base/base.html"/>
 
 <%block name="title">
     ${_('%s File Edit') % c.repo_name}
 </%block>
 
 <%block name="js_extra">
   <script type="text/javascript" src="${h.url('/codemirror/lib/codemirror.js')}"></script>
   <script type="text/javascript" src="${h.url('/js/codemirror_loadmode.js')}"></script>
   <script type="text/javascript" src="${h.url('/codemirror/mode/meta.js')}"></script>
 </%block>
 <%block name="css_extra">
   <link rel="stylesheet" type="text/css" href="${h.url('/codemirror/lib/codemirror.css')}"/>
 </%block>
 
 <%block name="header_menu">
     ${self.menu('repositories')}
 </%block>
 
 <%def name="breadcrumbs_links()">
     ${_('Edit file')} @ ${h.show_id(c.cs)}
 </%def>
 
 <%def name="main()">
 ${self.repo_context_bar('files')}
 <div class="panel panel-primary">
     <div class="panel-heading clearfix">
         <div class="pull-left">
             ${self.breadcrumbs()}
         </div>
         <div class="pull-right">
             <a href="#">${_('Branch')}: ${c.cs.branch}</a>
         </div>
     </div>
     <div class="panel-body" id="edit">
         <div id="files_data">
             ${h.form(h.url.current(),method='post',id='eform')}
             <h3 class="files_location">
               ${_('Location')}: ${h.files_breadcrumbs(c.repo_name,c.cs.raw_id,c.file.path)}
             </h3>
             <div id="body" class="panel panel-default">
               <div class="panel-heading clearfix form-inline form-group-sm">
                     <span>
                         <span><i class="icon-doc-inv"></i></span>
                         <span>${h.link_to(h.show_id(c.file.changeset),h.url('changeset_home',repo_name=c.repo_name,revision=c.file.changeset.raw_id))}</span>
                         <span>${h.format_byte_size(c.file.size,binary=True)}</span>
                         <span>${c.file.mimetype}</span>
                         <select class="form-control" id="mimetype" name="mimetype"></select>
                     </span>
                     <span class="pull-right buttons">
                       ${h.link_to(_('Show Annotation'),h.url('files_annotate_home',repo_name=c.repo_name,revision=c.cs.raw_id,f_path=c.f_path),class_="btn btn-default btn-xs")}
                       ${h.link_to(_('Show as Raw'),h.url('files_raw_home',repo_name=c.repo_name,revision=c.cs.raw_id,f_path=c.f_path),class_="btn btn-default btn-xs")}
                       ${h.link_to(_('Download as Raw'),h.url('files_rawfile_home',repo_name=c.repo_name,revision=c.cs.raw_id,f_path=c.f_path),class_="btn btn-default btn-xs")}
                       % if h.HasRepoPermissionLevel('write')(c.repo_name):
                        % if not c.file.is_binary:
                         ${h.link_to(_('Source'),h.url('files_home',repo_name=c.repo_name,revision=c.cs.raw_id,f_path=c.f_path),class_="btn btn-default btn-xs")}
                        % endif
                       % endif
                     </span>
               </div>
               <div class="panel-body no-padding">
-                <textarea id="editor" name="content" style="display:none">${h.escape(c.file.content)|n}</textarea>
+                <textarea id="editor" name="content" style="display:none">${h.escape(safe_unicode(c.file.content))|n}</textarea>
               </div>
             </div>
             <div>
               <div class="form-group">
                   <label>${_('Commit Message')}</label>
                   <textarea class="form-control" id="commit" name="message" placeholder="${c.default_message}"></textarea>
               </div>
               <div class="form-group buttons">
                 ${h.submit('commit',_('Commit Changes'),class_="btn btn-success")}
                 ${h.reset('reset',_('Reset'),class_="btn btn-default")}
               </div>
             </div>
             ${h.end_form()}
         </div>
     </div>
 </div>
 
 <script type="text/javascript">
     $(document).ready(function(){
         var reset_url = ${h.jshtml(h.url('files_home',repo_name=c.repo_name,revision=c.cs.raw_id,f_path=c.file.path))};
         var myCodeMirror = initCodeMirror('editor', ${h.jshtml(request.script_name)}, reset_url);
 
        //inject new modes, based on codeMirrors modeInfo object
         var $mimetype_select = $('#mimetype');
         $mimetype_select.each(function(){
             var modes_select = this;
             var index = 1;
             for(var i=0;i<CodeMirror.modeInfo.length;i++){
                 var m = CodeMirror.modeInfo[i];
                 var opt = new Option(m.name, m.mime);
                 $(opt).attr('mode', m.mode);
                 if (m.mime == 'text/plain') {
                     // default plain text
                     $(opt).prop('selected', true);
                     modes_select.options[0] = opt;
                 } else {
                     modes_select.options[index++] = opt;
                 }
             }
         });
         // try to detect the mode based on the file we edit
         var detected_mode = CodeMirror.findModeByExtension(${h.js(c.file.extension)});
         if(detected_mode){
             setCodeMirrorMode(myCodeMirror, detected_mode);
             $($mimetype_select.find('option[value="'+detected_mode.mime+'"]')[0]).prop('selected', true);
         }
 
         $mimetype_select.on('change', function(e){
             var selected = e.currentTarget;
             var node = selected.options[selected.selectedIndex];
             var detected_mode = CodeMirror.findModeByMIME(node.value);
             setCodeMirrorMode(myCodeMirror, detected_mode);
         });
     });
 </script>
 </%def>

kallithea/tests/vcs/test_git.py

@@ -1,844 +1,844 b''
 import datetime
 import os
 import sys
 import urllib2
 
 import mock
 import pytest
 
 from kallithea.lib.vcs.backends.git import GitChangeset, GitRepository
 from kallithea.lib.vcs.exceptions import NodeDoesNotExistError, RepositoryError, VCSError
 from kallithea.lib.vcs.nodes import DirNode, FileNode, NodeKind, NodeState
 from kallithea.model.scm import ScmModel
 from kallithea.tests.vcs.base import _BackendTestMixin
 from kallithea.tests.vcs.conf import TEST_GIT_REPO, TEST_GIT_REPO_CLONE, TESTS_TMP_PATH, get_new_dir
 
 
 class TestGitRepository(object):
 
     def __check_for_existing_repo(self):
         if os.path.exists(TEST_GIT_REPO_CLONE):
             pytest.fail('Cannot test git clone repo as location %s already '
                       'exists. You should manually remove it first.'
                       % TEST_GIT_REPO_CLONE)
 
     def setup_method(self):
         self.repo = GitRepository(TEST_GIT_REPO)
 
     def test_wrong_repo_path(self):
         wrong_repo_path = os.path.join(TESTS_TMP_PATH, 'errorrepo')
         with pytest.raises(RepositoryError):
             GitRepository(wrong_repo_path)
 
     def test_git_cmd_injection(self):
         repo_inject_path = TEST_GIT_REPO + '; echo "Cake";'
         with pytest.raises(urllib2.URLError):
             # Should fail because URL will contain the parts after ; too
             GitRepository(get_new_dir('injection-repo'), src_url=repo_inject_path, update_after_clone=True, create=True)
 
         with pytest.raises(RepositoryError):
             # Should fail on direct clone call, which as of this writing does not happen outside of class
             clone_fail_repo = GitRepository(get_new_dir('injection-repo'), create=True)
             clone_fail_repo.clone(repo_inject_path, update_after_clone=True,)
 
         # Verify correct quoting of evil characters that should work on posix file systems
         if sys.platform == 'win32':
             # windows does not allow '"' in dir names
             # and some versions of the git client don't like ` and '
             tricky_path = get_new_dir("tricky-path-repo-$")
         else:
             tricky_path = get_new_dir("tricky-path-repo-$'\"`")
         successfully_cloned = GitRepository(tricky_path, src_url=TEST_GIT_REPO, update_after_clone=True, create=True)
         # Repo should have been created
         assert not successfully_cloned._repo.bare
 
         if sys.platform == 'win32':
             # windows does not allow '"' in dir names
             # and some versions of the git client don't like ` and '
             tricky_path_2 = get_new_dir("tricky-path-2-repo-$")
         else:
             tricky_path_2 = get_new_dir("tricky-path-2-repo-$'\"`")
         successfully_cloned2 = GitRepository(tricky_path_2, src_url=tricky_path, bare=True, create=True)
         # Repo should have been created and thus used correct quoting for clone
         assert successfully_cloned2._repo.bare
 
         # Should pass because URL has been properly quoted
         successfully_cloned.pull(tricky_path_2)
         successfully_cloned2.fetch(tricky_path)
 
     def test_repo_create_with_spaces_in_path(self):
         repo_path = get_new_dir("path with spaces")
         repo = GitRepository(repo_path, src_url=None, bare=True, create=True)
         # Repo should have been created
         assert repo._repo.bare
 
     def test_repo_clone(self):
         self.__check_for_existing_repo()
         repo = GitRepository(TEST_GIT_REPO)
         repo_clone = GitRepository(TEST_GIT_REPO_CLONE,
             src_url=TEST_GIT_REPO, create=True, update_after_clone=True)
         assert len(repo.revisions) == len(repo_clone.revisions)
         # Checking hashes of changesets should be enough
         for changeset in repo.get_changesets():
             raw_id = changeset.raw_id
             assert raw_id == repo_clone.get_changeset(raw_id).raw_id
 
     def test_repo_clone_with_spaces_in_path(self):
         repo_path = get_new_dir("path with spaces")
         successfully_cloned = GitRepository(repo_path, src_url=TEST_GIT_REPO, update_after_clone=True, create=True)
         # Repo should have been created
         assert not successfully_cloned._repo.bare
 
         successfully_cloned.pull(TEST_GIT_REPO)
         self.repo.fetch(repo_path)
 
     def test_repo_clone_without_create(self):
         with pytest.raises(RepositoryError):
             GitRepository(TEST_GIT_REPO_CLONE + '_wo_create', src_url=TEST_GIT_REPO)
 
     def test_repo_clone_with_update(self):
         repo = GitRepository(TEST_GIT_REPO)
         clone_path = TEST_GIT_REPO_CLONE + '_with_update'
         repo_clone = GitRepository(clone_path,
             create=True, src_url=TEST_GIT_REPO, update_after_clone=True)
         assert len(repo.revisions) == len(repo_clone.revisions)
 
         # check if current workdir was updated
         fpath = os.path.join(clone_path, 'MANIFEST.in')
         assert os.path.isfile(fpath) == True, 'Repo was cloned and updated but file %s could not be found' % fpath
 
     def test_repo_clone_without_update(self):
         repo = GitRepository(TEST_GIT_REPO)
         clone_path = TEST_GIT_REPO_CLONE + '_without_update'
         repo_clone = GitRepository(clone_path,
             create=True, src_url=TEST_GIT_REPO, update_after_clone=False)
         assert len(repo.revisions) == len(repo_clone.revisions)
         # check if current workdir was *NOT* updated
         fpath = os.path.join(clone_path, 'MANIFEST.in')
         # Make sure it's not bare repo
         assert not repo_clone._repo.bare
         assert os.path.isfile(fpath) == False, 'Repo was cloned and updated but file %s was found' % fpath
 
     def test_repo_clone_into_bare_repo(self):
         repo = GitRepository(TEST_GIT_REPO)
         clone_path = TEST_GIT_REPO_CLONE + '_bare.git'
         repo_clone = GitRepository(clone_path, create=True,
             src_url=repo.path, bare=True)
         assert repo_clone._repo.bare
 
     def test_create_repo_is_not_bare_by_default(self):
         repo = GitRepository(get_new_dir('not-bare-by-default'), create=True)
         assert not repo._repo.bare
 
     def test_create_bare_repo(self):
         repo = GitRepository(get_new_dir('bare-repo'), create=True, bare=True)
         assert repo._repo.bare
 
     def test_revisions(self):
         # there are 112 revisions (by now)
         # so we can assume they would be available from now on
         subset = set([
             'c1214f7e79e02fc37156ff215cd71275450cffc3',
             '38b5fe81f109cb111f549bfe9bb6b267e10bc557',
             'fa6600f6848800641328adbf7811fd2372c02ab2',
             '102607b09cdd60e2793929c4f90478be29f85a17',
             '49d3fd156b6f7db46313fac355dca1a0b94a0017',
             '2d1028c054665b962fa3d307adfc923ddd528038',
             'd7e0d30fbcae12c90680eb095a4f5f02505ce501',
             'ff7ca51e58c505fec0dd2491de52c622bb7a806b',
             'dd80b0f6cf5052f17cc738c2951c4f2070200d7f',
             '8430a588b43b5d6da365400117c89400326e7992',
             'd955cd312c17b02143c04fa1099a352b04368118',
             'f67b87e5c629c2ee0ba58f85197e423ff28d735b',
             'add63e382e4aabc9e1afdc4bdc24506c269b7618',
             'f298fe1189f1b69779a4423f40b48edf92a703fc',
             'bd9b619eb41994cac43d67cf4ccc8399c1125808',
             '6e125e7c890379446e98980d8ed60fba87d0f6d1',
             'd4a54db9f745dfeba6933bf5b1e79e15d0af20bd',
             '0b05e4ed56c802098dfc813cbe779b2f49e92500',
             '191caa5b2c81ed17c0794bf7bb9958f4dcb0b87e',
             '45223f8f114c64bf4d6f853e3c35a369a6305520',
             'ca1eb7957a54bce53b12d1a51b13452f95bc7c7e',
             'f5ea29fc42ef67a2a5a7aecff10e1566699acd68',
             '27d48942240f5b91dfda77accd2caac94708cc7d',
             '622f0eb0bafd619d2560c26f80f09e3b0b0d78af',
             'e686b958768ee96af8029fe19c6050b1a8dd3b2b'])
         assert subset.issubset(set(self.repo.revisions))
 
     def test_slicing(self):
         # 4 1 5 10 95
         for sfrom, sto, size in [(0, 4, 4), (1, 2, 1), (10, 15, 5),
                                  (10, 20, 10), (5, 100, 95)]:
             revs = list(self.repo[sfrom:sto])
             assert len(revs) == size
             assert revs[0] == self.repo.get_changeset(sfrom)
             assert revs[-1] == self.repo.get_changeset(sto - 1)
 
     def test_branches(self):
         # TODO: Need more tests here
         # Removed (those are 'remotes' branches for cloned repo)
         #assert 'master' in self.repo.branches
         #assert 'gittree' in self.repo.branches
         #assert 'web-branch' in self.repo.branches
         for name, id in self.repo.branches.items():
             assert isinstance(self.repo.get_changeset(id), GitChangeset)
 
     def test_tags(self):
         # TODO: Need more tests here
         assert 'v0.1.1' in self.repo.tags
         assert 'v0.1.2' in self.repo.tags
         for name, id in self.repo.tags.items():
             assert isinstance(self.repo.get_changeset(id), GitChangeset)
 
     def _test_single_changeset_cache(self, revision):
         chset = self.repo.get_changeset(revision)
         assert revision in self.repo.changesets
         assert chset is self.repo.changesets[revision]
 
     def test_initial_changeset(self):
         id = self.repo.revisions[0]
         init_chset = self.repo.get_changeset(id)
         assert init_chset.message == 'initial import\n'
         assert init_chset.author == 'Marcin Kuzminski <[e-mail]>'
         for path in ('vcs/__init__.py',
                      'vcs/backends/BaseRepository.py',
                      'vcs/backends/__init__.py'):
             assert isinstance(init_chset.get_node(path), FileNode)
         for path in ('', 'vcs', 'vcs/backends'):
             assert isinstance(init_chset.get_node(path), DirNode)
 
         with pytest.raises(NodeDoesNotExistError):
             init_chset.get_node(path='foobar')
 
         node = init_chset.get_node('vcs/')
         assert hasattr(node, 'kind')
         assert node.kind == NodeKind.DIR
 
         node = init_chset.get_node('vcs')
         assert hasattr(node, 'kind')
         assert node.kind == NodeKind.DIR
 
         node = init_chset.get_node('vcs/__init__.py')
         assert hasattr(node, 'kind')
         assert node.kind == NodeKind.FILE
 
     def test_not_existing_changeset(self):
         with pytest.raises(RepositoryError):
             self.repo.get_changeset('f' * 40)
 
     def test_changeset10(self):
 
         chset10 = self.repo.get_changeset(self.repo.revisions[9])
         readme = """===
 VCS
 ===
 
 Various Version Control System management abstraction layer for Python.
 
 Introduction
 ------------
 
 TODO: To be written...
 
 """
         node = chset10.get_node('README.rst')
         assert node.kind == NodeKind.FILE
         assert node.content == readme
 
 
 class TestGitChangeset(object):
 
     def setup_method(self):
         self.repo = GitRepository(TEST_GIT_REPO)
 
     def test_default_changeset(self):
         tip = self.repo.get_changeset()
         assert tip == self.repo.get_changeset(None)
         assert tip == self.repo.get_changeset('tip')
 
     def test_root_node(self):
         tip = self.repo.get_changeset()
         assert tip.root is tip.get_node('')
 
     def test_lazy_fetch(self):
         """
         Test if changeset's nodes expands and are cached as we walk through
         the revision. This test is somewhat hard to write as order of tests
         is a key here. Written by running command after command in a shell.
         """
         commit_id = '2a13f185e4525f9d4b59882791a2d397b90d5ddc'
         assert commit_id in self.repo.revisions
         chset = self.repo.get_changeset(commit_id)
         assert len(chset.nodes) == 0
         root = chset.root
         assert len(chset.nodes) == 1
         assert len(root.nodes) == 8
         # accessing root.nodes updates chset.nodes
         assert len(chset.nodes) == 9
 
         docs = root.get_node('docs')
         # we haven't yet accessed anything new as docs dir was already cached
         assert len(chset.nodes) == 9
         assert len(docs.nodes) == 8
         # accessing docs.nodes updates chset.nodes
         assert len(chset.nodes) == 17
 
         assert docs is chset.get_node('docs')
         assert docs is root.nodes[0]
         assert docs is root.dirs[0]
         assert docs is chset.get_node('docs')
 
     def test_nodes_with_changeset(self):
         commit_id = '2a13f185e4525f9d4b59882791a2d397b90d5ddc'
         chset = self.repo.get_changeset(commit_id)
         root = chset.root
         docs = root.get_node('docs')
         assert docs is chset.get_node('docs')
         api = docs.get_node('api')
         assert api is chset.get_node('docs/api')
         index = api.get_node('index.rst')
         assert index is chset.get_node('docs/api/index.rst')
         assert index is chset.get_node('docs') \
                              .get_node('api') \
                              .get_node('index.rst')
 
     def test_branch_and_tags(self):
         # Those tests seem to show wrong results:
         # in Git, only heads have a branch - most changesets don't
         rev0 = self.repo.revisions[0]
         chset0 = self.repo.get_changeset(rev0)
         assert chset0.branch is None # should be 'master'?
         assert chset0.branches == [] # should be 'master'?
         assert chset0.tags == []
 
         rev10 = self.repo.revisions[10]
         chset10 = self.repo.get_changeset(rev10)
         assert chset10.branch is None # should be 'master'?
         assert chset10.branches == [] # should be 'master'?
         assert chset10.tags == []
 
         rev44 = self.repo.revisions[44]
         chset44 = self.repo.get_changeset(rev44)
         assert chset44.branch is None # should be 'web-branch'?
         assert chset44.branches == [] # should be 'web-branch'?
 
         tip = self.repo.get_changeset('tip')
         assert 'tip' not in tip.tags # it should be?
         assert not tip.tags # how it is!
 
     def _test_slices(self, limit, offset):
         count = self.repo.count()
         changesets = self.repo.get_changesets(limit=limit, offset=offset)
         idx = 0
         for changeset in changesets:
             rev = offset + idx
             idx += 1
             rev_id = self.repo.revisions[rev]
             if idx > limit:
                 pytest.fail("Exceeded limit already (getting revision %s, "
                     "there are %s total revisions, offset=%s, limit=%s)"
                     % (rev_id, count, offset, limit))
             assert changeset == self.repo.get_changeset(rev_id)
         result = list(self.repo.get_changesets(limit=limit, offset=offset))
         start = offset
         end = limit and offset + limit or None
         sliced = list(self.repo[start:end])
         pytest.failUnlessEqual(result, sliced,
             msg="Comparison failed for limit=%s, offset=%s"
             "(get_changeset returned: %s and sliced: %s"
             % (limit, offset, result, sliced))
 
     def _test_file_size(self, revision, path, size):
         node = self.repo.get_changeset(revision).get_node(path)
         assert node.is_file()
         assert node.size == size
 
     def test_file_size(self):
         to_check = (
             ('c1214f7e79e02fc37156ff215cd71275450cffc3',
                 'vcs/backends/BaseRepository.py', 502),
             ('d7e0d30fbcae12c90680eb095a4f5f02505ce501',
                 'vcs/backends/hg.py', 854),
             ('6e125e7c890379446e98980d8ed60fba87d0f6d1',
                 'setup.py', 1068),
             ('d955cd312c17b02143c04fa1099a352b04368118',
                 'vcs/backends/base.py', 2921),
             ('ca1eb7957a54bce53b12d1a51b13452f95bc7c7e',
                 'vcs/backends/base.py', 3936),
             ('f50f42baeed5af6518ef4b0cb2f1423f3851a941',
                 'vcs/backends/base.py', 6189),
         )
         for revision, path, size in to_check:
             self._test_file_size(revision, path, size)
 
     def _test_dir_size(self, revision, path, size):
         node = self.repo.get_changeset(revision).get_node(path)
         assert node.size == size
 
     def test_dir_size(self):
         to_check = (
             ('5f2c6ee195929b0be80749243c18121c9864a3b3', '/', 674076),
             ('7ab37bc680b4aa72c34d07b230c866c28e9fc204', '/', 674049),
             ('6892503fb8f2a552cef5f4d4cc2cdbd13ae1cd2f', '/', 671830),
         )
         for revision, path, size in to_check:
             self._test_dir_size(revision, path, size)
 
     def test_repo_size(self):
         assert self.repo.size == 674076
 
     def test_file_history(self):
         # we can only check if those revisions are present in the history
         # as we cannot update this test every time file is changed
         files = {
             'setup.py': [
                 '54386793436c938cff89326944d4c2702340037d',
                 '51d254f0ecf5df2ce50c0b115741f4cf13985dab',
                 '998ed409c795fec2012b1c0ca054d99888b22090',
                 '5e0eb4c47f56564395f76333f319d26c79e2fb09',
                 '0115510b70c7229dbc5dc49036b32e7d91d23acd',
                 '7cb3fd1b6d8c20ba89e2264f1c8baebc8a52d36e',
                 '2a13f185e4525f9d4b59882791a2d397b90d5ddc',
                 '191caa5b2c81ed17c0794bf7bb9958f4dcb0b87e',
                 'ff7ca51e58c505fec0dd2491de52c622bb7a806b',
             ],
             'vcs/nodes.py': [
                 '33fa3223355104431402a888fa77a4e9956feb3e',
                 'fa014c12c26d10ba682fadb78f2a11c24c8118e1',
                 'e686b958768ee96af8029fe19c6050b1a8dd3b2b',
                 'ab5721ca0a081f26bf43d9051e615af2cc99952f',
                 'c877b68d18e792a66b7f4c529ea02c8f80801542',
                 '4313566d2e417cb382948f8d9d7c765330356054',
                 '6c2303a793671e807d1cfc70134c9ca0767d98c2',
                 '54386793436c938cff89326944d4c2702340037d',
                 '54000345d2e78b03a99d561399e8e548de3f3203',
                 '1c6b3677b37ea064cb4b51714d8f7498f93f4b2b',
                 '2d03ca750a44440fb5ea8b751176d1f36f8e8f46',
                 '2a08b128c206db48c2f0b8f70df060e6db0ae4f8',
                 '30c26513ff1eb8e5ce0e1c6b477ee5dc50e2f34b',
                 'ac71e9503c2ca95542839af0ce7b64011b72ea7c',
                 '12669288fd13adba2a9b7dd5b870cc23ffab92d2',
                 '5a0c84f3e6fe3473e4c8427199d5a6fc71a9b382',
                 '12f2f5e2b38e6ff3fbdb5d722efed9aa72ecb0d5',
                 '5eab1222a7cd4bfcbabc218ca6d04276d4e27378',
                 'f50f42baeed5af6518ef4b0cb2f1423f3851a941',
                 'd7e390a45f6aa96f04f5e7f583ad4f867431aa25',
                 'f15c21f97864b4f071cddfbf2750ec2e23859414',
                 'e906ef056cf539a4e4e5fc8003eaf7cf14dd8ade',
                 'ea2b108b48aa8f8c9c4a941f66c1a03315ca1c3b',
                 '84dec09632a4458f79f50ddbbd155506c460b4f9',
                 '0115510b70c7229dbc5dc49036b32e7d91d23acd',
                 '2a13f185e4525f9d4b59882791a2d397b90d5ddc',
                 '3bf1c5868e570e39569d094f922d33ced2fa3b2b',
                 'b8d04012574729d2c29886e53b1a43ef16dd00a1',
                 '6970b057cffe4aab0a792aa634c89f4bebf01441',
                 'dd80b0f6cf5052f17cc738c2951c4f2070200d7f',
                 'ff7ca51e58c505fec0dd2491de52c622bb7a806b',
             ],
             'vcs/backends/git.py': [
                 '4cf116ad5a457530381135e2f4c453e68a1b0105',
                 '9a751d84d8e9408e736329767387f41b36935153',
                 'cb681fb539c3faaedbcdf5ca71ca413425c18f01',
                 '428f81bb652bcba8d631bce926e8834ff49bdcc6',
                 '180ab15aebf26f98f714d8c68715e0f05fa6e1c7',
                 '2b8e07312a2e89e92b90426ab97f349f4bce2a3a',
                 '50e08c506174d8645a4bb517dd122ac946a0f3bf',
                 '54000345d2e78b03a99d561399e8e548de3f3203',
             ],
         }
         for path, revs in files.items():
             node = self.repo.get_changeset(revs[0]).get_node(path)
             node_revs = [chset.raw_id for chset in node.history]
             assert set(revs).issubset(set(node_revs)), "We assumed that %s is subset of revisions for which file %s " \
                 "has been changed, and history of that node returned: %s" \
                 % (revs, path, node_revs)
 
     def test_file_annotate(self):
         files = {
             'vcs/backends/__init__.py': {
                 'c1214f7e79e02fc37156ff215cd71275450cffc3': {
                     'lines_no': 1,
                     'changesets': [
                         'c1214f7e79e02fc37156ff215cd71275450cffc3',
                     ],
                 },
                 '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647': {
                     'lines_no': 21,
                     'changesets': [
                         '49d3fd156b6f7db46313fac355dca1a0b94a0017',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                     ],
                 },
                 'e29b67bd158580fc90fc5e9111240b90e6e86064': {
                     'lines_no': 32,
                     'changesets': [
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '5eab1222a7cd4bfcbabc218ca6d04276d4e27378',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '992f38217b979d0b0987d0bae3cc26dac85d9b19',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '54000345d2e78b03a99d561399e8e548de3f3203',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '78c3f0c23b7ee935ec276acb8b8212444c33c396',
                         '992f38217b979d0b0987d0bae3cc26dac85d9b19',
                         '992f38217b979d0b0987d0bae3cc26dac85d9b19',
                         '992f38217b979d0b0987d0bae3cc26dac85d9b19',
                         '992f38217b979d0b0987d0bae3cc26dac85d9b19',
                         '2a13f185e4525f9d4b59882791a2d397b90d5ddc',
                         '992f38217b979d0b0987d0bae3cc26dac85d9b19',
                         '78c3f0c23b7ee935ec276acb8b8212444c33c396',
                         '992f38217b979d0b0987d0bae3cc26dac85d9b19',
                         '992f38217b979d0b0987d0bae3cc26dac85d9b19',
                         '992f38217b979d0b0987d0bae3cc26dac85d9b19',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '992f38217b979d0b0987d0bae3cc26dac85d9b19',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '992f38217b979d0b0987d0bae3cc26dac85d9b19',
                         '992f38217b979d0b0987d0bae3cc26dac85d9b19',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                         '16fba1ae9334d79b66d7afed2c2dfbfa2ae53647',
                     ],
                 },
             },
         }
 
         for fname, revision_dict in files.items():
             for rev, data in revision_dict.items():
                 cs = self.repo.get_changeset(rev)
 
                 l1_1 = [x[1] for x in cs.get_file_annotate(fname)]
                 l1_2 = [x[2]().raw_id for x in cs.get_file_annotate(fname)]
                 assert l1_1 == l1_2
                 l1 = l1_1
                 l2 = files[fname][rev]['changesets']
                 assert l1 == l2, "The lists of revision for %s@rev %s" \
                     "from annotation list should match each other, " \
                     "got \n%s \nvs \n%s " % (fname, rev, l1, l2)
 
     def test_files_state(self):
         """
         Tests state of FileNodes.
         """
         node = self.repo \
             .get_changeset('e6ea6d16e2f26250124a1f4b4fe37a912f9d86a0') \
             .get_node('vcs/utils/diffs.py')
         assert node.state, NodeState.ADDED
         assert node.added
         assert not node.changed
         assert not node.not_changed
         assert not node.removed
 
         node = self.repo \
             .get_changeset('33fa3223355104431402a888fa77a4e9956feb3e') \
             .get_node('.hgignore')
         assert node.state, NodeState.CHANGED
         assert not node.added
         assert node.changed
         assert not node.not_changed
         assert not node.removed
 
         node = self.repo \
             .get_changeset('e29b67bd158580fc90fc5e9111240b90e6e86064') \
             .get_node('setup.py')
         assert node.state, NodeState.NOT_CHANGED
         assert not node.added
         assert not node.changed
         assert node.not_changed
         assert not node.removed
 
         # If node has REMOVED state then trying to fetch it would raise
         # ChangesetError exception
         chset = self.repo.get_changeset(
             'fa6600f6848800641328adbf7811fd2372c02ab2')
         path = 'vcs/backends/BaseRepository.py'
         with pytest.raises(NodeDoesNotExistError):
             chset.get_node(path)
         # but it would be one of ``removed`` (changeset's attribute)
         assert path in [rf.path for rf in chset.removed]
 
         chset = self.repo.get_changeset(
             '54386793436c938cff89326944d4c2702340037d')
         changed = ['setup.py', 'tests/test_nodes.py', 'vcs/backends/hg.py',
             'vcs/nodes.py']
         assert set(changed) == set([f.path for f in chset.changed])
 
     def test_commit_message_is_unicode(self):
         for cs in self.repo:
             assert isinstance(cs.message, unicode)
 
     def test_changeset_author_is_unicode(self):
         for cs in self.repo:
             assert isinstance(cs.author, unicode)
 
-    def test_repo_files_content_is_unicode(self):
+    def test_repo_files_content_is_bytes(self):
         changeset = self.repo.get_changeset()
         for node in changeset.get_node('/'):
             if node.is_file():
-                assert isinstance(node.content, unicode)
+                assert isinstance(node.content, bytes)
 
     def test_wrong_path(self):
         # There is 'setup.py' in the root dir but not there:
         path = 'foo/bar/setup.py'
         tip = self.repo.get_changeset()
         with pytest.raises(VCSError):
             tip.get_node(path)
 
     def test_author_email(self):
         assert '[e-mail]' == self.repo.get_changeset('c1214f7e79e02fc37156ff215cd71275450cffc3').author_email
         assert '[e-mail]' == self.repo.get_changeset('ff7ca51e58c505fec0dd2491de52c622bb7a806b').author_email
         assert '' == self.repo.get_changeset('8430a588b43b5d6da365400117c89400326e7992').author_email
 
     def test_author_username(self):
         assert 'Marcin Kuzminski' == self.repo.get_changeset('c1214f7e79e02fc37156ff215cd71275450cffc3').author_name
         assert 'Lukasz Balcerzak' == self.repo.get_changeset('ff7ca51e58c505fec0dd2491de52c622bb7a806b').author_name
         assert 'marcink none@none' == self.repo.get_changeset('8430a588b43b5d6da365400117c89400326e7992').author_name
 
 
 class TestGitSpecificWithRepo(_BackendTestMixin):
     backend_alias = 'git'
 
     @classmethod
     def _get_commits(cls):
         return [
             {
                 'message': 'Initial',
                 'author': 'Joe Doe <[e-mail]>',
                 'date': datetime.datetime(2010, 1, 1, 20),
                 'added': [
                     FileNode('foobar/static/js/admin/base.js', content='base'),
                     FileNode('foobar/static/admin', content='admin',
                         mode=0o120000), # this is a link
                     FileNode('foo', content='foo'),
                 ],
             },
             {
                 'message': 'Second',
                 'author': 'Joe Doe <[e-mail]>',
                 'date': datetime.datetime(2010, 1, 1, 22),
                 'added': [
                     FileNode('foo2', content='foo2'),
                 ],
             },
         ]
 
     def test_paths_slow_traversing(self):
         cs = self.repo.get_changeset()
         assert cs.get_node('foobar').get_node('static').get_node('js').get_node('admin').get_node('base.js').content == 'base'
 
     def test_paths_fast_traversing(self):
         cs = self.repo.get_changeset()
         assert cs.get_node('foobar/static/js/admin/base.js').content == 'base'
 
     def test_workdir_get_branch(self):
         self.repo.run_git_command(['checkout', '-b', 'production'])
         # Regression test: one of following would fail if we don't check
         # .git/HEAD file
         self.repo.run_git_command(['checkout', 'production'])
         assert self.repo.workdir.get_branch() == 'production'
         self.repo.run_git_command(['checkout', 'master'])
         assert self.repo.workdir.get_branch() == 'master'
 
     def test_get_diff_runs_git_command_with_hashes(self):
         self.repo._run_git_command = mock.Mock(return_value=(b'', b''))
         self.repo.get_diff(0, 1)
         self.repo._run_git_command.assert_called_once_with(
             ['diff', '-U3', '--full-index', '--binary', '-p', '-M', '--abbrev=40',
              self.repo._get_revision(0), self.repo._get_revision(1)], cwd=self.repo.path)
 
     def test_get_diff_runs_git_command_with_str_hashes(self):
         self.repo._run_git_command = mock.Mock(return_value=(b'', b''))
         self.repo.get_diff(self.repo.EMPTY_CHANGESET, 1)
         self.repo._run_git_command.assert_called_once_with(
             ['show', '-U3', '--full-index', '--binary', '-p', '-M', '--abbrev=40',
              self.repo._get_revision(1)], cwd=self.repo.path)
 
     def test_get_diff_runs_git_command_with_path_if_its_given(self):
         self.repo._run_git_command = mock.Mock(return_value=(b'', b''))
         self.repo.get_diff(0, 1, 'foo')
         self.repo._run_git_command.assert_called_once_with(
             ['diff', '-U3', '--full-index', '--binary', '-p', '-M', '--abbrev=40',
              self.repo._get_revision(0), self.repo._get_revision(1), '--', 'foo'], cwd=self.repo.path)
 
     def test_get_diff_does_not_sanitize_valid_context(self):
         almost_overflowed_long_int = 2**31-1
 
         self.repo._run_git_command = mock.Mock(return_value=(b'', b''))
         self.repo.get_diff(0, 1, 'foo', context=almost_overflowed_long_int)
         self.repo._run_git_command.assert_called_once_with(
             ['diff', '-U' + str(almost_overflowed_long_int), '--full-index', '--binary', '-p', '-M', '--abbrev=40',
              self.repo._get_revision(0), self.repo._get_revision(1), '--', 'foo'], cwd=self.repo.path)
 
     def test_get_diff_sanitizes_overflowing_context(self):
         overflowed_long_int = 2**31
         sanitized_overflowed_long_int = overflowed_long_int-1
 
         self.repo._run_git_command = mock.Mock(return_value=(b'', b''))
         self.repo.get_diff(0, 1, 'foo', context=overflowed_long_int)
 
         self.repo._run_git_command.assert_called_once_with(
             ['diff', '-U' + str(sanitized_overflowed_long_int), '--full-index', '--binary', '-p', '-M', '--abbrev=40',
              self.repo._get_revision(0), self.repo._get_revision(1), '--', 'foo'], cwd=self.repo.path)
 
     def test_get_diff_does_not_sanitize_zero_context(self):
         zero_context = 0
 
         self.repo._run_git_command = mock.Mock(return_value=(b'', b''))
         self.repo.get_diff(0, 1, 'foo', context=zero_context)
 
         self.repo._run_git_command.assert_called_once_with(
             ['diff', '-U' + str(zero_context), '--full-index', '--binary', '-p', '-M', '--abbrev=40',
              self.repo._get_revision(0), self.repo._get_revision(1), '--', 'foo'], cwd=self.repo.path)
 
     def test_get_diff_sanitizes_negative_context(self):
         negative_context = -10
 
         self.repo._run_git_command = mock.Mock(return_value=(b'', b''))
         self.repo.get_diff(0, 1, 'foo', context=negative_context)
 
         self.repo._run_git_command.assert_called_once_with(
             ['diff', '-U0', '--full-index', '--binary', '-p', '-M', '--abbrev=40',
              self.repo._get_revision(0), self.repo._get_revision(1), '--', 'foo'], cwd=self.repo.path)
 
 
 class TestGitRegression(_BackendTestMixin):
     backend_alias = 'git'
 
     @classmethod
     def _get_commits(cls):
         return [
             {
                 'message': 'Initial',
                 'author': 'Joe Doe <[e-mail]>',
                 'date': datetime.datetime(2010, 1, 1, 20),
                 'added': [
                     FileNode('bot/__init__.py', content='base'),
                     FileNode('bot/templates/404.html', content='base'),
                     FileNode('bot/templates/500.html', content='base'),
                 ],
             },
             {
                 'message': 'Second',
                 'author': 'Joe Doe <[e-mail]>',
                 'date': datetime.datetime(2010, 1, 1, 22),
                 'added': [
                     FileNode('bot/build/migrations/1.py', content='foo2'),
                     FileNode('bot/build/migrations/2.py', content='foo2'),
                     FileNode('bot/build/static/templates/f.html', content='foo2'),
                     FileNode('bot/build/static/templates/f1.html', content='foo2'),
                     FileNode('bot/build/templates/err.html', content='foo2'),
                     FileNode('bot/build/templates/err2.html', content='foo2'),
                 ],
             },
         ]
 
     def test_similar_paths(self):
         cs = self.repo.get_changeset()
         paths = lambda *n: [x.path for x in n]
         assert paths(*cs.get_nodes('bot')) == ['bot/build', 'bot/templates', 'bot/__init__.py']
         assert paths(*cs.get_nodes('bot/build')) == ['bot/build/migrations', 'bot/build/static', 'bot/build/templates']
         assert paths(*cs.get_nodes('bot/build/static')) == ['bot/build/static/templates']
         # this get_nodes below causes troubles !
         assert paths(*cs.get_nodes('bot/build/static/templates')) == ['bot/build/static/templates/f.html', 'bot/build/static/templates/f1.html']
         assert paths(*cs.get_nodes('bot/build/templates')) == ['bot/build/templates/err.html', 'bot/build/templates/err2.html']
         assert paths(*cs.get_nodes('bot/templates/')) == ['bot/templates/404.html', 'bot/templates/500.html']
 
 
 class TestGitHooks(object):
     """
     Tests related to hook functionality of Git repositories.
     """
 
     def setup_method(self):
         # For each run we want a fresh repo.
         self.repo_directory = get_new_dir("githookrepo")
         self.repo = GitRepository(self.repo_directory, create=True)
 
         # Create a dictionary where keys are hook names, and values are paths to
         # them. Deduplicates code in tests a bit.
         self.hook_directory = self.repo.get_hook_location()
         self.kallithea_hooks = dict((h, os.path.join(self.hook_directory, h)) for h in ("pre-receive", "post-receive"))
 
     def test_hooks_created_if_missing(self):
         """
         Tests if hooks are installed in repository if they are missing.
         """
 
         for hook, hook_path in self.kallithea_hooks.iteritems():
             if os.path.exists(hook_path):
                 os.remove(hook_path)
 
         ScmModel().install_git_hooks(repo=self.repo)
 
         for hook, hook_path in self.kallithea_hooks.iteritems():
             assert os.path.exists(hook_path)
 
     def test_kallithea_hooks_updated(self):
         """
         Tests if hooks are updated if they are Kallithea hooks already.
         """
 
         for hook, hook_path in self.kallithea_hooks.iteritems():
             with open(hook_path, "w") as f:
                 f.write("KALLITHEA_HOOK_VER=0.0.0\nJUST_BOGUS")
 
         ScmModel().install_git_hooks(repo=self.repo)
 
         for hook, hook_path in self.kallithea_hooks.iteritems():
             with open(hook_path) as f:
                 assert "JUST_BOGUS" not in f.read()
 
     def test_custom_hooks_untouched(self):
         """
         Tests if hooks are left untouched if they are not Kallithea hooks.
         """
 
         for hook, hook_path in self.kallithea_hooks.iteritems():
             with open(hook_path, "w") as f:
                 f.write("#!/bin/bash\n#CUSTOM_HOOK")
 
         ScmModel().install_git_hooks(repo=self.repo)
 
         for hook, hook_path in self.kallithea_hooks.iteritems():
             with open(hook_path) as f:
                 assert "CUSTOM_HOOK" in f.read()
 
     def test_custom_hooks_forced_update(self):
         """
         Tests if hooks are forcefully updated even though they are custom hooks.
         """
 
         for hook, hook_path in self.kallithea_hooks.iteritems():
             with open(hook_path, "w") as f:
                 f.write("#!/bin/bash\n#CUSTOM_HOOK")
 
         ScmModel().install_git_hooks(repo=self.repo, force_create=True)
 
         for hook, hook_path in self.kallithea_hooks.iteritems():
             with open(hook_path) as f:
                 assert "KALLITHEA_HOOK_VER" in f.read()

kallithea/tests/vcs/test_hg.py

@@ -1,592 +1,592 b''
 import os
 
 import mock
 import pytest
 
 from kallithea.lib.vcs.backends.hg import MercurialChangeset, MercurialRepository
 from kallithea.lib.vcs.exceptions import NodeDoesNotExistError, RepositoryError, VCSError
 from kallithea.lib.vcs.nodes import NodeKind, NodeState
 from kallithea.lib.vcs.utils import safe_str
 from kallithea.tests.vcs.conf import TEST_HG_REPO, TEST_HG_REPO_CLONE, TEST_HG_REPO_PULL, TESTS_TMP_PATH
 
 
 class TestMercurialRepository(object):
 
     def __check_for_existing_repo(self):
         if os.path.exists(TEST_HG_REPO_CLONE):
             pytest.fail('Cannot test mercurial clone repo as location %s already '
                       'exists. You should manually remove it first.'
                       % TEST_HG_REPO_CLONE)
 
     def setup_method(self):
         self.repo = MercurialRepository(safe_str(TEST_HG_REPO))
 
     def test_wrong_repo_path(self):
         wrong_repo_path = os.path.join(TESTS_TMP_PATH, 'errorrepo')
         with pytest.raises(RepositoryError):
             MercurialRepository(wrong_repo_path)
 
     def test_unicode_path_repo(self):
         with pytest.raises(VCSError):
             MercurialRepository(u'iShouldFail')
 
     def test_repo_clone(self):
         self.__check_for_existing_repo()
         repo = MercurialRepository(safe_str(TEST_HG_REPO))
         repo_clone = MercurialRepository(TEST_HG_REPO_CLONE,
             src_url=TEST_HG_REPO, update_after_clone=True)
         assert len(repo.revisions) == len(repo_clone.revisions)
         # Checking hashes of changesets should be enough
         for changeset in repo.get_changesets():
             raw_id = changeset.raw_id
             assert raw_id == repo_clone.get_changeset(raw_id).raw_id
 
     def test_repo_clone_with_update(self):
         repo = MercurialRepository(safe_str(TEST_HG_REPO))
         repo_clone = MercurialRepository(TEST_HG_REPO_CLONE + '_w_update',
             src_url=TEST_HG_REPO, update_after_clone=True)
         assert len(repo.revisions) == len(repo_clone.revisions)
 
         # check if current workdir was updated
         assert os.path.isfile(
             os.path.join(
                 TEST_HG_REPO_CLONE + '_w_update', 'MANIFEST.in'
             )
         )
 
     def test_repo_clone_without_update(self):
         repo = MercurialRepository(safe_str(TEST_HG_REPO))
         repo_clone = MercurialRepository(TEST_HG_REPO_CLONE + '_wo_update',
             src_url=TEST_HG_REPO, update_after_clone=False)
         assert len(repo.revisions) == len(repo_clone.revisions)
         assert not os.path.isfile(
             os.path.join(
                 TEST_HG_REPO_CLONE + '_wo_update', 'MANIFEST.in'
             )
         )
 
     def test_pull(self):
         if os.path.exists(TEST_HG_REPO_PULL):
             pytest.fail('Cannot test mercurial pull command as location %s '
                       'already exists. You should manually remove it first'
                       % TEST_HG_REPO_PULL)
         repo_new = MercurialRepository(TEST_HG_REPO_PULL, create=True)
         assert len(self.repo.revisions) > len(repo_new.revisions)
 
         repo_new.pull(self.repo.path)
         repo_new = MercurialRepository(TEST_HG_REPO_PULL)
         assert len(self.repo.revisions) == len(repo_new.revisions)
 
     def test_revisions(self):
         # there are 21 revisions at bitbucket now
         # so we can assume they would be available from now on
         subset = set(['b986218ba1c9b0d6a259fac9b050b1724ed8e545',
                  '3d8f361e72ab303da48d799ff1ac40d5ac37c67e',
                  '6cba7170863a2411822803fa77a0a264f1310b35',
                  '56349e29c2af3ac913b28bde9a2c6154436e615b',
                  '2dda4e345facb0ccff1a191052dd1606dba6781d',
                  '6fff84722075f1607a30f436523403845f84cd9e',
                  '7d4bc8ec6be56c0f10425afb40b6fc315a4c25e7',
                  '3803844fdbd3b711175fc3da9bdacfcd6d29a6fb',
                  'dc5d2c0661b61928834a785d3e64a3f80d3aad9c',
                  'be90031137367893f1c406e0a8683010fd115b79',
                  'db8e58be770518cbb2b1cdfa69146e47cd481481',
                  '84478366594b424af694a6c784cb991a16b87c21',
                  '17f8e105dddb9f339600389c6dc7175d395a535c',
                  '20a662e756499bde3095ffc9bc0643d1def2d0eb',
                  '2e319b85e70a707bba0beff866d9f9de032aa4f9',
                  '786facd2c61deb9cf91e9534735124fb8fc11842',
                  '94593d2128d38210a2fcd1aabff6dda0d6d9edf8',
                  'aa6a0de05b7612707db567078e130a6cd114a9a7',
                  'eada5a770da98ab0dd7325e29d00e0714f228d09'
                 ])
         assert subset.issubset(set(self.repo.revisions))
 
         # check if we have the proper order of revisions
         org = ['b986218ba1c9b0d6a259fac9b050b1724ed8e545',
                 '3d8f361e72ab303da48d799ff1ac40d5ac37c67e',
                 '6cba7170863a2411822803fa77a0a264f1310b35',
                 '56349e29c2af3ac913b28bde9a2c6154436e615b',
                 '2dda4e345facb0ccff1a191052dd1606dba6781d',
                 '6fff84722075f1607a30f436523403845f84cd9e',
                 '7d4bc8ec6be56c0f10425afb40b6fc315a4c25e7',
                 '3803844fdbd3b711175fc3da9bdacfcd6d29a6fb',
                 'dc5d2c0661b61928834a785d3e64a3f80d3aad9c',
                 'be90031137367893f1c406e0a8683010fd115b79',
                 'db8e58be770518cbb2b1cdfa69146e47cd481481',
                 '84478366594b424af694a6c784cb991a16b87c21',
                 '17f8e105dddb9f339600389c6dc7175d395a535c',
                 '20a662e756499bde3095ffc9bc0643d1def2d0eb',
                 '2e319b85e70a707bba0beff866d9f9de032aa4f9',
                 '786facd2c61deb9cf91e9534735124fb8fc11842',
                 '94593d2128d38210a2fcd1aabff6dda0d6d9edf8',
                 'aa6a0de05b7612707db567078e130a6cd114a9a7',
                 'eada5a770da98ab0dd7325e29d00e0714f228d09',
                 '2c1885c735575ca478bf9e17b0029dca68824458',
                 'd9bcd465040bf869799b09ad732c04e0eea99fe9',
                 '469e9c847fe1f6f7a697b8b25b4bc5b48780c1a7',
                 '4fb8326d78e5120da2c7468dcf7098997be385da',
                 '62b4a097164940bd66030c4db51687f3ec035eed',
                 '536c1a19428381cfea92ac44985304f6a8049569',
                 '965e8ab3c44b070cdaa5bf727ddef0ada980ecc4',
                 '9bb326a04ae5d98d437dece54be04f830cf1edd9',
                 'f8940bcb890a98c4702319fbe36db75ea309b475',
                 'ff5ab059786ebc7411e559a2cc309dfae3625a3b',
                 '6b6ad5f82ad5bb6190037671bd254bd4e1f4bf08',
                 'ee87846a61c12153b51543bf860e1026c6d3dcba', ]
         assert org == self.repo.revisions[:31]
 
     def test_iter_slice(self):
         sliced = list(self.repo[:10])
         itered = list(self.repo)[:10]
         assert sliced == itered
 
     def test_slicing(self):
         # 4 1 5 10 95
         for sfrom, sto, size in [(0, 4, 4), (1, 2, 1), (10, 15, 5),
                                  (10, 20, 10), (5, 100, 95)]:
             revs = list(self.repo[sfrom:sto])
             assert len(revs) == size
             assert revs[0] == self.repo.get_changeset(sfrom)
             assert revs[-1] == self.repo.get_changeset(sto - 1)
 
     def test_branches(self):
         # TODO: Need more tests here
 
         # active branches
         assert 'default' in self.repo.branches
         assert 'stable' in self.repo.branches
 
         # closed
         assert 'git' in self.repo._get_branches(closed=True)
         assert 'web' in self.repo._get_branches(closed=True)
 
         for name, id in self.repo.branches.items():
             assert isinstance(self.repo.get_changeset(id), MercurialChangeset)
 
     def test_tip_in_tags(self):
         # tip is always a tag
         assert 'tip' in self.repo.tags
 
     def test_tip_changeset_in_tags(self):
         tip = self.repo.get_changeset()
         assert self.repo.tags['tip'] == tip.raw_id
 
     def test_initial_changeset(self):
 
         init_chset = self.repo.get_changeset(0)
         assert init_chset.message == 'initial import'
         assert init_chset.author == 'Marcin Kuzminski <[e-mail]>'
         assert sorted(init_chset._file_paths) == sorted([
             'vcs/__init__.py',
             'vcs/backends/BaseRepository.py',
             'vcs/backends/__init__.py',
         ])
 
         assert sorted(init_chset._dir_paths) == sorted(['', 'vcs', 'vcs/backends'])
 
         with pytest.raises(NodeDoesNotExistError):
             init_chset.get_node(path='foobar')
 
         node = init_chset.get_node('vcs/')
         assert hasattr(node, 'kind')
         assert node.kind == NodeKind.DIR
 
         node = init_chset.get_node('vcs')
         assert hasattr(node, 'kind')
         assert node.kind == NodeKind.DIR
 
         node = init_chset.get_node('vcs/__init__.py')
         assert hasattr(node, 'kind')
         assert node.kind == NodeKind.FILE
 
     def test_not_existing_changeset(self):
         # rawid
         with pytest.raises(RepositoryError):
             self.repo.get_changeset('abcd' * 10)
         # shortid
         with pytest.raises(RepositoryError):
             self.repo.get_changeset('erro' * 4)
         # numeric
         with pytest.raises(RepositoryError):
             self.repo.get_changeset(self.repo.count() + 1)
 
         # Small chance we ever get to this one
         revision = pow(2, 30)
         with pytest.raises(RepositoryError):
             self.repo.get_changeset(revision)
 
     def test_changeset10(self):
 
         chset10 = self.repo.get_changeset(10)
         readme = """===
 VCS
 ===
 
 Various Version Control System management abstraction layer for Python.
 
 Introduction
 ------------
 
 TODO: To be written...
 
 """
         node = chset10.get_node('README.rst')
         assert node.kind == NodeKind.FILE
         assert node.content == readme
 
     @mock.patch('kallithea.lib.vcs.backends.hg.repository.diffopts')
     def test_get_diff_does_not_sanitize_zero_context(self, mock_diffopts):
         zero_context = 0
 
         self.repo.get_diff(0, 1, 'foo', context=zero_context)
 
         mock_diffopts.assert_called_once_with(git=True, showfunc=True, ignorews=False, context=zero_context)
 
     @mock.patch('kallithea.lib.vcs.backends.hg.repository.diffopts')
     def test_get_diff_sanitizes_negative_context(self, mock_diffopts):
         negative_context = -10
         zero_context = 0
 
         self.repo.get_diff(0, 1, 'foo', context=negative_context)
 
         mock_diffopts.assert_called_once_with(git=True, showfunc=True, ignorews=False, context=zero_context)
 
 
 class TestMercurialChangeset(object):
 
     def setup_method(self):
         self.repo = MercurialRepository(safe_str(TEST_HG_REPO))
 
     def _test_equality(self, changeset):
         revision = changeset.revision
         assert changeset == self.repo.get_changeset(revision)
 
     def test_equality(self):
         revs = [0, 10, 20]
         changesets = [self.repo.get_changeset(rev) for rev in revs]
         for changeset in changesets:
             self._test_equality(changeset)
 
     def test_default_changeset(self):
         tip = self.repo.get_changeset('tip')
         assert tip == self.repo.get_changeset()
         assert tip == self.repo.get_changeset(revision=None)
         assert tip == list(self.repo[-1:])[0]
 
     def test_root_node(self):
         tip = self.repo.get_changeset('tip')
         assert tip.root is tip.get_node('')
 
     def test_lazy_fetch(self):
         """
         Test if changeset's nodes expands and are cached as we walk through
         the revision. This test is somewhat hard to write as order of tests
         is a key here. Written by running command after command in a shell.
         """
         chset = self.repo.get_changeset(45)
         assert len(chset.nodes) == 0
         root = chset.root
         assert len(chset.nodes) == 1
         assert len(root.nodes) == 8
         # accessing root.nodes updates chset.nodes
         assert len(chset.nodes) == 9
 
         docs = root.get_node('docs')
         # we haven't yet accessed anything new as docs dir was already cached
         assert len(chset.nodes) == 9
         assert len(docs.nodes) == 8
         # accessing docs.nodes updates chset.nodes
         assert len(chset.nodes) == 17
 
         assert docs is chset.get_node('docs')
         assert docs is root.nodes[0]
         assert docs is root.dirs[0]
         assert docs is chset.get_node('docs')
 
     def test_nodes_with_changeset(self):
         chset = self.repo.get_changeset(45)
         root = chset.root
         docs = root.get_node('docs')
         assert docs is chset.get_node('docs')
         api = docs.get_node('api')
         assert api is chset.get_node('docs/api')
         index = api.get_node('index.rst')
         assert index is chset.get_node('docs/api/index.rst')
         assert index is chset.get_node('docs').get_node('api').get_node('index.rst')
 
     def test_branch_and_tags(self):
         chset0 = self.repo.get_changeset(0)
         assert chset0.branch == 'default'
         assert chset0.branches == ['default']
         assert chset0.tags == []
 
         chset10 = self.repo.get_changeset(10)
         assert chset10.branch == 'default'
         assert chset10.branches == ['default']
         assert chset10.tags == []
 
         chset44 = self.repo.get_changeset(44)
         assert chset44.branch == 'web'
         assert chset44.branches == ['web']
 
         tip = self.repo.get_changeset('tip')
         assert 'tip' in tip.tags
 
     def _test_file_size(self, revision, path, size):
         node = self.repo.get_changeset(revision).get_node(path)
         assert node.is_file()
         assert node.size == size
 
     def test_file_size(self):
         to_check = (
             (10, 'setup.py', 1068),
             (20, 'setup.py', 1106),
             (60, 'setup.py', 1074),
 
             (10, 'vcs/backends/base.py', 2921),
             (20, 'vcs/backends/base.py', 3936),
             (60, 'vcs/backends/base.py', 6189),
         )
         for revision, path, size in to_check:
             self._test_file_size(revision, path, size)
 
     def _test_dir_size(self, revision, path, size):
         node = self.repo.get_changeset(revision).get_node(path)
         assert not node.is_file()
         assert node.size == size
 
     def test_dir_size(self):
         to_check = (
             ('96507bd11ecc', '/', 682421),
             ('a53d9201d4bc', '/', 682410),
             ('90243de06161', '/', 682006),
         )
         for revision, path, size in to_check:
             self._test_dir_size(revision, path, size)
 
     def test_repo_size(self):
         assert self.repo.size == 682421
 
     def test_file_history(self):
         # we can only check if those revisions are present in the history
         # as we cannot update this test every time file is changed
         files = {
             'setup.py': [7, 18, 45, 46, 47, 69, 77],
             'vcs/nodes.py': [7, 8, 24, 26, 30, 45, 47, 49, 56, 57, 58, 59, 60,
                 61, 73, 76],
             'vcs/backends/hg.py': [4, 5, 6, 11, 12, 13, 14, 15, 16, 21, 22, 23,
                 26, 27, 28, 30, 31, 33, 35, 36, 37, 38, 39, 40, 41, 44, 45, 47,
                 48, 49, 53, 54, 55, 58, 60, 61, 67, 68, 69, 70, 73, 77, 78, 79,
                 82],
         }
         for path, revs in files.items():
             tip = self.repo.get_changeset(revs[-1])
             node = tip.get_node(path)
             node_revs = [chset.revision for chset in node.history]
             assert set(revs).issubset(set(node_revs)), \
                 "We assumed that %s is subset of revisions for which file %s " \
                 "has been changed, and history of that node returned: %s" \
                 % (revs, path, node_revs)
 
     def test_file_annotate(self):
         files = {
                  'vcs/backends/__init__.py':
                   {89: {'lines_no': 31,
                         'changesets': [32, 32, 61, 32, 32, 37, 32, 32, 32, 44,
                                        37, 37, 37, 37, 45, 37, 44, 37, 37, 37,
                                        32, 32, 32, 32, 37, 32, 37, 37, 32,
                                        32, 32]},
                    20: {'lines_no': 1,
                         'changesets': [4]},
                    55: {'lines_no': 31,
                         'changesets': [32, 32, 45, 32, 32, 37, 32, 32, 32, 44,
                                        37, 37, 37, 37, 45, 37, 44, 37, 37, 37,
                                        32, 32, 32, 32, 37, 32, 37, 37, 32,
                                        32, 32]}},
                  'vcs/exceptions.py':
                  {89: {'lines_no': 18,
                        'changesets': [16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
                                       16, 16, 17, 16, 16, 18, 18, 18]},
                   20: {'lines_no': 18,
                        'changesets': [16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
                                       16, 16, 17, 16, 16, 18, 18, 18]},
                   55: {'lines_no': 18, 'changesets': [16, 16, 16, 16, 16, 16,
                                                       16, 16, 16, 16, 16, 16,
                                                       17, 16, 16, 18, 18, 18]}},
                  'MANIFEST.in': {89: {'lines_no': 5,
                                       'changesets': [7, 7, 7, 71, 71]},
                                  20: {'lines_no': 3,
                                       'changesets': [7, 7, 7]},
                                  55: {'lines_no': 3,
                                      'changesets': [7, 7, 7]}}}
 
         for fname, revision_dict in files.items():
             for rev, data in revision_dict.items():
                 cs = self.repo.get_changeset(rev)
                 l1_1 = [x[1] for x in cs.get_file_annotate(fname)]
                 l1_2 = [x[2]().raw_id for x in cs.get_file_annotate(fname)]
                 assert l1_1 == l1_2
                 l1 = l1_2 = [x[2]().revision for x in cs.get_file_annotate(fname)]
                 l2 = files[fname][rev]['changesets']
                 assert l1 == l2, "The lists of revision for %s@rev%s" \
                     "from annotation list should match each other," \
                     "got \n%s \nvs \n%s " % (fname, rev, l1, l2)
 
     def test_changeset_state(self):
         """
         Tests which files have been added/changed/removed at particular revision
         """
 
         # rev 46ad32a4f974:
         # hg st --rev 46ad32a4f974
         #    changed: 13
         #    added:   20
         #    removed: 1
         changed = set(['.hgignore'
             , 'README.rst', 'docs/conf.py', 'docs/index.rst', 'setup.py'
             , 'tests/test_hg.py', 'tests/test_nodes.py', 'vcs/__init__.py'
             , 'vcs/backends/__init__.py', 'vcs/backends/base.py'
             , 'vcs/backends/hg.py', 'vcs/nodes.py', 'vcs/utils/__init__.py'])
 
         added = set(['docs/api/backends/hg.rst'
             , 'docs/api/backends/index.rst', 'docs/api/index.rst'
             , 'docs/api/nodes.rst', 'docs/api/web/index.rst'
             , 'docs/api/web/simplevcs.rst', 'docs/installation.rst'
             , 'docs/quickstart.rst', 'setup.cfg', 'vcs/utils/baseui_config.py'
             , 'vcs/utils/web.py', 'vcs/web/__init__.py', 'vcs/web/exceptions.py'
             , 'vcs/web/simplevcs/__init__.py', 'vcs/web/simplevcs/exceptions.py'
             , 'vcs/web/simplevcs/middleware.py', 'vcs/web/simplevcs/models.py'
             , 'vcs/web/simplevcs/settings.py', 'vcs/web/simplevcs/utils.py'
             , 'vcs/web/simplevcs/views.py'])
 
         removed = set(['docs/api.rst'])
 
         chset64 = self.repo.get_changeset('46ad32a4f974')
         assert set((node.path for node in chset64.added)) == added
         assert set((node.path for node in chset64.changed)) == changed
         assert set((node.path for node in chset64.removed)) == removed
 
         # rev b090f22d27d6:
         # hg st --rev b090f22d27d6
         #    changed: 13
         #    added:   20
         #    removed: 1
         chset88 = self.repo.get_changeset('b090f22d27d6')
         assert set((node.path for node in chset88.added)) == set()
         assert set((node.path for node in chset88.changed)) == set(['.hgignore'])
         assert set((node.path for node in chset88.removed)) == set()
 
         # 85:
         #    added:   2 ['vcs/utils/diffs.py', 'vcs/web/simplevcs/views/diffs.py']
         #    changed: 4 ['vcs/web/simplevcs/models.py', ...]
         #    removed: 1 ['vcs/utils/web.py']
         chset85 = self.repo.get_changeset(85)
         assert set((node.path for node in chset85.added)) == set([
             'vcs/utils/diffs.py',
             'vcs/web/simplevcs/views/diffs.py'
         ])
 
         assert set((node.path for node in chset85.changed)) == set([
             'vcs/web/simplevcs/models.py',
             'vcs/web/simplevcs/utils.py',
             'vcs/web/simplevcs/views/__init__.py',
             'vcs/web/simplevcs/views/repository.py',
         ])
 
         assert set((node.path for node in chset85.removed)) == set([
             'vcs/utils/web.py'
         ])
 
 
     def test_files_state(self):
         """
         Tests state of FileNodes.
         """
         chset = self.repo.get_changeset(85)
         node = chset.get_node('vcs/utils/diffs.py')
         assert node.state, NodeState.ADDED
         assert node.added
         assert not node.changed
         assert not node.not_changed
         assert not node.removed
 
         chset = self.repo.get_changeset(88)
         node = chset.get_node('.hgignore')
         assert node.state, NodeState.CHANGED
         assert not node.added
         assert node.changed
         assert not node.not_changed
         assert not node.removed
 
         chset = self.repo.get_changeset(85)
         node = chset.get_node('setup.py')
         assert node.state, NodeState.NOT_CHANGED
         assert not node.added
         assert not node.changed
         assert node.not_changed
         assert not node.removed
 
         # If node has REMOVED state then trying to fetch it would raise
         # ChangesetError exception
         chset = self.repo.get_changeset(2)
         path = 'vcs/backends/BaseRepository.py'
         with pytest.raises(NodeDoesNotExistError):
             chset.get_node(path)
         # but it would be one of ``removed`` (changeset's attribute)
         assert path in [rf.path for rf in chset.removed]
 
     def test_commit_message_is_unicode(self):
         for cm in self.repo:
             assert isinstance(cm.message, unicode)
 
     def test_changeset_author_is_unicode(self):
         for cm in self.repo:
             assert isinstance(cm.author, unicode)
 
-    def test_repo_files_content_is_unicode(self):
+    def test_repo_files_content_is_bytes(self):
         test_changeset = self.repo.get_changeset(100)
         for node in test_changeset.get_node('/'):
             if node.is_file():
-                assert isinstance(node.content, unicode)
+                assert isinstance(node.content, bytes)
 
     def test_wrong_path(self):
         # There is 'setup.py' in the root dir but not there:
         path = 'foo/bar/setup.py'
         with pytest.raises(VCSError):
             self.repo.get_changeset().get_node(path)
 
     def test_archival_file(self):
         # TODO:
         pass
 
     def test_archival_as_generator(self):
         # TODO:
         pass
 
     def test_archival_wrong_kind(self):
         tip = self.repo.get_changeset()
         with pytest.raises(VCSError):
             tip.fill_archive(kind='error')
 
     def test_archival_empty_prefix(self):
         # TODO:
         pass
 
     def test_author_email(self):
         assert '[e-mail]' == self.repo.get_changeset('b986218ba1c9').author_email
         assert '[e-mail]' == self.repo.get_changeset('3803844fdbd3').author_email
         assert '' == self.repo.get_changeset('84478366594b').author_email
 
     def test_author_username(self):
         assert 'Marcin Kuzminski' == self.repo.get_changeset('b986218ba1c9').author_name
         assert 'Lukasz Balcerzak' == self.repo.get_changeset('3803844fdbd3').author_name
         assert 'marcink' == self.repo.get_changeset('84478366594b').author_name
 
     def test_successors(self):
         init_chset = self.repo.get_changeset(0)
         assert init_chset.successors == []
 
     def test_predecessors(self):
         init_chset = self.repo.get_changeset(0)
         assert len(init_chset.predecessors) == 0