rhodecode/lib/markup_renderer.py

# -*- coding: utf-8 -*-

# Copyright (C) 2011-2016  RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/


"""
Renderer for markup languages with ability to parse using rst or markdown
"""

import re
import os
import logging
import itertools

from mako.lookup import TemplateLookup

from docutils.core import publish_parts
from docutils.parsers.rst import directives
import markdown

from rhodecode.lib.markdown_ext import (
    UrlizeExtension, GithubFlavoredMarkdownExtension)
from rhodecode.lib.utils2 import safe_unicode, md5_safe, MENTIONS_REGEX

log = logging.getLogger(__name__)

# default renderer used to generate automated comments
DEFAULT_COMMENTS_RENDERER = 'rst'


class MarkupRenderer(object):
    RESTRUCTUREDTEXT_DISALLOWED_DIRECTIVES = ['include', 'meta', 'raw']

    MARKDOWN_PAT = re.compile(r'\.(md|mkdn?|mdown|markdown)$', re.IGNORECASE)
    RST_PAT = re.compile(r'\.re?st$', re.IGNORECASE)
    PLAIN_PAT = re.compile(r'^readme$', re.IGNORECASE)

    # list of readme files to search in file tree and display in summary
    # attached weights defines the search  order lower is first
    ALL_READMES = [
        ('readme', 0), ('README', 0), ('Readme', 0),
        ('doc/readme', 1), ('doc/README', 1), ('doc/Readme', 1),
        ('Docs/readme', 2), ('Docs/README', 2), ('Docs/Readme', 2),
        ('DOCS/readme', 2), ('DOCS/README', 2), ('DOCS/Readme', 2),
        ('docs/readme', 2), ('docs/README', 2), ('docs/Readme', 2),
    ]
    # extension together with weights. Lower is first means we control how
    # extensions are attached to readme names with those.
    PLAIN_EXTS = [
        ('', 0),  # special case that renders READMES names without extension
        ('.text', 2), ('.TEXT', 2),
        ('.txt', 3), ('.TXT', 3)
    ]

    RST_EXTS = [
        ('.rst', 1), ('.rest', 1),
        ('.RST', 2), ('.REST', 2)
    ]

    MARKDOWN_EXTS = [
        ('.md', 1), ('.MD', 1),
        ('.mkdn', 2), ('.MKDN', 2),
        ('.mdown', 3), ('.MDOWN', 3),
        ('.markdown', 4), ('.MARKDOWN', 4)
    ]

    ALL_EXTS = PLAIN_EXTS + MARKDOWN_EXTS + RST_EXTS

    def _detect_renderer(self, source, filename=None):
        """
        runs detection of what renderer should be used for generating html
        from a markup language

        filename can be also explicitly a renderer name

        :param source:
        :param filename:
        """

        if MarkupRenderer.MARKDOWN_PAT.findall(filename):
            detected_renderer = 'markdown'
        elif MarkupRenderer.RST_PAT.findall(filename):
            detected_renderer = 'rst'
        elif MarkupRenderer.PLAIN_PAT.findall(filename):
            detected_renderer = 'rst'
        else:
            detected_renderer = 'plain'

        return getattr(MarkupRenderer, detected_renderer)

    @classmethod
    def renderer_from_filename(cls, filename, exclude):
        """
        Detect renderer markdown/rst from filename and optionally use exclude
        list to remove some options. This is mostly used in helpers.
        Returns None when no renderer can be detected.
        """
        def _filter(elements):
            if isinstance(exclude, (list, tuple)):
                return [x for x in elements if x not in exclude]
            return elements

        if filename.endswith(
                tuple(_filter([x[0] for x in cls.MARKDOWN_EXTS if x[0]]))):
            return 'markdown'
        if filename.endswith(tuple(_filter([x[0] for x in cls.RST_EXTS if x[0]]))):
            return 'rst'

        return None

    @classmethod
    def generate_readmes(cls, all_readmes, extensions):
        combined = itertools.product(all_readmes, extensions)
        # sort by filename weight(y[0][1]) + extensions weight(y[1][1])
        prioritized_readmes = sorted(combined, key=lambda y: y[0][1] + y[1][1])
        # filename, extension
        return [''.join([x[0][0], x[1][0]]) for x in prioritized_readmes]

    def pick_readme_order(self, default_renderer):

        if default_renderer == 'markdown':
            markdown = self.generate_readmes(self.ALL_READMES, self.MARKDOWN_EXTS)
            readme_order = markdown + self.generate_readmes(
                self.ALL_READMES, self.RST_EXTS + self.PLAIN_EXTS)
        elif default_renderer == 'rst':
            markdown = self.generate_readmes(self.ALL_READMES, self.RST_EXTS)
            readme_order = markdown + self.generate_readmes(
                self.ALL_READMES, self.MARKDOWN_EXTS + self.PLAIN_EXTS)
        else:
            readme_order = self.generate_readmes(self.ALL_READMES, self.ALL_EXTS)

        return readme_order

    def render(self, source, filename=None):
        """
        Renders a given filename using detected renderer
        it detects renderers based on file extension or mimetype.
        At last it will just do a simple html replacing new lines with <br/>

        :param file_name:
        :param source:
        """

        renderer = self._detect_renderer(source, filename)
        readme_data = renderer(source)
        return readme_data

    @classmethod
    def _flavored_markdown(cls, text):
        """
        Github style flavored markdown

        :param text:
        """

        # Extract pre blocks.
        extractions = {}

        def pre_extraction_callback(matchobj):
            digest = md5_safe(matchobj.group(0))
            extractions[digest] = matchobj.group(0)
            return "{gfm-extraction-%s}" % digest
        pattern = re.compile(r'<pre>.*?</pre>', re.MULTILINE | re.DOTALL)
        text = re.sub(pattern, pre_extraction_callback, text)

        # Prevent foo_bar_baz from ending up with an italic word in the middle.
        def italic_callback(matchobj):
            s = matchobj.group(0)
            if list(s).count('_') >= 2:
                return s.replace('_', r'\_')
            return s
        text = re.sub(r'^(?! {4}|\t)\w+_\w+_\w[\w_]*', italic_callback, text)

        # Insert pre block extractions.
        def pre_insert_callback(matchobj):
            return '\n\n' + extractions[matchobj.group(1)]
        text = re.sub(r'\{gfm-extraction-([0-9a-f]{32})\}',
                      pre_insert_callback, text)

        return text

    @classmethod
    def urlify_text(cls, text):
        url_pat = re.compile(r'(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]'
                             r'|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)')

        def url_func(match_obj):
            url_full = match_obj.groups()[0]
            return '<a href="%(url)s">%(url)s</a>' % ({'url': url_full})

        return url_pat.sub(url_func, text)

    @classmethod
    def plain(cls, source, universal_newline=True):
        source = safe_unicode(source)
        if universal_newline:
            newline = '\n'
            source = newline.join(source.splitlines())

        source = cls.urlify_text(source)
        return '<br />' + source.replace("\n", '<br />')

    @classmethod
    def markdown(cls, source, safe=True, flavored=True, mentions=False):
        # It does not allow to insert inline HTML. In presence of HTML tags, it
        # will replace them instead with [HTML_REMOVED]. This is controlled by
        # the safe_mode=True parameter of the markdown method.
        extensions = ['codehilite', 'extra', 'def_list', 'sane_lists']
        if flavored:
            extensions.append(GithubFlavoredMarkdownExtension())

        if mentions:
            mention_pat = re.compile(MENTIONS_REGEX)

            def wrapp(match_obj):
                uname = match_obj.groups()[0]
                return ' **@%(uname)s** ' % {'uname': uname}
            mention_hl = mention_pat.sub(wrapp, source).strip()
            # we extracted mentions render with this using Mentions false
            return cls.markdown(mention_hl, safe=safe, flavored=flavored,
                                mentions=False)

        source = safe_unicode(source)
        try:
            if flavored:
                source = cls._flavored_markdown(source)
            return markdown.markdown(
                source, extensions, safe_mode=True, enable_attributes=False)
        except Exception:
            log.exception('Error when rendering Markdown')
            if safe:
                log.debug('Fallback to render in plain mode')
                return cls.plain(source)
            else:
                raise

    @classmethod
    def rst(cls, source, safe=True, mentions=False):
        if mentions:
            mention_pat = re.compile(MENTIONS_REGEX)

            def wrapp(match_obj):
                uname = match_obj.groups()[0]
                return ' **@%(uname)s** ' % {'uname': uname}
            mention_hl = mention_pat.sub(wrapp, source).strip()
            # we extracted mentions render with this using Mentions false
            return cls.rst(mention_hl, safe=safe, mentions=False)

        source = safe_unicode(source)
        try:
            docutils_settings = dict(
                [(alias, None) for alias in
                 cls.RESTRUCTUREDTEXT_DISALLOWED_DIRECTIVES])

            docutils_settings.update({'input_encoding': 'unicode',
                                      'report_level': 4})

            for k, v in docutils_settings.iteritems():
                directives.register_directive(k, v)

            parts = publish_parts(source=source,
                                  writer_name="html4css1",
                                  settings_overrides=docutils_settings)

            return parts['html_title'] + parts["fragment"]
        except Exception:
            log.exception('Error when rendering RST')
            if safe:
                log.debug('Fallbacking to render in plain mode')
                return cls.plain(source)
            else:
                raise


class RstTemplateRenderer(object):

    def __init__(self):
        base = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
        rst_template_dirs = [os.path.join(base, 'templates', 'rst_templates')]
        self.template_store = TemplateLookup(
            directories=rst_template_dirs,
            input_encoding='utf-8',
            imports=['from rhodecode.lib import helpers as h'])

    def _get_template(self, templatename):
        return self.template_store.get_template(templatename)

    def render(self, template_name, **kwargs):
        template = self._get_template(template_name)
        return template.render(**kwargs)