rhodecode/lib/index/search_utils.py

# -*- coding: utf-8 -*-

# Copyright (C) 2012-2019 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import re

import pygments.filter
import pygments.filters
from pygments.token import Comment

HL_BEG_MARKER = '__RCSearchHLMarkBEG__'
HL_END_MARKER = '__RCSearchHLMarkEND__'
HL_MARKER_RE = '{}(.*?){}'.format(HL_BEG_MARKER, HL_END_MARKER)


class ElasticSearchHLFilter(pygments.filters.Filter):
    _names = [HL_BEG_MARKER, HL_END_MARKER]

    def __init__(self, **options):
        pygments.filters.Filter.__init__(self, **options)

    def filter(self, lexer, stream):
        def tokenize(_value):
            for token in re.split('({}|{})'.format(
                    self._names[0], self._names[1]), _value):
                if token:
                    yield token

        hl = False
        for ttype, value in stream:

            if self._names[0] in value or self._names[1] in value:
                for item in tokenize(value):
                    if item == self._names[0]:
                        # skip marker, but start HL
                        hl = True
                        continue
                    elif item == self._names[1]:
                        hl = False
                        continue

                    if hl:
                        yield Comment.ElasticMatch, item
                    else:
                        yield ttype, item
            else:
                if hl:
                    yield Comment.ElasticMatch, value
                else:
                    yield ttype, value


def extract_phrases(text_query):
    """
    Extracts phrases from search term string making sure phrases
    contained in double quotes are kept together - and discarding empty values
    or fully whitespace values eg.

    'some   text "a phrase" more' => ['some', 'text', 'a phrase', 'more']

    """

    in_phrase = False
    buf = ''
    phrases = []
    for char in text_query:
        if in_phrase:
            if char == '"':  # end phrase
                phrases.append(buf)
                buf = ''
                in_phrase = False
                continue
            else:
                buf += char
                continue
        else:
            if char == '"':  # start phrase
                in_phrase = True
                phrases.append(buf)
                buf = ''
                continue
            elif char == ' ':
                phrases.append(buf)
                buf = ''
                continue
            else:
                buf += char

    phrases.append(buf)
    phrases = [phrase.strip() for phrase in phrases if phrase.strip()]
    return phrases


def get_matching_phrase_offsets(text, phrases):
    """
    Returns a list of string offsets in `text` that the list of `terms` match

    >>> get_matching_phrase_offsets('some text here', ['some', 'here'])
    [(0, 4), (10, 14)]

    """
    phrases = phrases or []
    offsets = []

    for phrase in phrases:
        for match in re.finditer(phrase, text):
            offsets.append((match.start(), match.end()))

    return offsets


def get_matching_markers_offsets(text, markers=None):
    """
    Returns a list of string offsets in `text` that the are between matching markers

    >>> get_matching_markers_offsets('$1some$2 text $1here$2 marked', ['\$1(.*?)\$2'])
    [(0, 5), (16, 22)]

    """
    markers = markers or [HL_MARKER_RE]
    offsets = []

    if markers:
        for mark in markers:
            for match in re.finditer(mark, text):
                offsets.append((match.start(), match.end()))

    return offsets


def normalize_text_for_matching(x):
    """
    Replaces all non alfanum characters to spaces and lower cases the string,
    useful for comparing two text strings without punctuation
    """
    return re.sub(r'[^\w]', ' ', x.lower())


def get_matching_line_offsets(lines, terms=None, markers=None):
    """ Return a set of `lines` indices (starting from 1) matching a
    text search query, along with `context` lines above/below matching lines

    :param lines: list of strings representing lines
    :param terms: search term string to match in lines eg. 'some text'
    :param markers: instead of terms, use highlight markers instead that
        mark beginning and end for matched item. eg. ['START(.*?)END']

     eg.

    text = '''
    words words words
    words words words
    some text some
    words words words
    words words words
    text here what
    '''
    get_matching_line_offsets(text, 'text', context=1)
    6, {3: [(5, 9)], 6: [(0, 4)]]

    """
    matching_lines = {}
    line_index = 0

    if terms:
        phrases = [normalize_text_for_matching(phrase)
                   for phrase in extract_phrases(terms)]

        for line_index, line in enumerate(lines.splitlines(), start=1):
            normalized_line = normalize_text_for_matching(line)
            match_offsets = get_matching_phrase_offsets(normalized_line, phrases)
            if match_offsets:
                matching_lines[line_index] = match_offsets

    else:
        markers = markers or [HL_MARKER_RE]
        for line_index, line in enumerate(lines.splitlines(), start=1):
            match_offsets = get_matching_markers_offsets(line, markers=markers)
            if match_offsets:
                matching_lines[line_index] = match_offsets

    return line_index, matching_lines