# HG changeset patch # User Marcin Kuzminski # Date 2012-02-16 03:41:58 # Node ID b6c902d88472b28c1b7bd43c5b92774c1d694776 # Parent 4d3179d2adfee7afd100befbf47befebad3bc34a bumbed whoosh to 2.3.X series - fixed some html issues on search results diff --git a/requires.txt b/requires.txt --- a/requires.txt +++ b/requires.txt @@ -5,7 +5,7 @@ formencode==1.2.4 SQLAlchemy==0.7.4 Mako==0.5.0 pygments>=1.4 -whoosh<1.8 +whoosh<2.4 celery>=2.2.5,<2.3 babel python-dateutil>=1.5.0,<2.0.0 diff --git a/rhodecode/__init__.py b/rhodecode/__init__.py --- a/rhodecode/__init__.py +++ b/rhodecode/__init__.py @@ -44,7 +44,7 @@ requirements = [ "SQLAlchemy==0.7.4", "Mako==0.5.0", "pygments>=1.4", - "whoosh<1.8", + "whoosh<2.4", "celery>=2.2.5,<2.3", "babel", "python-dateutil>=1.5.0,<2.0.0", diff --git a/rhodecode/controllers/search.py b/rhodecode/controllers/search.py --- a/rhodecode/controllers/search.py +++ b/rhodecode/controllers/search.py @@ -26,7 +26,7 @@ import logging import traceback from pylons.i18n.translation import _ -from pylons import request, config, session, tmpl_context as c +from pylons import request, config, tmpl_context as c from rhodecode.lib.auth import LoginRequired from rhodecode.lib.base import BaseController, render @@ -76,7 +76,7 @@ class SearchController(BaseController): cur_query = u'repository:%s %s' % (c.repo_name, cur_query) try: query = qp.parse(unicode(cur_query)) - + # extract words for highlight if isinstance(query, Phrase): highlight_items.update(query.words) elif isinstance(query, Prefix): @@ -92,18 +92,22 @@ class SearchController(BaseController): log.debug(highlight_items) results = searcher.search(query) res_ln = len(results) - c.runtime = '%s results (%.3f seconds)' \ - % (res_ln, results.runtime) + c.runtime = '%s results (%.3f seconds)' % ( + res_ln, results.runtime + ) def url_generator(**kw): return update_params("?q=%s&type=%s" \ % (c.cur_query, c.cur_search), **kw) c.formated_results = Page( - ResultWrapper(search_type, searcher, matcher, - highlight_items), - page=p, item_count=res_ln, - items_per_page=10, url=url_generator) + ResultWrapper(search_type, searcher, matcher, + highlight_items), + page=p, + item_count=res_ln, + items_per_page=10, + url=url_generator + ) except QueryParserError: c.runtime = _('Invalid search query. Try quoting it.') @@ -117,5 +121,6 @@ class SearchController(BaseController): log.error(traceback.format_exc()) c.runtime = _('An error occurred during this search operation') + # Return a rendered template return render('/search/search.html') diff --git a/rhodecode/lib/indexers/__init__.py b/rhodecode/lib/indexers/__init__.py --- a/rhodecode/lib/indexers/__init__.py +++ b/rhodecode/lib/indexers/__init__.py @@ -37,38 +37,39 @@ from whoosh.analysis import RegexTokeniz from whoosh.fields import TEXT, ID, STORED, Schema, FieldType from whoosh.index import create_in, open_dir from whoosh.formats import Characters -from whoosh.highlight import highlight, SimpleFragmenter, HtmlFormatter +from whoosh.highlight import highlight, HtmlFormatter, ContextFragmenter from webhelpers.html.builder import escape from sqlalchemy import engine_from_config -from vcs.utils.lazy import LazyProperty from rhodecode.model import init_model from rhodecode.model.scm import ScmModel from rhodecode.model.repo import RepoModel from rhodecode.config.environment import load_environment -from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP +from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP, LazyProperty from rhodecode.lib.utils import BasePasterCommand, Command, add_cache -#EXTENSIONS WE WANT TO INDEX CONTENT OFF +# EXTENSIONS WE WANT TO INDEX CONTENT OFF INDEX_EXTENSIONS = LANGUAGES_EXTENSIONS_MAP.keys() -#CUSTOM ANALYZER wordsplit + lowercase filter +# CUSTOM ANALYZER wordsplit + lowercase filter ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter() #INDEX SCHEMA DEFINITION -SCHEMA = Schema(owner=TEXT(), - repository=TEXT(stored=True), - path=TEXT(stored=True), - content=FieldType(format=Characters(ANALYZER), - scorable=True, stored=True), - modtime=STORED(), extension=TEXT(stored=True)) - +SCHEMA = Schema( + owner=TEXT(), + repository=TEXT(stored=True), + path=TEXT(stored=True), + content=FieldType(format=Characters(), analyzer=ANALYZER, + scorable=True, stored=True), + modtime=STORED(), + extension=TEXT(stored=True) +) IDX_NAME = 'HG_INDEX' FORMATTER = HtmlFormatter('span', between='\n...\n') -FRAGMENTER = SimpleFragmenter(200) +FRAGMENTER = ContextFragmenter(200) class MakeIndex(BasePasterCommand): @@ -136,7 +137,7 @@ class ResultWrapper(object): self.searcher = searcher self.matcher = matcher self.highlight_items = highlight_items - self.fragment_size = 200 / 2 + self.fragment_size = 200 @LazyProperty def doc_ids(self): @@ -172,10 +173,10 @@ class ResultWrapper(object): """ i, j = key.start, key.stop - slice = [] + slices = [] for docid in self.doc_ids[i:j]: - slice.append(self.get_full_content(docid)) - return slice + slices.append(self.get_full_content(docid)) + return slices def get_full_content(self, docid): res = self.searcher.stored_fields(docid[0]) @@ -183,9 +184,9 @@ class ResultWrapper(object): + len(res['repository']):].lstrip('/') content_short = self.get_short_content(res, docid[1]) - res.update({'content_short':content_short, - 'content_short_hl':self.highlight(content_short), - 'f_path':f_path}) + res.update({'content_short': content_short, + 'content_short_hl': self.highlight(content_short), + 'f_path': f_path}) return res @@ -217,10 +218,12 @@ class ResultWrapper(object): def highlight(self, content, top=5): if self.search_type != 'content': return '' - hl = highlight(escape(content), - self.highlight_items, - analyzer=ANALYZER, - fragmenter=FRAGMENTER, - formatter=FORMATTER, - top=top) + hl = highlight( + text=escape(content), + terms=self.highlight_items, + analyzer=ANALYZER, + fragmenter=FRAGMENTER, + formatter=FORMATTER, + top=top + ) return hl diff --git a/rhodecode/lib/indexers/daemon.py b/rhodecode/lib/indexers/daemon.py --- a/rhodecode/lib/indexers/daemon.py +++ b/rhodecode/lib/indexers/daemon.py @@ -49,7 +49,6 @@ from vcs.exceptions import ChangesetErro from whoosh.index import create_in, open_dir - log = logging.getLogger('whooshIndexer') # create logger log.setLevel(logging.DEBUG) @@ -68,12 +67,13 @@ ch.setFormatter(formatter) # add ch to logger log.addHandler(ch) + class WhooshIndexingDaemon(object): """ Daemon for atomic jobs """ - def __init__(self, indexname='HG_INDEX', index_location=None, + def __init__(self, indexname=IDX_NAME, index_location=None, repo_location=None, sa=None, repo_list=None): self.indexname = indexname @@ -95,7 +95,6 @@ class WhooshIndexingDaemon(object): self.repo_paths = filtered_repo_paths - self.initial = False if not os.path.isdir(self.index_location): os.makedirs(self.index_location) @@ -155,7 +154,6 @@ class WhooshIndexingDaemon(object): modtime=self.get_node_mtime(node), extension=node.extension) - def build_index(self): if os.path.exists(self.index_location): log.debug('removing previous index') @@ -177,7 +175,6 @@ class WhooshIndexingDaemon(object): writer.commit(merge=True) log.debug('>>> FINISHED BUILDING INDEX <<<') - def update_index(self): log.debug('STARTING INCREMENTAL INDEXING UPDATE') diff --git a/rhodecode/public/css/pygments.css b/rhodecode/public/css/pygments.css --- a/rhodecode/public/css/pygments.css +++ b/rhodecode/public/css/pygments.css @@ -65,10 +65,20 @@ div.codeblock .code-body table td { div.code-body { background-color: #FFFFFF; } -div.code-body pre .match{ + +div.codeblock .code-header .search-path { + padding: 0px 0px 0px 10px; +} + +div.search-code-body { + background-color: #FFFFFF; + padding: 5px 0px 5px 10px; +} + +div.search-code-body pre .match{ background-color: #FAFFA6; } -div.code-body pre .break{ +div.search-code-body pre .break{ background-color: #DDE7EF; width: 100%; color: #747474; diff --git a/rhodecode/templates/search/search_content.html b/rhodecode/templates/search/search_content.html --- a/rhodecode/templates/search/search_content.html +++ b/rhodecode/templates/search/search_content.html @@ -5,10 +5,11 @@
-
${h.link_to(h.literal('%s » %s' % (sr['repository'],sr['f_path'])), - h.url('files_home',repo_name=sr['repository'],revision='tip',f_path=sr['f_path']))}
+
${h.link_to(h.literal('%s » %s' % (sr['repository'],sr['f_path'])), + h.url('files_home',repo_name=sr['repository'],revision='tip',f_path=sr['f_path']))} +
-
+ 
${h.literal(sr['content_short_hl'])}