utils/conf

- created temporary utils2
- made config.conf for storing some configurations
- fixed some dependency import problems
- code cleanup
- rc-extensions now properly work for celery

rhodecode/config/conf.py

@@ -0,0 +1,58 b''
+# -*- coding: utf-8 -*-
+"""
+    package.rhodecode.config.conf
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Various config settings for RhodeCode
+
+    :created_on: Mar 7, 2012
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>
+    :license: <name>, see LICENSE_FILE for more details.
+"""
+from rhodecode import EXTENSIONS
+
+from rhodecode.lib.utils2 import __get_lem
+
+
+# language map is also used by whoosh indexer, which for those specified
+# extensions will index it's content
+LANGUAGES_EXTENSIONS_MAP = __get_lem()
+
+#==============================================================================
+# WHOOSH INDEX EXTENSIONS
+#==============================================================================
+# EXTENSIONS WE WANT TO INDEX CONTENT OFF USING WHOOSH
+INDEX_EXTENSIONS = LANGUAGES_EXTENSIONS_MAP.keys()
+
+# list of readme files to search in file tree and display in summary
+# attached weights defines the search  order lower is first
+ALL_READMES = [
+    ('readme', 0), ('README', 0), ('Readme', 0),
+    ('doc/readme', 1), ('doc/README', 1), ('doc/Readme', 1),
+    ('Docs/readme', 2), ('Docs/README', 2), ('Docs/Readme', 2),
+    ('DOCS/readme', 2), ('DOCS/README', 2), ('DOCS/Readme', 2),
+    ('docs/readme', 2), ('docs/README', 2), ('docs/Readme', 2),
+]
+
+# extension together with weights to search lower is first
+RST_EXTS = [
+    ('', 0), ('.rst', 1), ('.rest', 1),
+    ('.RST', 2), ('.REST', 2),
+    ('.txt', 3), ('.TXT', 3)
+]
+
+MARKDOWN_EXTS = [
+    ('.md', 1), ('.MD', 1),
+    ('.mkdn', 2), ('.MKDN', 2),
+    ('.mdown', 3), ('.MDOWN', 3),
+    ('.markdown', 4), ('.MARKDOWN', 4)
+]
+
+PLAIN_EXTS = [('.text', 2), ('.TEXT', 2)]
+
+ALL_EXTS = MARKDOWN_EXTS + RST_EXTS + PLAIN_EXTS
+
+DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S"
+
+DATE_FORMAT = "%Y-%m-%d"

rhodecode/lib/utils2.py

@@ -0,0 +1,405 b''
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.lib.utils
+    ~~~~~~~~~~~~~~~~~~~
+
+    Some simple helper functions
+
+    :created_on: Jan 5, 2011
+    :author: marcink
+    :copyright: (C) 2011-2012 Marcin Kuzminski <[e-mail]>
+    :license: GPLv3, see COPYING for more details.
+"""
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import re
+from rhodecode.lib.vcs.utils.lazy import LazyProperty
+
+
+def __get_lem():
+    """
+    Get language extension map based on what's inside pygments lexers
+    """
+    from pygments import lexers
+    from string import lower
+    from collections import defaultdict
+
+    d = defaultdict(lambda: [])
+
+    def __clean(s):
+        s = s.lstrip('*')
+        s = s.lstrip('.')
+
+        if s.find('[') != -1:
+            exts = []
+            start, stop = s.find('['), s.find(']')
+
+            for suffix in s[start + 1:stop]:
+                exts.append(s[:s.find('[')] + suffix)
+            return map(lower, exts)
+        else:
+            return map(lower, [s])
+
+    for lx, t in sorted(lexers.LEXERS.items()):
+        m = map(__clean, t[-2])
+        if m:
+            m = reduce(lambda x, y: x + y, m)
+            for ext in m:
+                desc = lx.replace('Lexer', '')
+                d[ext].append(desc)
+
+    return dict(d)
+
+def str2bool(_str):
+    """
+    returs True/False value from given string, it tries to translate the
+    string into boolean
+
+    :param _str: string value to translate into boolean
+    :rtype: boolean
+    :returns: boolean from given string
+    """
+    if _str is None:
+        return False
+    if _str in (True, False):
+        return _str
+    _str = str(_str).strip().lower()
+    return _str in ('t', 'true', 'y', 'yes', 'on', '1')
+
+
+def convert_line_endings(line, mode):
+    """
+    Converts a given line  "line end" accordingly to given mode
+
+    Available modes are::
+        0 - Unix
+        1 - Mac
+        2 - DOS
+
+    :param line: given line to convert
+    :param mode: mode to convert to
+    :rtype: str
+    :return: converted line according to mode
+    """
+    from string import replace
+
+    if mode == 0:
+            line = replace(line, '\r\n', '\n')
+            line = replace(line, '\r', '\n')
+    elif mode == 1:
+            line = replace(line, '\r\n', '\r')
+            line = replace(line, '\n', '\r')
+    elif mode == 2:
+            line = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", line)
+    return line
+
+
+def detect_mode(line, default):
+    """
+    Detects line break for given line, if line break couldn't be found
+    given default value is returned
+
+    :param line: str line
+    :param default: default
+    :rtype: int
+    :return: value of line end on of 0 - Unix, 1 - Mac, 2 - DOS
+    """
+    if line.endswith('\r\n'):
+        return 2
+    elif line.endswith('\n'):
+        return 0
+    elif line.endswith('\r'):
+        return 1
+    else:
+        return default
+
+
+def generate_api_key(username, salt=None):
+    """
+    Generates unique API key for given username, if salt is not given
+    it'll be generated from some random string
+
+    :param username: username as string
+    :param salt: salt to hash generate KEY
+    :rtype: str
+    :returns: sha1 hash from username+salt
+    """
+    from tempfile import _RandomNameSequence
+    import hashlib
+
+    if salt is None:
+        salt = _RandomNameSequence().next()
+
+    return hashlib.sha1(username + salt).hexdigest()
+
+
+def safe_unicode(str_, from_encoding=None):
+    """
+    safe unicode function. Does few trick to turn str_ into unicode
+
+    In case of UnicodeDecode error we try to return it with encoding detected
+    by chardet library if it fails fallback to unicode with errors replaced
+
+    :param str_: string to decode
+    :rtype: unicode
+    :returns: unicode object
+    """
+    if isinstance(str_, unicode):
+        return str_
+
+    if not from_encoding:
+        import rhodecode
+        DEFAULT_ENCODING = rhodecode.CONFIG.get('default_encoding','utf8')
+        from_encoding = DEFAULT_ENCODING
+
+    try:
+        return unicode(str_)
+    except UnicodeDecodeError:
+        pass
+
+    try:
+        return unicode(str_, from_encoding)
+    except UnicodeDecodeError:
+        pass
+
+    try:
+        import chardet
+        encoding = chardet.detect(str_)['encoding']
+        if encoding is None:
+            raise Exception()
+        return str_.decode(encoding)
+    except (ImportError, UnicodeDecodeError, Exception):
+        return unicode(str_, from_encoding, 'replace')
+
+
+def safe_str(unicode_, to_encoding=None):
+    """
+    safe str function. Does few trick to turn unicode_ into string
+
+    In case of UnicodeEncodeError we try to return it with encoding detected
+    by chardet library if it fails fallback to string with errors replaced
+
+    :param unicode_: unicode to encode
+    :rtype: str
+    :returns: str object
+    """
+
+    # if it's not basestr cast to str
+    if not isinstance(unicode_, basestring):
+        return str(unicode_)
+
+    if isinstance(unicode_, str):
+        return unicode_
+
+    if not to_encoding:
+        import rhodecode
+        DEFAULT_ENCODING = rhodecode.CONFIG.get('default_encoding','utf8')
+        to_encoding = DEFAULT_ENCODING
+
+    try:
+        return unicode_.encode(to_encoding)
+    except UnicodeEncodeError:
+        pass
+
+    try:
+        import chardet
+        encoding = chardet.detect(unicode_)['encoding']
+        print encoding
+        if encoding is None:
+            raise UnicodeEncodeError()
+
+        return unicode_.encode(encoding)
+    except (ImportError, UnicodeEncodeError):
+        return unicode_.encode(to_encoding, 'replace')
+
+    return safe_str
+
+
+def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs):
+    """
+    Custom engine_from_config functions that makes sure we use NullPool for
+    file based sqlite databases. This prevents errors on sqlite. This only
+    applies to sqlalchemy versions < 0.7.0
+
+    """
+    import sqlalchemy
+    from sqlalchemy import engine_from_config as efc
+    import logging
+
+    if int(sqlalchemy.__version__.split('.')[1]) < 7:
+
+        # This solution should work for sqlalchemy < 0.7.0, and should use
+        # proxy=TimerProxy() for execution time profiling
+
+        from sqlalchemy.pool import NullPool
+        url = configuration[prefix + 'url']
+
+        if url.startswith('sqlite'):
+            kwargs.update({'poolclass': NullPool})
+        return efc(configuration, prefix, **kwargs)
+    else:
+        import time
+        from sqlalchemy import event
+        from sqlalchemy.engine import Engine
+
+        log = logging.getLogger('sqlalchemy.engine')
+        BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = xrange(30, 38)
+        engine = efc(configuration, prefix, **kwargs)
+
+        def color_sql(sql):
+            COLOR_SEQ = "\033[1;%dm"
+            COLOR_SQL = YELLOW
+            normal = '\x1b[0m'
+            return ''.join([COLOR_SEQ % COLOR_SQL, sql, normal])
+
+        if configuration['debug']:
+            #attach events only for debug configuration
+
+            def before_cursor_execute(conn, cursor, statement,
+                                    parameters, context, executemany):
+                context._query_start_time = time.time()
+                log.info(color_sql(">>>>> STARTING QUERY >>>>>"))
+
+
+            def after_cursor_execute(conn, cursor, statement,
+                                    parameters, context, executemany):
+                total = time.time() - context._query_start_time
+                log.info(color_sql("<<<<< TOTAL TIME: %f <<<<<" % total))
+
+            event.listen(engine, "before_cursor_execute",
+                         before_cursor_execute)
+            event.listen(engine, "after_cursor_execute",
+                         after_cursor_execute)
+
+    return engine
+
+
+def age(curdate):
+    """
+    turns a datetime into an age string.
+
+    :param curdate: datetime object
+    :rtype: unicode
+    :returns: unicode words describing age
+    """
+
+    from datetime import datetime
+    from webhelpers.date import time_ago_in_words
+
+    _ = lambda s: s
+
+    if not curdate:
+        return ''
+
+    agescales = [(_(u"year"), 3600 * 24 * 365),
+                 (_(u"month"), 3600 * 24 * 30),
+                 (_(u"day"), 3600 * 24),
+                 (_(u"hour"), 3600),
+                 (_(u"minute"), 60),
+                 (_(u"second"), 1), ]
+
+    age = datetime.now() - curdate
+    age_seconds = (age.days * agescales[2][1]) + age.seconds
+    pos = 1
+    for scale in agescales:
+        if scale[1] <= age_seconds:
+            if pos == 6:
+                pos = 5
+            return '%s %s' % (time_ago_in_words(curdate,
+                                                agescales[pos][0]), _('ago'))
+        pos += 1
+
+    return _(u'just now')
+
+
+def uri_filter(uri):
+    """
+    Removes user:password from given url string
+
+    :param uri:
+    :rtype: unicode
+    :returns: filtered list of strings
+    """
+    if not uri:
+        return ''
+
+    proto = ''
+
+    for pat in ('https://', 'http://'):
+        if uri.startswith(pat):
+            uri = uri[len(pat):]
+            proto = pat
+            break
+
+    # remove passwords and username
+    uri = uri[uri.find('@') + 1:]
+
+    # get the port
+    cred_pos = uri.find(':')
+    if cred_pos == -1:
+        host, port = uri, None
+    else:
+        host, port = uri[:cred_pos], uri[cred_pos + 1:]
+
+    return filter(None, [proto, host, port])
+
+
+def credentials_filter(uri):
+    """
+    Returns a url with removed credentials
+
+    :param uri:
+    """
+
+    uri = uri_filter(uri)
+    #check if we have port
+    if len(uri) > 2 and uri[2]:
+        uri[2] = ':' + uri[2]
+
+    return ''.join(uri)
+
+
+def get_changeset_safe(repo, rev):
+    """
+    Safe version of get_changeset if this changeset doesn't exists for a
+    repo it returns a Dummy one instead
+
+    :param repo:
+    :param rev:
+    """
+    from rhodecode.lib.vcs.backends.base import BaseRepository
+    from rhodecode.lib.vcs.exceptions import RepositoryError
+    if not isinstance(repo, BaseRepository):
+        raise Exception('You must pass an Repository '
+                        'object as first argument got %s', type(repo))
+
+    try:
+        cs = repo.get_changeset(rev)
+    except RepositoryError:
+        from rhodecode.lib.utils import EmptyChangeset
+        cs = EmptyChangeset(requested_revision=rev)
+    return cs
+
+
+def extract_mentioned_users(s):
+    """
+    Returns unique usernames from given string s that have @mention
+
+    :param s: string to get mentions
+    """
+    usrs = {}
+    for username in re.findall(r'(?:^@|\s@)(\w+)', s):
+        usrs[username] = username
+
+    return sorted(usrs.keys())

rhodecode/config/environment.py

@@ -2,40 +2,45 b''
 
 import os
 import logging
+import rhodecode
 
 from mako.lookup import TemplateLookup
 from pylons.configuration import PylonsConfig
 from pylons.error import handle_mako_error
 
-import rhodecode
+# don't remove this import it does magic for celery
+from rhodecode.lib import celerypylons
+
 import rhodecode.lib.app_globals as app_globals
-import rhodecode.lib.helpers
 
 from rhodecode.config.routing import make_map
-# don't remove this import it does magic for celery
-from rhodecode.lib import celerypylons, str2bool
-from rhodecode.lib import engine_from_config
+
+from rhodecode.lib import helpers
 from rhodecode.lib.auth import set_available_permissions
-from rhodecode.lib.utils import repo2db_mapper, make_ui, set_rhodecode_config
+from rhodecode.lib.utils import repo2db_mapper, make_ui, set_rhodecode_config,\
+    load_rcextensions
+from rhodecode.lib.utils2 import engine_from_config, str2bool
 from rhodecode.model import init_model
 from rhodecode.model.scm import ScmModel
-from rhodecode.lib.vcs.utils.fakemod import create_module
 
 log = logging.getLogger(__name__)
 
 
 def load_environment(global_conf, app_conf, initial=False):
-    """Configure the Pylons environment via the ``pylons.config``
+    """
+    Configure the Pylons environment via the ``pylons.config``
     object
     """
     config = PylonsConfig()
 
     # Pylons paths
     root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-    paths = dict(root=root,
-                 controllers=os.path.join(root, 'controllers'),
-                 static_files=os.path.join(root, 'public'),
-                 templates=[os.path.join(root, 'templates')])
+    paths = dict(
+        root=root,
+        controllers=os.path.join(root, 'controllers'),
+        static_files=os.path.join(root, 'public'),
+        templates=[os.path.join(root, 'templates')]
+    )
 
     # Initialize config with the basic options
     config.init_app(global_conf, app_conf, package='rhodecode', paths=paths)
@@ -45,14 +50,11 b' def load_environment(global_conf, app_co'
 
     config['routes.map'] = make_map(config)
     config['pylons.app_globals'] = app_globals.Globals(config)
-    config['pylons.h'] = rhodecode.lib.helpers
+    config['pylons.h'] = helpers
     rhodecode.CONFIG = config
 
-    path = os.path.join(config['here'], 'rcextensions', '__init__.py')
-    if os.path.isfile(path):
-        rcext = create_module('rc', path)
-        rhodecode.EXTENSIONS = rcext
-        log.debug('Found rcextensions now loading %s...' % rcext)
+    load_rcextensions(root_path=config['here'])
+
     # Setup cache object as early as possible
     import pylons
     pylons.cache._push_object(config['pylons.app_globals'].cache)

rhodecode/config/rcextensions/make_rcextensions.py

@@ -77,5 +77,3 b' class MakeRcExt(BasePasterCommand):'
 
     def update_parser(self):
         pass
-
-

rhodecode/controllers/admin/users_groups.py

@@ -32,8 +32,9 b' from pylons import request, session, tmp'
 from pylons.controllers.util import abort, redirect
 from pylons.i18n.translation import _
 
+from rhodecode.lib import helpers as h
 from rhodecode.lib.exceptions import UsersGroupsAssignedException
-from rhodecode.lib import helpers as h, safe_unicode
+from rhodecode.lib.utils2 import safe_unicode
 from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator
 from rhodecode.lib.base import BaseController, render
 

rhodecode/controllers/branches.py

@@ -31,7 +31,7 b' import binascii'
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseRepoController, render
 from rhodecode.lib.compat import OrderedDict
-from rhodecode.lib import safe_unicode
+from rhodecode.lib.utils2 import safe_unicode
 log = logging.getLogger(__name__)
 
 

rhodecode/controllers/feed.py

@@ -28,7 +28,7 b' import logging'
 from pylons import url, response, tmpl_context as c
 from pylons.i18n.translation import _
 
-from rhodecode.lib import safe_unicode
+from rhodecode.lib.utils2 import safe_unicode
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseRepoController
 

rhodecode/controllers/files.py

@@ -32,24 +32,26 b' from pylons.i18n.translation import _'
 from pylons.controllers.util import redirect
 from pylons.decorators import jsonify
 
-from rhodecode.lib.vcs.conf import settings
-from rhodecode.lib.vcs.exceptions import RepositoryError, ChangesetDoesNotExistError, \
-    EmptyRepositoryError, ImproperArchiveTypeError, VCSError, \
-    NodeAlreadyExistsError
-from rhodecode.lib.vcs.nodes import FileNode
+from rhodecode.lib import diffs
+from rhodecode.lib import helpers as h
 
 from rhodecode.lib.compat import OrderedDict
-from rhodecode.lib import convert_line_endings, detect_mode, safe_str
+from rhodecode.lib.utils2 import convert_line_endings, detect_mode, safe_str
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseRepoController, render
 from rhodecode.lib.utils import EmptyChangeset
-from rhodecode.lib import diffs
-import rhodecode.lib.helpers as h
+from rhodecode.lib.vcs.conf import settings
+from rhodecode.lib.vcs.exceptions import RepositoryError, \
+    ChangesetDoesNotExistError, EmptyRepositoryError, \
+    ImproperArchiveTypeError, VCSError, NodeAlreadyExistsError
+from rhodecode.lib.vcs.nodes import FileNode
+
 from rhodecode.model.repo import RepoModel
+from rhodecode.model.scm import ScmModel
+
 from rhodecode.controllers.changeset import anchor_url, _ignorews_url,\
     _context_url, get_line_ctx, get_ignore_ws
-from rhodecode.lib.diffs import wrapped_diff
-from rhodecode.model.scm import ScmModel
+
 
 log = logging.getLogger(__name__)
 
@@ -447,7 +449,7 b' class FilesController(BaseRepoController'
             ign_whitespace_lcl = get_ignore_ws(fid, request.GET)
 
             lim = request.GET.get('fulldiff') or self.cut_off_limit
-            _, cs1, cs2, diff, st = wrapped_diff(filenode_old=node1,
+            _, cs1, cs2, diff, st = diffs.wrapped_diff(filenode_old=node1,
                                          filenode_new=node2,
                                          cut_off_limit=lim,
                                          ignore_whitespace=ign_whitespace_lcl,

rhodecode/controllers/summary.py

@@ -40,15 +40,15 b' from pylons.i18n.translation import _'
 
 from beaker.cache import cache_region, region_invalidate
 
+from rhodecode.config.conf import ALL_READMES, ALL_EXTS, LANGUAGES_EXTENSIONS_MAP
 from rhodecode.model.db import Statistics, CacheInvalidation
-from rhodecode.lib import ALL_READMES, ALL_EXTS, safe_unicode
+from rhodecode.lib.utils2 import safe_unicode
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseRepoController, render
 from rhodecode.lib.utils import EmptyChangeset
 from rhodecode.lib.markup_renderer import MarkupRenderer
 from rhodecode.lib.celerylib import run_task
-from rhodecode.lib.celerylib.tasks import get_commits_stats, \
-    LANGUAGES_EXTENSIONS_MAP
+from rhodecode.lib.celerylib.tasks import get_commits_stats
 from rhodecode.lib.helpers import RepoPage
 from rhodecode.lib.compat import json, OrderedDict
 

rhodecode/lib/__init__.py

@@ -1,444 +1,4 b''
-# -*- coding: utf-8 -*-
-"""
-    rhodecode.lib.__init__
-    ~~~~~~~~~~~~~~~~~~~~~~~
-
-    Some simple helper functions
-
-    :created_on: Jan 5, 2011
-    :author: marcink
-    :copyright: (C) 2011-2012 Marcin Kuzminski <[e-mail]>
-    :license: GPLv3, see COPYING for more details.
-"""
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 import os
-import re
-from rhodecode import EXTENSIONS
-from rhodecode.lib.vcs.utils.lazy import LazyProperty
-
-
-def __get_lem():
-    from pygments import lexers
-    from string import lower
-    from collections import defaultdict
-
-    d = defaultdict(lambda: [])
-
-    def __clean(s):
-        s = s.lstrip('*')
-        s = s.lstrip('.')
-
-        if s.find('[') != -1:
-            exts = []
-            start, stop = s.find('['), s.find(']')
-
-            for suffix in s[start + 1:stop]:
-                exts.append(s[:s.find('[')] + suffix)
-            return map(lower, exts)
-        else:
-            return map(lower, [s])
-
-    for lx, t in sorted(lexers.LEXERS.items()):
-        m = map(__clean, t[-2])
-        if m:
-            m = reduce(lambda x, y: x + y, m)
-            for ext in m:
-                desc = lx.replace('Lexer', '')
-                d[ext].append(desc)
-
-    return dict(d)
-
-# language map is also used by whoosh indexer, which for those specified
-# extensions will index it's content
-LANGUAGES_EXTENSIONS_MAP = __get_lem()
-
-# Additional mappings that are not present in the pygments lexers
-LANGUAGES_EXTENSIONS_MAP.update(getattr(EXTENSIONS, 'EXTRA_MAPPINGS', {}))
-
-#==============================================================================
-# WHOOSH INDEX EXTENSIONS
-#==============================================================================
-# EXTENSIONS WE WANT TO INDEX CONTENT OFF USING WHOOSH
-INDEX_EXTENSIONS = LANGUAGES_EXTENSIONS_MAP.keys()
-
-#OVERRIDE OUR EXTENSIONS FROM RC-EXTENSIONS (if present)
-
-if getattr(EXTENSIONS, 'INDEX_EXTENSIONS', []) != []:
-    INDEX_EXTENSIONS = getattr(EXTENSIONS, 'INDEX_EXTENSIONS', [])
-
-#ADDITIONAL MAPPINGS
-INDEX_EXTENSIONS.extend(getattr(EXTENSIONS, 'EXTRA_INDEX_EXTENSIONS', []))
-
-# list of readme files to search in file tree and display in summary
-# attached weights defines the search  order lower is first
-ALL_READMES = [
-    ('readme', 0), ('README', 0), ('Readme', 0),
-    ('doc/readme', 1), ('doc/README', 1), ('doc/Readme', 1),
-    ('Docs/readme', 2), ('Docs/README', 2), ('Docs/Readme', 2),
-    ('DOCS/readme', 2), ('DOCS/README', 2), ('DOCS/Readme', 2),
-    ('docs/readme', 2), ('docs/README', 2), ('docs/Readme', 2),
-]
-
-# extension together with weights to search lower is first
-RST_EXTS = [
-    ('', 0), ('.rst', 1), ('.rest', 1),
-    ('.RST', 2), ('.REST', 2),
-    ('.txt', 3), ('.TXT', 3)
-]
-
-MARKDOWN_EXTS = [
-    ('.md', 1), ('.MD', 1),
-    ('.mkdn', 2), ('.MKDN', 2),
-    ('.mdown', 3), ('.MDOWN', 3),
-    ('.markdown', 4), ('.MARKDOWN', 4)
-]
-
-PLAIN_EXTS = [('.text', 2), ('.TEXT', 2)]
-
-ALL_EXTS = MARKDOWN_EXTS + RST_EXTS + PLAIN_EXTS
-
-
-def str2bool(_str):
-    """
-    returs True/False value from given string, it tries to translate the
-    string into boolean
-
-    :param _str: string value to translate into boolean
-    :rtype: boolean
-    :returns: boolean from given string
-    """
-    if _str is None:
-        return False
-    if _str in (True, False):
-        return _str
-    _str = str(_str).strip().lower()
-    return _str in ('t', 'true', 'y', 'yes', 'on', '1')
-
-
-def convert_line_endings(line, mode):
-    """
-    Converts a given line  "line end" accordingly to given mode
-
-    Available modes are::
-        0 - Unix
-        1 - Mac
-        2 - DOS
-
-    :param line: given line to convert
-    :param mode: mode to convert to
-    :rtype: str
-    :return: converted line according to mode
-    """
-    from string import replace
-
-    if mode == 0:
-            line = replace(line, '\r\n', '\n')
-            line = replace(line, '\r', '\n')
-    elif mode == 1:
-            line = replace(line, '\r\n', '\r')
-            line = replace(line, '\n', '\r')
-    elif mode == 2:
-            line = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", line)
-    return line
-
-
-def detect_mode(line, default):
-    """
-    Detects line break for given line, if line break couldn't be found
-    given default value is returned
-
-    :param line: str line
-    :param default: default
-    :rtype: int
-    :return: value of line end on of 0 - Unix, 1 - Mac, 2 - DOS
-    """
-    if line.endswith('\r\n'):
-        return 2
-    elif line.endswith('\n'):
-        return 0
-    elif line.endswith('\r'):
-        return 1
-    else:
-        return default
-
-
-def generate_api_key(username, salt=None):
-    """
-    Generates unique API key for given username, if salt is not given
-    it'll be generated from some random string
-
-    :param username: username as string
-    :param salt: salt to hash generate KEY
-    :rtype: str
-    :returns: sha1 hash from username+salt
-    """
-    from tempfile import _RandomNameSequence
-    import hashlib
-
-    if salt is None:
-        salt = _RandomNameSequence().next()
-
-    return hashlib.sha1(username + salt).hexdigest()
-
-
-def safe_unicode(str_, from_encoding=None):
-    """
-    safe unicode function. Does few trick to turn str_ into unicode
-
-    In case of UnicodeDecode error we try to return it with encoding detected
-    by chardet library if it fails fallback to unicode with errors replaced
-
-    :param str_: string to decode
-    :rtype: unicode
-    :returns: unicode object
-    """
-    if isinstance(str_, unicode):
-        return str_
-
-    if not from_encoding:
-        import rhodecode
-        DEFAULT_ENCODING = rhodecode.CONFIG.get('default_encoding','utf8')
-        from_encoding = DEFAULT_ENCODING
-
-    try:
-        return unicode(str_)
-    except UnicodeDecodeError:
-        pass
-
-    try:
-        return unicode(str_, from_encoding)
-    except UnicodeDecodeError:
-        pass
-
-    try:
-        import chardet
-        encoding = chardet.detect(str_)['encoding']
-        if encoding is None:
-            raise Exception()
-        return str_.decode(encoding)
-    except (ImportError, UnicodeDecodeError, Exception):
-        return unicode(str_, from_encoding, 'replace')
-
-
-def safe_str(unicode_, to_encoding=None):
-    """
-    safe str function. Does few trick to turn unicode_ into string
-
-    In case of UnicodeEncodeError we try to return it with encoding detected
-    by chardet library if it fails fallback to string with errors replaced
-
-    :param unicode_: unicode to encode
-    :rtype: str
-    :returns: str object
-    """
-
-    # if it's not basestr cast to str
-    if not isinstance(unicode_, basestring):
-        return str(unicode_)
-
-    if isinstance(unicode_, str):
-        return unicode_
-
-    if not to_encoding:
-        import rhodecode
-        DEFAULT_ENCODING = rhodecode.CONFIG.get('default_encoding','utf8')
-        to_encoding = DEFAULT_ENCODING
-
-    try:
-        return unicode_.encode(to_encoding)
-    except UnicodeEncodeError:
-        pass
-
-    try:
-        import chardet
-        encoding = chardet.detect(unicode_)['encoding']
-        print encoding
-        if encoding is None:
-            raise UnicodeEncodeError()
-
-        return unicode_.encode(encoding)
-    except (ImportError, UnicodeEncodeError):
-        return unicode_.encode(to_encoding, 'replace')
-
-    return safe_str
-
-
-def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs):
-    """
-    Custom engine_from_config functions that makes sure we use NullPool for
-    file based sqlite databases. This prevents errors on sqlite. This only
-    applies to sqlalchemy versions < 0.7.0
-
-    """
-    import sqlalchemy
-    from sqlalchemy import engine_from_config as efc
-    import logging
-
-    if int(sqlalchemy.__version__.split('.')[1]) < 7:
-
-        # This solution should work for sqlalchemy < 0.7.0, and should use
-        # proxy=TimerProxy() for execution time profiling
-
-        from sqlalchemy.pool import NullPool
-        url = configuration[prefix + 'url']
-
-        if url.startswith('sqlite'):
-            kwargs.update({'poolclass': NullPool})
-        return efc(configuration, prefix, **kwargs)
-    else:
-        import time
-        from sqlalchemy import event
-        from sqlalchemy.engine import Engine
-
-        log = logging.getLogger('sqlalchemy.engine')
-        BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = xrange(30, 38)
-        engine = efc(configuration, prefix, **kwargs)
-
-        def color_sql(sql):
-            COLOR_SEQ = "\033[1;%dm"
-            COLOR_SQL = YELLOW
-            normal = '\x1b[0m'
-            return ''.join([COLOR_SEQ % COLOR_SQL, sql, normal])
-
-        if configuration['debug']:
-            #attach events only for debug configuration
-
-            def before_cursor_execute(conn, cursor, statement,
-                                    parameters, context, executemany):
-                context._query_start_time = time.time()
-                log.info(color_sql(">>>>> STARTING QUERY >>>>>"))
-
-
-            def after_cursor_execute(conn, cursor, statement,
-                                    parameters, context, executemany):
-                total = time.time() - context._query_start_time
-                log.info(color_sql("<<<<< TOTAL TIME: %f <<<<<" % total))
-
-            event.listen(engine, "before_cursor_execute",
-                         before_cursor_execute)
-            event.listen(engine, "after_cursor_execute",
-                         after_cursor_execute)
-
-    return engine
-
-
-def age(curdate):
-    """
-    turns a datetime into an age string.
-
-    :param curdate: datetime object
-    :rtype: unicode
-    :returns: unicode words describing age
-    """
-
-    from datetime import datetime
-    from webhelpers.date import time_ago_in_words
-
-    _ = lambda s: s
-
-    if not curdate:
-        return ''
-
-    agescales = [(_(u"year"), 3600 * 24 * 365),
-                 (_(u"month"), 3600 * 24 * 30),
-                 (_(u"day"), 3600 * 24),
-                 (_(u"hour"), 3600),
-                 (_(u"minute"), 60),
-                 (_(u"second"), 1), ]
-
-    age = datetime.now() - curdate
-    age_seconds = (age.days * agescales[2][1]) + age.seconds
-    pos = 1
-    for scale in agescales:
-        if scale[1] <= age_seconds:
-            if pos == 6:
-                pos = 5
-            return '%s %s' % (time_ago_in_words(curdate,
-                                                agescales[pos][0]), _('ago'))
-        pos += 1
-
-    return _(u'just now')
-
-
-def uri_filter(uri):
-    """
-    Removes user:password from given url string
-
-    :param uri:
-    :rtype: unicode
-    :returns: filtered list of strings
-    """
-    if not uri:
-        return ''
-
-    proto = ''
-
-    for pat in ('https://', 'http://'):
-        if uri.startswith(pat):
-            uri = uri[len(pat):]
-            proto = pat
-            break
-
-    # remove passwords and username
-    uri = uri[uri.find('@') + 1:]
-
-    # get the port
-    cred_pos = uri.find(':')
-    if cred_pos == -1:
-        host, port = uri, None
-    else:
-        host, port = uri[:cred_pos], uri[cred_pos + 1:]
-
-    return filter(None, [proto, host, port])
-
-
-def credentials_filter(uri):
-    """
-    Returns a url with removed credentials
-
-    :param uri:
-    """
-
-    uri = uri_filter(uri)
-    #check if we have port
-    if len(uri) > 2 and uri[2]:
-        uri[2] = ':' + uri[2]
-
-    return ''.join(uri)
-
-
-def get_changeset_safe(repo, rev):
-    """
-    Safe version of get_changeset if this changeset doesn't exists for a
-    repo it returns a Dummy one instead
-
-    :param repo:
-    :param rev:
-    """
-    from rhodecode.lib.vcs.backends.base import BaseRepository
-    from rhodecode.lib.vcs.exceptions import RepositoryError
-    if not isinstance(repo, BaseRepository):
-        raise Exception('You must pass an Repository '
-                        'object as first argument got %s', type(repo))
-
-    try:
-        cs = repo.get_changeset(rev)
-    except RepositoryError:
-        from rhodecode.lib.utils import EmptyChangeset
-        cs = EmptyChangeset(requested_revision=rev)
-    return cs
 
 
 def get_current_revision(quiet=False):
@@ -462,16 +22,3 b' def get_current_revision(quiet=False):'
             print ("Cannot retrieve rhodecode's revision. Original error "
                    "was: %s" % err)
         return None
-
-
-def extract_mentioned_users(s):
-    """
-    Returns unique usernames from given string s that have @mention
-
-    :param s: string to get mentions
-    """
-    usrs = {}
-    for username in re.findall(r'(?:^@|\s@)(\w+)', s):
-        usrs[username] = username
-
-    return sorted(usrs.keys())

rhodecode/lib/auth.py

@@ -43,7 +43,7 b' if __platform__ in PLATFORM_WIN:'
 if __platform__ in PLATFORM_OTHERS:
     import bcrypt
 
-from rhodecode.lib import str2bool, safe_unicode
+from rhodecode.lib.utils2 import str2bool, safe_unicode
 from rhodecode.lib.exceptions import LdapPasswordError, LdapUsernameError
 from rhodecode.lib.utils import get_repo_slug, get_repos_group_slug
 from rhodecode.lib.auth_ldap import AuthLdap
@@ -795,7 +795,7 b' class HasPermissionAnyMiddleware(object)'
         try:
             self.user_perms = set([usr.permissions['repositories'][repo_name]])
         except Exception:
-            log.error('Exception while accessing permissions %s' % 
+            log.error('Exception while accessing permissions %s' %
                       traceback.format_exc())
             self.user_perms = set()
         self.granted_for = ''

rhodecode/lib/base.py

@@ -15,7 +15,7 b' from pylons.templating import render_mak'
 
 from rhodecode import __version__, BACKENDS
 
-from rhodecode.lib import str2bool, safe_unicode
+from rhodecode.lib.utils2 import str2bool, safe_unicode
 from rhodecode.lib.auth import AuthUser, get_container_username, authfunc,\
     HasPermissionAnyMiddleware, CookieStoreWrapper
 from rhodecode.lib.utils import get_repo_slug, invalidate_cache

rhodecode/lib/caching_query.py

@@ -24,7 +24,7 b' from beaker.exceptions import BeakerExce'
 from sqlalchemy.orm.interfaces import MapperOption
 from sqlalchemy.orm.query import Query
 from sqlalchemy.sql import visitors
-from rhodecode.lib import safe_str
+from rhodecode.lib.utils2 import safe_str
 
 
 class CachingQuery(Query):

rhodecode/lib/celerylib/__init__.py

@@ -36,7 +36,7 b' from decorator import decorator'
 
 from rhodecode.lib.vcs.utils.lazy import LazyProperty
 from rhodecode import CELERY_ON
-from rhodecode.lib import str2bool, safe_str
+from rhodecode.lib.utils2 import str2bool, safe_str
 from rhodecode.lib.pidlock import DaemonLock, LockHeld
 from rhodecode.model import init_model
 from rhodecode.model import meta

rhodecode/lib/celerylib/tasks.py

@@ -40,7 +40,7 b' from pylons.i18n.translation import _'
 from rhodecode.lib.vcs import get_backend
 
 from rhodecode import CELERY_ON
-from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP, safe_str
+from rhodecode.lib.utils2 import safe_str
 from rhodecode.lib.celerylib import run_task, locked_task, dbsession, \
     str2bool, __get_lockkey, LockHeld, DaemonLock, get_session
 from rhodecode.lib.helpers import person
@@ -147,6 +147,7 b' def get_commits_stats(repo_name, ts_min_'
              last_rev, last_rev + parse_limit)
         )
         for cs in repo[last_rev:last_rev + parse_limit]:
+            log.debug('parsing %s' % cs)
             last_cs = cs  # remember last parsed changeset
             k = lmktime([cs.date.timetuple()[0], cs.date.timetuple()[1],
                           cs.date.timetuple()[2], 0, 0, 0, 0, 0, 0])
@@ -233,10 +234,10 b' def get_commits_stats(repo_name, ts_min_'
             lock.release()
             return False
 
-        #final release
+        # final release
         lock.release()
 
-        #execute another task if celery is enabled
+        # execute another task if celery is enabled
         if len(repo.revisions) > 1 and CELERY_ON:
             run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y)
         return True
@@ -395,6 +396,7 b' def create_repo_fork(form_data, cur_user'
     DBS.commit()
 
 def __get_codes_stats(repo_name):
+    from rhodecode.config.conf import  LANGUAGES_EXTENSIONS_MAP
     repo = Repository.get_by_repo_name(repo_name).scm_instance
 
     tip = repo.get_changeset()

rhodecode/lib/celerypylons/commands.py

@@ -1,9 +1,9 b''
 import rhodecode
-from rhodecode.lib.utils import BasePasterCommand, Command
+from rhodecode.lib.utils import BasePasterCommand, Command, load_rcextensions
 from celery.app import app_or_default
 from celery.bin import camqadm, celerybeat, celeryd, celeryev
 
-from rhodecode.lib import str2bool
+from rhodecode.lib.utils2 import str2bool
 
 __all__ = ['CeleryDaemonCommand', 'CeleryBeatCommand',
            'CAMQPAdminCommand', 'CeleryEventCommand']
@@ -39,9 +39,11 b' class CeleryCommand(BasePasterCommand):'
             raise Exception('Please enable celery_on in .ini config '
                             'file before running celeryd')
         rhodecode.CELERY_ON = CELERY_ON
+        load_rcextensions(config['here'])
         cmd = self.celery_command(app_or_default())
         return cmd.run(**vars(self.options))
 
+
 class CeleryDaemonCommand(CeleryCommand):
     """Start the celery worker
 
@@ -82,6 +84,7 b' class CAMQPAdminCommand(CeleryCommand):'
     parser = Command.standard_parser(quiet=True)
     celery_command = camqadm.AMQPAdminCommand
 
+
 class CeleryEventCommand(CeleryCommand):
     """Celery event command.
 

rhodecode/lib/dbmigrate/migrate/exceptions.py

@@ -71,9 +71,6 b' class InvalidScriptError(ScriptError):'
     """Invalid script error."""
 
 
-class InvalidVersionError(Error):
-    """Invalid version error."""
-
 # migrate.changeset
 
 class NotSupportedError(Error):

rhodecode/lib/dbmigrate/schema/db_1_2_0.py

@@ -39,7 +39,7 b' from rhodecode.lib.vcs.utils.helpers imp'
 from rhodecode.lib.vcs.exceptions import VCSError
 from rhodecode.lib.vcs.utils.lazy import LazyProperty
 
-from rhodecode.lib import str2bool, safe_str, get_changeset_safe, \
+from rhodecode.lib.utils2 import str2bool, safe_str, get_changeset_safe, \
     generate_api_key, safe_unicode
 from rhodecode.lib.exceptions import UsersGroupsAssignedException
 from rhodecode.lib.compat import json
@@ -717,7 +717,7 b' class Repository(Base, BaseModel):'
         return repo
 
 
-class RepoGroup(Base, BaseModel):
+class Group(Base, BaseModel):
     __tablename__ = 'groups'
     __table_args__ = (UniqueConstraint('group_name', 'group_parent_id'),
                       CheckConstraint('group_id != group_parent_id'), {'extend_existing':True},)
@@ -728,8 +728,7 b' class RepoGroup(Base, BaseModel):'
     group_parent_id = Column("group_parent_id", Integer(), ForeignKey('groups.group_id'), nullable=True, unique=None, default=None)
     group_description = Column("group_description", String(length=10000, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
-    parent_group = relationship('RepoGroup', remote_side=group_id)
-
+    parent_group = relationship('Group', remote_side=group_id)
 
     def __init__(self, group_name='', parent_group=None):
         self.group_name = group_name

rhodecode/lib/helpers.py

@@ -39,7 +39,8 b' from webhelpers.html.tags import _set_in'
 
 from rhodecode.lib.annotate import annotate_highlight
 from rhodecode.lib.utils import repo_name_slug
-from rhodecode.lib import str2bool, safe_unicode, safe_str, get_changeset_safe
+from rhodecode.lib.utils2 import str2bool, safe_unicode, safe_str, \
+    get_changeset_safe
 from rhodecode.lib.markup_renderer import MarkupRenderer
 
 log = logging.getLogger(__name__)
@@ -319,7 +320,7 b' flash = _Flash()'
 # SCM FILTERS available via h.
 #==============================================================================
 from rhodecode.lib.vcs.utils import author_name, author_email
-from rhodecode.lib import credentials_filter, age as _age
+from rhodecode.lib.utils2 import credentials_filter, age as _age
 from rhodecode.model.db import User
 
 age = lambda  x: _age(x)

rhodecode/lib/hooks.py

@@ -139,7 +139,7 b' def log_push_action(ui, repo, **kwargs):'
 def log_create_repository(repository_dict, created_by, **kwargs):
     """
     Post create repository Hook. This is a dummy function for admins to re-use
-    if needed. It's taken from rhodecode-extensions module and executed 
+    if needed. It's taken from rhodecode-extensions module and executed
     if present
 
     :param repository: dict dump of repository object

rhodecode/lib/indexers/__init__.py

@@ -47,9 +47,9 b' from rhodecode.model import init_model'
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.repo import RepoModel
 from rhodecode.config.environment import load_environment
-from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP, INDEX_EXTENSIONS, \
-    LazyProperty
-from rhodecode.lib.utils import BasePasterCommand, Command, add_cache
+from rhodecode.lib.utils2 import LazyProperty
+from rhodecode.lib.utils import BasePasterCommand, Command, add_cache,\
+    load_rcextensions
 
 # CUSTOM ANALYZER wordsplit + lowercase filter
 ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter()
@@ -88,13 +88,12 b' class MakeIndex(BasePasterCommand):'
         add_cache(config)
         engine = engine_from_config(config, 'sqlalchemy.db1.')
         init_model(engine)
-
         index_location = config['index_dir']
         repo_location = self.options.repo_location \
             if self.options.repo_location else RepoModel().repos_path
         repo_list = map(strip, self.options.repo_list.split(',')) \
             if self.options.repo_list else None
-
+        load_rcextensions(config['here'])
         #======================================================================
         # WHOOSH DAEMON
         #======================================================================
@@ -104,7 +103,7 b' class MakeIndex(BasePasterCommand):'
             l = DaemonLock(file_=jn(dn(dn(index_location)), 'make_index.lock'))
             WhooshIndexingDaemon(index_location=index_location,
                                  repo_location=repo_location,
-                                 repo_list=repo_list)\
+                                 repo_list=repo_list,)\
                 .run(full_index=self.options.full_index)
             l.release()
         except LockHeld:

rhodecode/lib/indexers/daemon.py

@@ -38,10 +38,10 b' from os.path import join as jn'
 project_path = dn(dn(dn(dn(os.path.realpath(__file__)))))
 sys.path.append(project_path)
 
-
+from rhodecode.config.conf import INDEX_EXTENSIONS
 from rhodecode.model.scm import ScmModel
-from rhodecode.lib import safe_unicode
-from rhodecode.lib.indexers import INDEX_EXTENSIONS, SCHEMA, IDX_NAME
+from rhodecode.lib.utils2 import safe_unicode
+from rhodecode.lib.indexers import SCHEMA, IDX_NAME
 
 from rhodecode.lib.vcs.exceptions import ChangesetError, RepositoryError, \
     NodeDoesNotExistError
@@ -117,10 +117,9 b' class WhooshIndexingDaemon(object):'
         """
 
         node = self.get_node(repo, path)
-
+        indexed = indexed_w_content = 0
         # we just index the content of chosen files, and skip binary files
         if node.extension in INDEX_EXTENSIONS and not node.is_binary:
-
             u_content = node.content
             if not isinstance(u_content, unicode):
                 log.warning('  >> %s Could not get this content as unicode '
@@ -128,11 +127,13 b' class WhooshIndexingDaemon(object):'
                 u_content = u''
             else:
                 log.debug('    >> %s [WITH CONTENT]' % path)
+                indexed_w_content += 1
 
         else:
             log.debug('    >> %s' % path)
             # just index file name without it's content
             u_content = u''
+            indexed += 1
 
         writer.add_document(
             owner=unicode(repo.contact),
@@ -142,6 +143,7 b' class WhooshIndexingDaemon(object):'
             modtime=self.get_node_mtime(node),
             extension=node.extension
         )
+        return indexed, indexed_w_content
 
     def build_index(self):
         if os.path.exists(self.index_location):
@@ -153,19 +155,25 b' class WhooshIndexingDaemon(object):'
 
         idx = create_in(self.index_location, SCHEMA, indexname=IDX_NAME)
         writer = idx.writer()
-
+        log.debug('BUILDIN INDEX FOR EXTENSIONS %s' % INDEX_EXTENSIONS)
         for repo_name, repo in self.repo_paths.items():
             log.debug('building index @ %s' % repo.path)
-
+            i_cnt = iwc_cnt = 0
             for idx_path in self.get_paths(repo):
-                self.add_doc(writer, idx_path, repo, repo_name)
+                i, iwc = self.add_doc(writer, idx_path, repo, repo_name)
+                i_cnt += i
+                iwc_cnt += iwc
+            log.debug('added %s files %s with content for repo %s' % (
+                         i_cnt + iwc_cnt, iwc_cnt, repo.path)
+            )
 
         log.debug('>> COMMITING CHANGES <<')
         writer.commit(merge=True)
         log.debug('>>> FINISHED BUILDING INDEX <<<')
 
     def update_index(self):
-        log.debug('STARTING INCREMENTAL INDEXING UPDATE')
+        log.debug('STARTING INCREMENTAL INDEXING UPDATE FOR EXTENSIONS %s' %
+                  INDEX_EXTENSIONS)
 
         idx = open_dir(self.index_location, indexname=self.indexname)
         # The set of all paths in the index
@@ -204,14 +212,19 b' class WhooshIndexingDaemon(object):'
         # Loop over the files in the filesystem
         # Assume we have a function that gathers the filenames of the
         # documents to be indexed
+        ri_cnt = riwc_cnt = 0
         for repo_name, repo in self.repo_paths.items():
             for path in self.get_paths(repo):
                 if path in to_index or path not in indexed_paths:
                     # This is either a file that's changed, or a new file
                     # that wasn't indexed before. So index it!
-                    self.add_doc(writer, path, repo, repo_name)
+                    i, iwc = self.add_doc(writer, path, repo, repo_name)
                     log.debug('re indexing %s' % path)
-
+                    ri_cnt += i
+                    riwc_cnt += iwc
+        log.debug('added %s files %s with content for repo %s' % (
+                     ri_cnt + riwc_cnt, riwc_cnt, repo.path)
+        )
         log.debug('>> COMMITING CHANGES <<')
         writer.commit(merge=True)
         log.debug('>>> FINISHED REBUILDING INDEX <<<')

rhodecode/lib/markup_renderer.py

@@ -27,7 +27,7 b''
 import re
 import logging
 
-from rhodecode.lib import safe_unicode
+from rhodecode.lib.utils2 import safe_unicode
 
 log = logging.getLogger(__name__)
 

rhodecode/lib/middleware/https_fixup.py

@@ -23,7 +23,7 b''
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from rhodecode.lib import str2bool
+from rhodecode.lib.utils2 import str2bool
 
 
 class HttpsFixup(object):

rhodecode/lib/middleware/simplegit.py

@@ -69,7 +69,7 b' from dulwich.web import HTTPGitApplicati'
 
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 
-from rhodecode.lib import safe_str
+from rhodecode.lib.utils2 import safe_str
 from rhodecode.lib.base import BaseVCSController
 from rhodecode.lib.auth import get_container_username
 from rhodecode.lib.utils import is_valid_repo

rhodecode/lib/middleware/simplehg.py

@@ -34,7 +34,7 b' from mercurial.hgweb import hgweb_mod'
 
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 
-from rhodecode.lib import safe_str
+from rhodecode.lib.utils2 import safe_str
 from rhodecode.lib.base import BaseVCSController
 from rhodecode.lib.auth import get_container_username
 from rhodecode.lib.utils import make_ui, is_valid_repo, ui_sections

rhodecode/lib/utils.py

@@ -54,7 +54,8 b' from rhodecode.model.db import Repositor'
     UserLog, RepoGroup, RhodeCodeSetting, UserRepoGroupToPerm
 from rhodecode.model.meta import Session
 from rhodecode.model.repos_group import ReposGroupModel
-from rhodecode.lib import safe_str, safe_unicode
+from rhodecode.lib.utils2 import safe_str, safe_unicode
+from rhodecode.lib.vcs.utils.fakemod import create_module
 
 log = logging.getLogger(__name__)
 
@@ -62,7 +63,8 b" REMOVED_REPO_PAT = re.compile(r'rm__\\d{8"
 
 
 def recursive_replace(str_, replace=' '):
-    """Recursive replace of given sign to just one instance
+    """
+    Recursive replace of given sign to just one instance
 
     :param str_: given string
     :param replace: char to find and replace multiple instances
@@ -80,7 +82,8 b" def recursive_replace(str_, replace=' ')"
 
 
 def repo_name_slug(value):
-    """Return slug of name of repository
+    """
+    Return slug of name of repository
     This function is called on each creation/modification
     of repository to prevent bad names in repo
     """
@@ -263,7 +266,8 b" ui_sections = ['alias', 'auth',"
 
 
 def make_ui(read_from='file', path=None, checkpaths=True):
-    """A function that will read python rc files or database
+    """
+    A function that will read python rc files or database
     and make an mercurial ui object from read options
 
     :param path: path to mercurial config file
@@ -489,6 +493,30 b' def add_cache(settings):'
             beaker.cache.cache_regions[region] = region_settings
 
 
+def load_rcextensions(root_path):
+    import rhodecode
+    from rhodecode.config import conf
+
+    path = os.path.join(root_path, 'rcextensions', '__init__.py')
+    if os.path.isfile(path):
+        rcext = create_module('rc', path)
+        EXT = rhodecode.EXTENSIONS = rcext
+        log.debug('Found rcextensions now loading %s...' % rcext)
+
+        # Additional mappings that are not present in the pygments lexers
+        conf.LANGUAGES_EXTENSIONS_MAP.update(getattr(EXT, 'EXTRA_MAPPINGS', {}))
+
+        #OVERRIDE OUR EXTENSIONS FROM RC-EXTENSIONS (if present)
+
+        if getattr(EXT, 'INDEX_EXTENSIONS', []) != []:
+            log.debug('settings custom INDEX_EXTENSIONS')
+            conf.INDEX_EXTENSIONS = getattr(EXT, 'INDEX_EXTENSIONS', [])
+
+        #ADDITIONAL MAPPINGS
+        log.debug('adding extra into INDEX_EXTENSIONS')
+        conf.INDEX_EXTENSIONS.extend(getattr(EXT, 'EXTRA_INDEX_EXTENSIONS', []))
+
+
 #==============================================================================
 # TEST FUNCTIONS AND CREATORS
 #==============================================================================

rhodecode/model/comment.py

@@ -29,7 +29,7 b' import traceback'
 from pylons.i18n.translation import _
 from sqlalchemy.util.compat import defaultdict
 
-from rhodecode.lib import extract_mentioned_users
+from rhodecode.lib.utils2 import extract_mentioned_users
 from rhodecode.lib import helpers as h
 from rhodecode.model import BaseModel
 from rhodecode.model.db import ChangesetComment, User, Repository, Notification

rhodecode/model/db.py

@@ -39,7 +39,8 b' from rhodecode.lib.vcs.utils.helpers imp'
 from rhodecode.lib.vcs.exceptions import VCSError
 from rhodecode.lib.vcs.utils.lazy import LazyProperty
 
-from rhodecode.lib import str2bool, safe_str, get_changeset_safe, safe_unicode
+from rhodecode.lib.utils2 import str2bool, safe_str, get_changeset_safe, \
+    safe_unicode
 from rhodecode.lib.compat import json
 from rhodecode.lib.caching_query import FromCache
 

rhodecode/model/notification.py

@@ -32,6 +32,7 b' import datetime'
 from pylons.i18n.translation import _
 
 import rhodecode
+from rhodecode.config.conf import DATETIME_FORMAT
 from rhodecode.lib import helpers as h
 from rhodecode.model import BaseModel
 from rhodecode.model.db import Notification, User, UserNotification
@@ -176,8 +177,6 b' class NotificationModel(BaseModel):'
             notification.TYPE_REGISTRATION: _('registered in RhodeCode')
         }
 
-        DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S"
-
         tmpl = "%(user)s %(action)s %(when)s"
         if show_age:
             when = h.age(notification.created_on)

rhodecode/model/repo.py

@@ -30,8 +30,7 b' from datetime import datetime'
 
 from rhodecode.lib.vcs.backends import get_backend
 
-from rhodecode.lib import LazyProperty
-from rhodecode.lib import safe_str, safe_unicode
+from rhodecode.lib.utils2 import LazyProperty, safe_str, safe_unicode
 from rhodecode.lib.caching_query import FromCache
 from rhodecode.lib.hooks import log_create_repository
 

rhodecode/model/repos_group.py

@@ -28,7 +28,7 b' import logging'
 import traceback
 import shutil
 
-from rhodecode.lib import LazyProperty
+from rhodecode.lib.utils2 import LazyProperty
 
 from rhodecode.model import BaseModel
 from rhodecode.model.db import RepoGroup, RhodeCodeUi, UserRepoGroupToPerm, \

rhodecode/model/scm.py

@@ -35,7 +35,7 b' from rhodecode.lib.vcs.nodes import File'
 
 from rhodecode import BACKENDS
 from rhodecode.lib import helpers as h
-from rhodecode.lib import safe_str
+from rhodecode.lib.utils2 import safe_str
 from rhodecode.lib.auth import HasRepoPermissionAny, HasReposGroupPermissionAny
 from rhodecode.lib.utils import get_repos as get_filesystem_repos, make_ui, \
     action_logger, EmptyChangeset, REMOVED_REPO_PAT

rhodecode/model/user.py

@@ -29,7 +29,7 b' import traceback'
 from pylons import url
 from pylons.i18n.translation import _
 
-from rhodecode.lib import safe_unicode
+from rhodecode.lib.utils2 import safe_unicode, generate_api_key
 from rhodecode.lib.caching_query import FromCache
 
 from rhodecode.model import BaseModel
@@ -40,7 +40,7 b' from rhodecode.lib.exceptions import Def'
     UserOwnsReposException
 
 from sqlalchemy.exc import DatabaseError
-from rhodecode.lib import generate_api_key
+
 from sqlalchemy.orm import joinedload
 
 log = logging.getLogger(__name__)

rhodecode/public/css/style.css

@@ -3112,7 +3112,12 b' table.code-browser .browser-dir {'
 	top: 5px;
 	width: 16px;
 }
-
+div#legend_data{
+	padding-left:10px;
+}
+div#legend_container table{
+	border: none !important;
+}
 div#legend_container table,div#legend_choices table {
 	width: auto !important;
 }

rhodecode/tests/functional/test_login.py

@@ -1,10 +1,11 b''
 # -*- coding: utf-8 -*-
 from rhodecode.tests import *
 from rhodecode.model.db import User, Notification
-from rhodecode.lib import generate_api_key
+from rhodecode.lib.utils2 import generate_api_key
 from rhodecode.lib.auth import check_password
 from rhodecode.model.meta import Session
 
+
 class TestLoginController(TestController):
 
     def tearDown(self):

rhodecode/tests/test_libs.py

@@ -65,22 +65,20 b' TEST_URLS += ['
 
 class TestLibs(unittest.TestCase):
 
-
     def test_uri_filter(self):
-        from rhodecode.lib import uri_filter
+        from rhodecode.lib.utils2 import uri_filter
 
         for url in TEST_URLS:
             self.assertEqual(uri_filter(url[0]), url[1])
 
     def test_credentials_filter(self):
-        from rhodecode.lib import credentials_filter
+        from rhodecode.lib.utils2 import credentials_filter
 
         for url in TEST_URLS:
             self.assertEqual(credentials_filter(url[0]), url[2])
 
-
     def test_str2bool(self):
-        from rhodecode.lib import str2bool
+        from rhodecode.lib.utils2 import str2bool
         test_cases = [
             ('t', True),
             ('true', True),
@@ -103,9 +101,8 b' class TestLibs(unittest.TestCase):'
         for case in test_cases:
             self.assertEqual(str2bool(case[0]), case[1])
 
-
     def test_mention_extractor(self):
-        from rhodecode.lib import extract_mentioned_users
+        from rhodecode.lib.utils2 import extract_mentioned_users
         sample = ("@first hi there @marcink here's my email [e-mail] "
                   "@lukaszb check it pls @ ttwelve @D[] @one@two@three "
                   "@MARCIN    @maRCiN @2one_more22")