Remove wrong/unnecessary/unfixable comment(s)

rhodecode/lib/hooks.py

@@ -1,476 +1,468 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 rhodecode.lib.hooks
 ~~~~~~~~~~~~~~~~~~~
 
 Hooks runned by rhodecode
 
 :created_on: Aug 6, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH.
 :license: GPLv3, see LICENSE for more details.
 """
 
 import os
 import sys
 import time
 import binascii
 
 from rhodecode.lib.vcs.utils.hgcompat import nullrev, revrange
 from rhodecode.lib import helpers as h
 from rhodecode.lib.utils import action_logger
 from rhodecode.lib.vcs.backends.base import EmptyChangeset
 from rhodecode.lib.exceptions import HTTPLockedRC, UserCreationError
 from rhodecode.lib.utils2 import safe_str, _extract_extras
 from rhodecode.model.db import Repository, User
 
 
 def _get_scm_size(alias, root_path):
 
     if not alias.startswith('.'):
         alias += '.'
 
     size_scm, size_root = 0, 0
     for path, dirs, files in os.walk(safe_str(root_path)):
         if path.find(alias) != -1:
             for f in files:
                 try:
                     size_scm += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
         else:
             for f in files:
                 try:
                     size_root += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
 
     size_scm_f = h.format_byte_size(size_scm)
     size_root_f = h.format_byte_size(size_root)
     size_total_f = h.format_byte_size(size_root + size_scm)
 
     return size_scm_f, size_root_f, size_total_f
 
 
 def repo_size(ui, repo, hooktype=None, **kwargs):
     """
     Presents size of repository after push
 
     :param ui:
     :param repo:
     :param hooktype:
     """
 
     size_hg_f, size_root_f, size_total_f = _get_scm_size('.hg', repo.root)
 
     last_cs = repo[len(repo) - 1]
 
     msg = ('Repository size .hg:%s repo:%s total:%s\n'
            'Last revision is now r%s:%s\n') % (
         size_hg_f, size_root_f, size_total_f, last_cs.rev(), last_cs.hex()[:12]
     )
 
     sys.stdout.write(msg)
 
 
 def pre_push(ui, repo, **kwargs):
     # pre push function, currently used to ban pushing when
     # repository is locked
     ex = _extract_extras()
 
     usr = User.get_by_username(ex.username)
     if ex.locked_by[0] and usr.user_id != int(ex.locked_by[0]):
         locked_by = User.get(ex.locked_by[0]).username
         # this exception is interpreted in git/hg middlewares and based
         # on that proper return code is server to client
         _http_ret = HTTPLockedRC(ex.repository, locked_by)
         if str(_http_ret.code).startswith('2'):
             #2xx Codes don't raise exceptions
             sys.stdout.write(_http_ret.title)
         else:
             raise _http_ret
 
 
 def pre_pull(ui, repo, **kwargs):
     # pre push function, currently used to ban pushing when
     # repository is locked
     ex = _extract_extras()
     if ex.locked_by[0]:
         locked_by = User.get(ex.locked_by[0]).username
         # this exception is interpreted in git/hg middlewares and based
         # on that proper return code is server to client
         _http_ret = HTTPLockedRC(ex.repository, locked_by)
         if str(_http_ret.code).startswith('2'):
             #2xx Codes don't raise exceptions
             sys.stdout.write(_http_ret.title)
         else:
             raise _http_ret
 
 
 def log_pull_action(ui, repo, **kwargs):
     """
     Logs user last pull action
 
     :param ui:
     :param repo:
     """
     ex = _extract_extras()
 
     user = User.get_by_username(ex.username)
     action = 'pull'
     action_logger(user, action, ex.repository, ex.ip, commit=True)
     # extension hook call
     from rhodecode import EXTENSIONS
     callback = getattr(EXTENSIONS, 'PULL_HOOK', None)
     if callable(callback):
         kw = {}
         kw.update(ex)
         callback(**kw)
 
     if ex.make_lock is not None and ex.make_lock:
         Repository.lock(Repository.get_by_repo_name(ex.repository), user.user_id)
         #msg = 'Made lock on repo `%s`' % repository
         #sys.stdout.write(msg)
 
     if ex.locked_by[0]:
         locked_by = User.get(ex.locked_by[0]).username
         _http_ret = HTTPLockedRC(ex.repository, locked_by)
         if str(_http_ret.code).startswith('2'):
             #2xx Codes don't raise exceptions
             sys.stdout.write(_http_ret.title)
     return 0
 
 
 def log_push_action(ui, repo, **kwargs):
     """
     Maps user last push action to new changeset id, from mercurial
 
     :param ui:
     :param repo: repo object containing the `ui` object
     """
 
     ex = _extract_extras()
 
     action_tmpl = ex.action + ':%s'
     revs = []
     if ex.scm == 'hg':
         node = kwargs['node']
 
         def get_revs(repo, rev_opt):
             if rev_opt:
                 revs = revrange(repo, rev_opt)
 
                 if len(revs) == 0:
                     return (nullrev, nullrev)
                 return max(revs), min(revs)
             else:
                 return len(repo) - 1, 0
 
         stop, start = get_revs(repo, [node + ':'])
         _h = binascii.hexlify
         revs = [_h(repo[r].node()) for r in xrange(start, stop + 1)]
     elif ex.scm == 'git':
         revs = kwargs.get('_git_revs', [])
         if '_git_revs' in kwargs:
             kwargs.pop('_git_revs')
 
     action = action_tmpl % ','.join(revs)
     action_logger(ex.username, action, ex.repository, ex.ip, commit=True)
 
     # extension hook call
     from rhodecode import EXTENSIONS
     callback = getattr(EXTENSIONS, 'PUSH_HOOK', None)
     if callable(callback):
         kw = {'pushed_revs': revs}
         kw.update(ex)
         callback(**kw)
 
     if ex.make_lock is not None and not ex.make_lock:
         Repository.unlock(Repository.get_by_repo_name(ex.repository))
         msg = 'Released lock on repo `%s`\n' % ex.repository
         sys.stdout.write(msg)
 
     if ex.locked_by[0]:
         locked_by = User.get(ex.locked_by[0]).username
         _http_ret = HTTPLockedRC(ex.repository, locked_by)
         if str(_http_ret.code).startswith('2'):
             #2xx Codes don't raise exceptions
             sys.stdout.write(_http_ret.title)
 
     return 0
 
 
 def log_create_repository(repository_dict, created_by, **kwargs):
     """
-    Post create repository Hook. This is a dummy function for admins to re-use
-    if needed. It's taken from rhodecode-extensions module and executed
-    if present
+    Post create repository Hook.
 
     :param repository: dict dump of repository object
     :param created_by: username who created repository
 
     available keys of repository_dict:
 
      'repo_type',
      'description',
      'private',
      'created_on',
      'enable_downloads',
      'repo_id',
      'user_id',
      'enable_statistics',
      'clone_uri',
      'fork_id',
      'group_id',
      'repo_name'
 
     """
     from rhodecode import EXTENSIONS
     callback = getattr(EXTENSIONS, 'CREATE_REPO_HOOK', None)
     if callable(callback):
         kw = {}
         kw.update(repository_dict)
         kw.update({'created_by': created_by})
         kw.update(kwargs)
         return callback(**kw)
 
     return 0
 
 
 def check_allowed_create_user(user_dict, created_by, **kwargs):
     # pre create hooks
     from rhodecode import EXTENSIONS
     callback = getattr(EXTENSIONS, 'PRE_CREATE_USER_HOOK', None)
     if callable(callback):
         allowed, reason = callback(created_by=created_by, **user_dict)
         if not allowed:
             raise UserCreationError(reason)
 
 
 def log_create_user(user_dict, created_by, **kwargs):
     """
-    Post create user Hook. This is a dummy function for admins to re-use
-    if needed. It's taken from rhodecode-extensions module and executed
-    if present
+    Post create user Hook.
 
     :param user_dict: dict dump of user object
 
     available keys for user_dict:
 
      'username',
      'full_name_or_username',
      'full_contact',
      'user_id',
      'name',
      'firstname',
      'short_contact',
      'admin',
      'lastname',
      'ip_addresses',
      'ldap_dn',
      'email',
      'api_key',
      'last_login',
      'full_name',
      'active',
      'password',
      'emails',
      'inherit_default_permissions'
 
     """
     from rhodecode import EXTENSIONS
     callback = getattr(EXTENSIONS, 'CREATE_USER_HOOK', None)
     if callable(callback):
         return callback(created_by=created_by, **user_dict)
 
     return 0
 
 
 def log_delete_repository(repository_dict, deleted_by, **kwargs):
     """
-    Post delete repository Hook. This is a dummy function for admins to re-use
-    if needed. It's taken from rhodecode-extensions module and executed
-    if present
+    Post delete repository Hook.
 
     :param repository: dict dump of repository object
     :param deleted_by: username who deleted the repository
 
     available keys of repository_dict:
 
      'repo_type',
      'description',
      'private',
      'created_on',
      'enable_downloads',
      'repo_id',
      'user_id',
      'enable_statistics',
      'clone_uri',
      'fork_id',
      'group_id',
      'repo_name'
 
     """
     from rhodecode import EXTENSIONS
     callback = getattr(EXTENSIONS, 'DELETE_REPO_HOOK', None)
     if callable(callback):
         kw = {}
         kw.update(repository_dict)
         kw.update({'deleted_by': deleted_by,
                    'deleted_on': time.time()})
         kw.update(kwargs)
         return callback(**kw)
 
     return 0
 
 
 def log_delete_user(user_dict, deleted_by, **kwargs):
     """
-    Post delete user Hook. This is a dummy function for admins to re-use
-    if needed. It's taken from rhodecode-extensions module and executed
-    if present
+    Post delete user Hook.
 
     :param user_dict: dict dump of user object
 
     available keys for user_dict:
 
      'username',
      'full_name_or_username',
      'full_contact',
      'user_id',
      'name',
      'firstname',
      'short_contact',
      'admin',
      'lastname',
      'ip_addresses',
      'ldap_dn',
      'email',
      'api_key',
      'last_login',
      'full_name',
      'active',
      'password',
      'emails',
      'inherit_default_permissions'
 
     """
     from rhodecode import EXTENSIONS
     callback = getattr(EXTENSIONS, 'DELETE_USER_HOOK', None)
     if callable(callback):
         return callback(deleted_by=deleted_by, **user_dict)
 
     return 0
 
 
 handle_git_pre_receive = (lambda repo_path, revs, env:
     handle_git_receive(repo_path, revs, env, hook_type='pre'))
 handle_git_post_receive = (lambda repo_path, revs, env:
     handle_git_receive(repo_path, revs, env, hook_type='post'))
 
 
 def handle_git_receive(repo_path, revs, env, hook_type='post'):
     """
     A really hacky method that is runned by git post-receive hook and logs
     an push action together with pushed revisions. It's executed by subprocess
     thus needs all info to be able to create a on the fly pylons enviroment,
     connect to database and run the logging code. Hacky as sh*t but works.
 
     :param repo_path:
     :param revs:
     :param env:
     """
     from paste.deploy import appconfig
     from sqlalchemy import engine_from_config
     from rhodecode.config.environment import load_environment
     from rhodecode.model import init_model
     from rhodecode.model.db import RhodeCodeUi
     from rhodecode.lib.utils import make_ui
     extras = _extract_extras(env)
 
     path, ini_name = os.path.split(extras['config'])
     conf = appconfig('config:%s' % ini_name, relative_to=path)
     load_environment(conf.global_conf, conf.local_conf, test_env=False,
                      test_index=False)
 
     engine = engine_from_config(conf, 'sqlalchemy.db1.')
     init_model(engine)
 
     baseui = make_ui('db')
     # fix if it's not a bare repo
     if repo_path.endswith(os.sep + '.git'):
         repo_path = repo_path[:-5]
 
     repo = Repository.get_by_full_path(repo_path)
     if not repo:
         raise OSError('Repository %s not found in database'
                       % (safe_str(repo_path)))
 
     _hooks = dict(baseui.configitems('hooks')) or {}
 
     if hook_type == 'pre':
         repo = repo.scm_instance
     else:
         #post push shouldn't use the cached instance never
         repo = repo.scm_instance_no_cache()
 
     if hook_type == 'pre':
         pre_push(baseui, repo)
 
     # if push hook is enabled via web interface
     elif hook_type == 'post' and _hooks.get(RhodeCodeUi.HOOK_PUSH):
         rev_data = []
         for l in revs:
             old_rev, new_rev, ref = l.split(' ')
             _ref_data = ref.split('/')
             if _ref_data[1] in ['tags', 'heads']:
                 rev_data.append({'old_rev': old_rev,
                                  'new_rev': new_rev,
                                  'ref': ref,
                                  'type': _ref_data[1],
                                  'name': _ref_data[2].strip()})
 
         git_revs = []
 
         for push_ref in rev_data:
             _type = push_ref['type']
             if _type == 'heads':
                 if push_ref['old_rev'] == EmptyChangeset().raw_id:
                     # update the symbolic ref if we push new repo
                     if repo.is_empty():
                         repo._repo.refs.set_symbolic_ref('HEAD',
                                             'refs/heads/%s' % push_ref['name'])
 
                     cmd = "for-each-ref --format='%(refname)' 'refs/heads/*'"
                     heads = repo.run_git_command(cmd)[0]
                     heads = heads.replace(push_ref['ref'], '')
                     heads = ' '.join(map(lambda c: c.strip('\n').strip(),
                                          heads.splitlines()))
                     cmd = (('log %(new_rev)s' % push_ref) +
                            ' --reverse --pretty=format:"%H" --not ' + heads)
                     git_revs += repo.run_git_command(cmd)[0].splitlines()
 
                 elif push_ref['new_rev'] == EmptyChangeset().raw_id:
                     #delete branch case
                     git_revs += ['delete_branch=>%s' % push_ref['name']]
                 else:
                     cmd = (('log %(old_rev)s..%(new_rev)s' % push_ref) +
                            ' --reverse --pretty=format:"%H"')
                     git_revs += repo.run_git_command(cmd)[0].splitlines()
 
             elif _type == 'tags':
                 git_revs += ['tag=>%s' % push_ref['name']]
 
         log_push_action(baseui, repo, _git_revs=git_revs)

rhodecode/lib/indexers/__init__.py

@@ -1,195 +1,195 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 rhodecode.lib.indexers.__init__
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Whoosh indexing module for RhodeCode
 
 :created_on: Aug 17, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH.
 :license: GPLv3, see LICENSE for more details.
 """
 
 import os
 import sys
 import logging
 from os.path import dirname as dn, join as jn
 
-#to get the rhodecode import
+# Add location of top level folder to sys.path
 sys.path.append(dn(dn(dn(os.path.realpath(__file__)))))
 
 from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter
 from whoosh.fields import TEXT, ID, STORED, NUMERIC, BOOLEAN, Schema, FieldType, DATETIME
 from whoosh.formats import Characters
 from whoosh.highlight import highlight as whoosh_highlight, HtmlFormatter, ContextFragmenter
 from rhodecode.lib.utils2 import LazyProperty
 
 log = logging.getLogger(__name__)
 
 # CUSTOM ANALYZER wordsplit + lowercase filter
 ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter()
 
 #INDEX SCHEMA DEFINITION
 SCHEMA = Schema(
     fileid=ID(unique=True),
     owner=TEXT(),
     repository=TEXT(stored=True),
     path=TEXT(stored=True),
     content=FieldType(format=Characters(), analyzer=ANALYZER,
                       scorable=True, stored=True),
     modtime=STORED(),
     extension=TEXT(stored=True)
 )
 
 IDX_NAME = 'HG_INDEX'
 FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n')
 FRAGMENTER = ContextFragmenter(200)
 
 CHGSETS_SCHEMA = Schema(
     raw_id=ID(unique=True, stored=True),
     date=NUMERIC(stored=True),
     last=BOOLEAN(),
     owner=TEXT(),
     repository=ID(unique=True, stored=True),
     author=TEXT(stored=True),
     message=FieldType(format=Characters(), analyzer=ANALYZER,
                       scorable=True, stored=True),
     parents=TEXT(),
     added=TEXT(),
     removed=TEXT(),
     changed=TEXT(),
 )
 
 CHGSET_IDX_NAME = 'CHGSET_INDEX'
 
 # used only to generate queries in journal
 JOURNAL_SCHEMA = Schema(
     username=TEXT(),
     date=DATETIME(),
     action=TEXT(),
     repository=TEXT(),
     ip=TEXT(),
 )
 
 
 class WhooshResultWrapper(object):
     def __init__(self, search_type, searcher, matcher, highlight_items,
                  repo_location):
         self.search_type = search_type
         self.searcher = searcher
         self.matcher = matcher
         self.highlight_items = highlight_items
         self.fragment_size = 200
         self.repo_location = repo_location
 
     @LazyProperty
     def doc_ids(self):
         docs_id = []
         while self.matcher.is_active():
             docnum = self.matcher.id()
             chunks = [offsets for offsets in self.get_chunks()]
             docs_id.append([docnum, chunks])
             self.matcher.next()
         return docs_id
 
     def __str__(self):
         return '<%s at %s>' % (self.__class__.__name__, len(self.doc_ids))
 
     def __repr__(self):
         return self.__str__()
 
     def __len__(self):
         return len(self.doc_ids)
 
     def __iter__(self):
         """
         Allows Iteration over results,and lazy generate content
 
         *Requires* implementation of ``__getitem__`` method.
         """
         for docid in self.doc_ids:
             yield self.get_full_content(docid)
 
     def __getitem__(self, key):
         """
         Slicing of resultWrapper
         """
         i, j = key.start, key.stop
 
         slices = []
         for docid in self.doc_ids[i:j]:
             slices.append(self.get_full_content(docid))
         return slices
 
     def get_full_content(self, docid):
         res = self.searcher.stored_fields(docid[0])
         log.debug('result: %s' % res)
         if self.search_type == 'content':
             full_repo_path = jn(self.repo_location, res['repository'])
             f_path = res['path'].split(full_repo_path)[-1]
             f_path = f_path.lstrip(os.sep)
             content_short = self.get_short_content(res, docid[1])
             res.update({'content_short': content_short,
                         'content_short_hl': self.highlight(content_short),
                         'f_path': f_path
             })
         elif self.search_type == 'path':
             full_repo_path = jn(self.repo_location, res['repository'])
             f_path = res['path'].split(full_repo_path)[-1]
             f_path = f_path.lstrip(os.sep)
             res.update({'f_path': f_path})
         elif self.search_type == 'message':
             res.update({'message_hl': self.highlight(res['message'])})
 
         log.debug('result: %s' % res)
 
         return res
 
     def get_short_content(self, res, chunks):
 
         return ''.join([res['content'][chunk[0]:chunk[1]] for chunk in chunks])
 
     def get_chunks(self):
         """
         Smart function that implements chunking the content
         but not overlap chunks so it doesn't highlight the same
         close occurrences twice.
         """
         memory = [(0, 0)]
         if self.matcher.supports('positions'):
             for span in self.matcher.spans():
                 start = span.startchar or 0
                 end = span.endchar or 0
                 start_offseted = max(0, start - self.fragment_size)
                 end_offseted = end + self.fragment_size
 
                 if start_offseted < memory[-1][1]:
                     start_offseted = memory[-1][1]
                 memory.append((start_offseted, end_offseted,))
                 yield (start_offseted, end_offseted,)
 
     def highlight(self, content, top=5):
         if self.search_type not in ['content', 'message']:
             return ''
         hl = whoosh_highlight(
             text=content,
             terms=self.highlight_items,
             analyzer=ANALYZER,
             fragmenter=FRAGMENTER,
             formatter=FORMATTER,
             top=top
         )
         return hl

rhodecode/lib/indexers/daemon.py

@@ -1,445 +1,445 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 rhodecode.lib.indexers.daemon
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 A daemon will read from task table and run tasks
 
 :created_on: Jan 26, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH.
 :license: GPLv3, see LICENSE for more details.
 """
 
 from __future__ import with_statement
 
 import os
 import sys
 import logging
 import traceback
 
 from shutil import rmtree
 from time import mktime
 
 from os.path import dirname as dn
 from os.path import join as jn
 
-#to get the rhodecode import
+# Add location of top level folder to sys.path
 project_path = dn(dn(dn(dn(os.path.realpath(__file__)))))
 sys.path.append(project_path)
 
 from rhodecode.config.conf import INDEX_EXTENSIONS
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.db import Repository
 from rhodecode.lib.utils2 import safe_unicode, safe_str
 from rhodecode.lib.indexers import SCHEMA, IDX_NAME, CHGSETS_SCHEMA, \
     CHGSET_IDX_NAME
 
 from rhodecode.lib.vcs.exceptions import ChangesetError, RepositoryError, \
     NodeDoesNotExistError
 
 from whoosh.index import create_in, open_dir, exists_in
 from whoosh.query import *
 from whoosh.qparser import QueryParser
 
 log = logging.getLogger('whoosh_indexer')
 
 
 class WhooshIndexingDaemon(object):
     """
     Daemon for atomic indexing jobs
     """
 
     def __init__(self, indexname=IDX_NAME, index_location=None,
                  repo_location=None, sa=None, repo_list=None,
                  repo_update_list=None):
         self.indexname = indexname
 
         self.index_location = index_location
         if not index_location:
             raise Exception('You have to provide index location')
 
         self.repo_location = repo_location
         if not repo_location:
             raise Exception('You have to provide repositories location')
 
         self.repo_paths = ScmModel(sa).repo_scan(self.repo_location)
 
         #filter repo list
         if repo_list:
             #Fix non-ascii repo names to unicode
             repo_list = map(safe_unicode, repo_list)
             self.filtered_repo_paths = {}
             for repo_name, repo in self.repo_paths.items():
                 if repo_name in repo_list:
                     self.filtered_repo_paths[repo_name] = repo
 
             self.repo_paths = self.filtered_repo_paths
 
         #filter update repo list
         self.filtered_repo_update_paths = {}
         if repo_update_list:
             self.filtered_repo_update_paths = {}
             for repo_name, repo in self.repo_paths.items():
                 if repo_name in repo_update_list:
                     self.filtered_repo_update_paths[repo_name] = repo
             self.repo_paths = self.filtered_repo_update_paths
 
         self.initial = True
         if not os.path.isdir(self.index_location):
             os.makedirs(self.index_location)
             log.info('Cannot run incremental index since it does not '
                      'yet exist running full build')
         elif not exists_in(self.index_location, IDX_NAME):
             log.info('Running full index build as the file content '
                      'index does not exist')
         elif not exists_in(self.index_location, CHGSET_IDX_NAME):
             log.info('Running full index build as the changeset '
                      'index does not exist')
         else:
             self.initial = False
 
     def _get_index_revision(self, repo):
         db_repo = Repository.get_by_repo_name(repo.name_unicode)
         landing_rev = 'tip'
         if db_repo:
             _rev_type, _rev = db_repo.landing_rev
             landing_rev = _rev
         return landing_rev
 
     def _get_index_changeset(self, repo, index_rev=None):
         if not index_rev:
             index_rev = self._get_index_revision(repo)
         cs = repo.get_changeset(index_rev)
         return cs
 
     def get_paths(self, repo):
         """
         recursive walk in root dir and return a set of all path in that dir
         based on repository walk function
         """
         index_paths_ = set()
         try:
             cs = self._get_index_changeset(repo)
             for _topnode, _dirs, files in cs.walk('/'):
                 for f in files:
                     index_paths_.add(jn(safe_str(repo.path), safe_str(f.path)))
 
         except RepositoryError:
             log.debug(traceback.format_exc())
             pass
         return index_paths_
 
     def get_node(self, repo, path, index_rev=None):
         """
         gets a filenode based on given full path.It operates on string for
         hg git compatability.
 
         :param repo: scm repo instance
         :param path: full path including root location
         :return: FileNode
         """
         root_path = safe_str(repo.path)+'/'
         parts = safe_str(path).partition(root_path)
         cs = self._get_index_changeset(repo, index_rev=index_rev)
         node = cs.get_node(parts[-1])
         return node
 
     def get_node_mtime(self, node):
         return mktime(node.last_changeset.date.timetuple())
 
     def add_doc(self, writer, path, repo, repo_name, index_rev=None):
         """
         Adding doc to writer this function itself fetches data from
         the instance of vcs backend
         """
 
         node = self.get_node(repo, path, index_rev)
         indexed = indexed_w_content = 0
         # we just index the content of chosen files, and skip binary files
         if node.extension in INDEX_EXTENSIONS and not node.is_binary:
             u_content = node.content
             if not isinstance(u_content, unicode):
                 log.warning('  >> %s Could not get this content as unicode '
                             'replacing with empty content' % path)
                 u_content = u''
             else:
                 log.debug('    >> %s [WITH CONTENT]' % path)
                 indexed_w_content += 1
 
         else:
             log.debug('    >> %s' % path)
             # just index file name without it's content
             u_content = u''
             indexed += 1
 
         p = safe_unicode(path)
         writer.add_document(
             fileid=p,
             owner=unicode(repo.contact),
             repository=safe_unicode(repo_name),
             path=p,
             content=u_content,
             modtime=self.get_node_mtime(node),
             extension=node.extension
         )
         return indexed, indexed_w_content
 
     def index_changesets(self, writer, repo_name, repo, start_rev=None):
         """
         Add all changeset in the vcs repo starting at start_rev
         to the index writer
 
         :param writer: the whoosh index writer to add to
         :param repo_name: name of the repository from whence the
           changeset originates including the repository group
         :param repo: the vcs repository instance to index changesets for,
           the presumption is the repo has changesets to index
         :param start_rev=None: the full sha id to start indexing from
           if start_rev is None then index from the first changeset in
           the repo
         """
 
         if start_rev is None:
             start_rev = repo[0].raw_id
 
         log.debug('indexing changesets in %s starting at rev: %s' %
                   (repo_name, start_rev))
 
         indexed = 0
         cs_iter = repo.get_changesets(start=start_rev)
         total = len(cs_iter)
         for cs in cs_iter:
             log.debug('    >> %s/%s' % (cs, total))
             writer.add_document(
                 raw_id=unicode(cs.raw_id),
                 owner=unicode(repo.contact),
                 date=cs._timestamp,
                 repository=safe_unicode(repo_name),
                 author=cs.author,
                 message=cs.message,
                 last=cs.last,
                 added=u' '.join([safe_unicode(node.path) for node in cs.added]).lower(),
                 removed=u' '.join([safe_unicode(node.path) for node in cs.removed]).lower(),
                 changed=u' '.join([safe_unicode(node.path) for node in cs.changed]).lower(),
                 parents=u' '.join([cs.raw_id for cs in cs.parents]),
             )
             indexed += 1
 
         log.debug('indexed %d changesets for repo %s' % (indexed, repo_name))
         return indexed
 
     def index_files(self, file_idx_writer, repo_name, repo):
         """
         Index files for given repo_name
 
         :param file_idx_writer: the whoosh index writer to add to
         :param repo_name: name of the repository we're indexing
         :param repo: instance of vcs repo
         """
         i_cnt = iwc_cnt = 0
         log.debug('building index for %s @revision:%s' % (repo.path,
                                                 self._get_index_revision(repo)))
         index_rev = self._get_index_revision(repo)
         for idx_path in self.get_paths(repo):
             i, iwc = self.add_doc(file_idx_writer, idx_path, repo, repo_name, index_rev)
             i_cnt += i
             iwc_cnt += iwc
 
         log.debug('added %s files %s with content for repo %s' %
                   (i_cnt + iwc_cnt, iwc_cnt, repo.path))
         return i_cnt, iwc_cnt
 
     def update_changeset_index(self):
         idx = open_dir(self.index_location, indexname=CHGSET_IDX_NAME)
 
         with idx.searcher() as searcher:
             writer = idx.writer()
             writer_is_dirty = False
             try:
                 indexed_total = 0
                 repo_name = None
                 for repo_name, repo in self.repo_paths.items():
                     # skip indexing if there aren't any revs in the repo
                     num_of_revs = len(repo)
                     if num_of_revs < 1:
                         continue
 
                     qp = QueryParser('repository', schema=CHGSETS_SCHEMA)
                     q = qp.parse(u"last:t AND %s" % repo_name)
 
                     results = searcher.search(q)
 
                     # default to scanning the entire repo
                     last_rev = 0
                     start_id = None
 
                     if len(results) > 0:
                         # assuming that there is only one result, if not this
                         # may require a full re-index.
                         start_id = results[0]['raw_id']
                         last_rev = repo.get_changeset(revision=start_id).revision
 
                     # there are new changesets to index or a new repo to index
                     if last_rev == 0 or num_of_revs > last_rev + 1:
                         # delete the docs in the index for the previous
                         # last changeset(s)
                         for hit in results:
                             q = qp.parse(u"last:t AND %s AND raw_id:%s" %
                                             (repo_name, hit['raw_id']))
                             writer.delete_by_query(q)
 
                         # index from the previous last changeset + all new ones
                         indexed_total += self.index_changesets(writer,
                                                 repo_name, repo, start_id)
                         writer_is_dirty = True
                 log.debug('indexed %s changesets for repo %s' % (
                           indexed_total, repo_name)
                 )
             finally:
                 if writer_is_dirty:
                     log.debug('>> COMMITING CHANGES TO CHANGESET INDEX<<')
                     writer.commit(merge=True)
                     log.debug('>>> FINISHED REBUILDING CHANGESET INDEX <<<')
                 else:
                     log.debug('>> NOTHING TO COMMIT TO CHANGESET INDEX<<')
 
     def update_file_index(self):
         log.debug((u'STARTING INCREMENTAL INDEXING UPDATE FOR EXTENSIONS %s '
                    'AND REPOS %s') % (INDEX_EXTENSIONS, self.repo_paths.keys()))
 
         idx = open_dir(self.index_location, indexname=self.indexname)
         # The set of all paths in the index
         indexed_paths = set()
         # The set of all paths we need to re-index
         to_index = set()
 
         writer = idx.writer()
         writer_is_dirty = False
         try:
             with idx.reader() as reader:
 
                 # Loop over the stored fields in the index
                 for fields in reader.all_stored_fields():
                     indexed_path = fields['path']
                     indexed_repo_path = fields['repository']
                     indexed_paths.add(indexed_path)
 
                     if not indexed_repo_path in self.filtered_repo_update_paths:
                         continue
 
                     repo = self.repo_paths[indexed_repo_path]
 
                     try:
                         node = self.get_node(repo, indexed_path)
                         # Check if this file was changed since it was indexed
                         indexed_time = fields['modtime']
                         mtime = self.get_node_mtime(node)
                         if mtime > indexed_time:
                             # The file has changed, delete it and add it to
                             # the list of files to reindex
                             log.debug(
                                 'adding to reindex list %s mtime: %s vs %s' % (
                                     indexed_path, mtime, indexed_time)
                             )
                             writer.delete_by_term('fileid', indexed_path)
                             writer_is_dirty = True
 
                             to_index.add(indexed_path)
                     except (ChangesetError, NodeDoesNotExistError):
                         # This file was deleted since it was indexed
                         log.debug('removing from index %s' % indexed_path)
                         writer.delete_by_term('path', indexed_path)
                         writer_is_dirty = True
 
             # Loop over the files in the filesystem
             # Assume we have a function that gathers the filenames of the
             # documents to be indexed
             ri_cnt_total = 0  # indexed
             riwc_cnt_total = 0  # indexed with content
             for repo_name, repo in self.repo_paths.items():
                 # skip indexing if there aren't any revisions
                 if len(repo) < 1:
                     continue
                 ri_cnt = 0   # indexed
                 riwc_cnt = 0  # indexed with content
                 for path in self.get_paths(repo):
                     path = safe_unicode(path)
                     if path in to_index or path not in indexed_paths:
 
                         # This is either a file that's changed, or a new file
                         # that wasn't indexed before. So index it!
                         i, iwc = self.add_doc(writer, path, repo, repo_name)
                         writer_is_dirty = True
                         log.debug('re indexing %s' % path)
                         ri_cnt += i
                         ri_cnt_total += 1
                         riwc_cnt += iwc
                         riwc_cnt_total += iwc
                 log.debug('added %s files %s with content for repo %s' % (
                              ri_cnt + riwc_cnt, riwc_cnt, repo.path)
                 )
             log.debug('indexed %s files in total and %s with content' % (
                         ri_cnt_total, riwc_cnt_total)
             )
         finally:
             if writer_is_dirty:
                 log.debug('>> COMMITING CHANGES TO FILE INDEX <<')
                 writer.commit(merge=True)
                 log.debug('>>> FINISHED REBUILDING FILE INDEX <<<')
             else:
                 log.debug('>> NOTHING TO COMMIT TO FILE INDEX <<')
                 writer.cancel()
 
     def build_indexes(self):
         if os.path.exists(self.index_location):
             log.debug('removing previous index')
             rmtree(self.index_location)
 
         if not os.path.exists(self.index_location):
             os.mkdir(self.index_location)
 
         chgset_idx = create_in(self.index_location, CHGSETS_SCHEMA,
                                indexname=CHGSET_IDX_NAME)
         chgset_idx_writer = chgset_idx.writer()
 
         file_idx = create_in(self.index_location, SCHEMA, indexname=IDX_NAME)
         file_idx_writer = file_idx.writer()
         log.debug('BUILDING INDEX FOR EXTENSIONS %s '
                   'AND REPOS %s' % (INDEX_EXTENSIONS, self.repo_paths.keys()))
 
         for repo_name, repo in self.repo_paths.items():
             # skip indexing if there aren't any revisions
             if len(repo) < 1:
                 continue
 
             self.index_files(file_idx_writer, repo_name, repo)
             self.index_changesets(chgset_idx_writer, repo_name, repo)
 
         log.debug('>> COMMITING CHANGES <<')
         file_idx_writer.commit(merge=True)
         chgset_idx_writer.commit(merge=True)
         log.debug('>>> FINISHED BUILDING INDEX <<<')
 
     def update_indexes(self):
         self.update_file_index()
         self.update_changeset_index()
 
     def run(self, full_index=False):
         """Run daemon"""
         if full_index or self.initial:
             self.build_indexes()
         else:
             self.update_indexes()

rhodecode/lib/paster_commands/cache_keys.py

@@ -1,85 +1,85 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 rhodecode.lib.paster_commands.cache_keys
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 cleanup-keys paster command for RhodeCode
 
 
 :created_on: mar 27, 2013
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH.
 :license: GPLv3, see LICENSE for more details.
 """
 
 from __future__ import with_statement
 
 import os
 import sys
 import logging
 
 from rhodecode.model.meta import Session
 from rhodecode.lib.utils import BasePasterCommand
 from rhodecode.model.db import CacheInvalidation
 
-# fix rhodecode import
+# Add location of top level folder to sys.path
 from os.path import dirname as dn
 rc_path = dn(dn(dn(os.path.realpath(__file__))))
 sys.path.append(rc_path)
 
 log = logging.getLogger(__name__)
 
 
 class Command(BasePasterCommand):
 
     max_args = 1
     min_args = 1
 
     usage = "CONFIG_FILE"
     group_name = "RhodeCode"
     takes_config_file = -1
     parser = BasePasterCommand.standard_parser(verbose=True)
     summary = "Cache keys utils"
 
     def command(self):
         #get SqlAlchemy session
         self._init_session()
         _caches = CacheInvalidation.query().order_by(CacheInvalidation.cache_key).all()
         if self.options.show:
             for c_obj in _caches:
                 print 'key:%s active:%s' % (c_obj.cache_key, c_obj.cache_active)
         elif self.options.cleanup:
             for c_obj in _caches:
                 Session().delete(c_obj)
                 print 'removing key:%s' % (c_obj.cache_key)
                 Session().commit()
         else:
             print 'nothing done exiting...'
         sys.exit(0)
 
     def update_parser(self):
         self.parser.add_option(
             '--show',
             action='store_true',
             dest='show',
             help=("show existing cache keys with together with status")
         )
 
         self.parser.add_option(
             '--cleanup',
             action="store_true",
             dest="cleanup",
             help="cleanup existing cache keys"
         )

rhodecode/lib/paster_commands/ishell.py

@@ -1,76 +1,76 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 rhodecode.lib.paster_commands.ishell
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 interactive shell paster command for RhodeCode
 
 :created_on: Apr 4, 2013
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH.
 :license: GPLv3, see LICENSE for more details.
 """
 
 from __future__ import with_statement
 
 import os
 import sys
 import logging
 
 from rhodecode.lib.utils import BasePasterCommand
 
-# fix rhodecode import
+# Add location of top level folder to sys.path
 from os.path import dirname as dn
 rc_path = dn(dn(dn(os.path.realpath(__file__))))
 sys.path.append(rc_path)
 
 log = logging.getLogger(__name__)
 
 
 class Command(BasePasterCommand):
 
     max_args = 1
     min_args = 1
 
     usage = "CONFIG_FILE"
     group_name = "RhodeCode"
     takes_config_file = -1
     parser = BasePasterCommand.standard_parser(verbose=True)
     summary = "Interactive shell"
 
     def command(self):
         #get SqlAlchemy session
         self._init_session()
 
         # imports, used in ipython shell
         import os
         import sys
         import time
         import shutil
         import datetime
         from rhodecode.model.db import *
 
         try:
             from IPython import embed
             from IPython.config.loader import Config
             cfg = Config()
             cfg.InteractiveShellEmbed.confirm_exit = False
             embed(config=cfg, banner1="RhodeCode IShell.")
         except ImportError:
             print 'ipython installation required for ishell'
             sys.exit(-1)
 
     def update_parser(self):
         pass

rhodecode/lib/paster_commands/make_index.py

@@ -1,113 +1,113 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 rhodecode.lib.paster_commands.make_index
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 make-index paster command for RhodeCode
 
 :created_on: Aug 17, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH.
 :license: GPLv3, see LICENSE for more details.
 
 """
 
 from __future__ import with_statement
 
 import os
 import sys
 import logging
 
 from rhodecode.lib.utils import BasePasterCommand
 from string import strip
 from shutil import rmtree
 from rhodecode.model.repo import RepoModel
 from rhodecode.lib.utils import BasePasterCommand, load_rcextensions
 
-# fix rhodecode import
+# Add location of top level folder to sys.path
 from os.path import dirname as dn
 rc_path = dn(dn(dn(os.path.realpath(__file__))))
 sys.path.append(rc_path)
 
 
 class Command(BasePasterCommand):
 
     max_args = 1
     min_args = 1
 
     usage = "CONFIG_FILE"
     group_name = "RhodeCode"
     takes_config_file = -1
     parser = BasePasterCommand.standard_parser(verbose=True)
     summary = "Creates or updates full text search index"
 
     def command(self):
         logging.config.fileConfig(self.path_to_ini_file)
         #get SqlAlchemy session
         self._init_session()
         from pylons import config
         index_location = config['index_dir']
         load_rcextensions(config['here'])
 
         repo_location = self.options.repo_location \
             if self.options.repo_location else RepoModel().repos_path
         repo_list = map(strip, self.options.repo_list.split(',')) \
             if self.options.repo_list else None
 
         repo_update_list = map(strip, self.options.repo_update_list.split(',')) \
             if self.options.repo_update_list else None
 
         #======================================================================
         # WHOOSH DAEMON
         #======================================================================
         from rhodecode.lib.pidlock import LockHeld, DaemonLock
         from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
         try:
             l = DaemonLock(file_=os.path.join(dn(dn(index_location)),
                                               'make_index.lock'))
             WhooshIndexingDaemon(index_location=index_location,
                                  repo_location=repo_location,
                                  repo_list=repo_list,
                                  repo_update_list=repo_update_list)\
                 .run(full_index=self.options.full_index)
             l.release()
         except LockHeld:
             sys.exit(1)
 
     def update_parser(self):
         self.parser.add_option('--repo-location',
                           action='store',
                           dest='repo_location',
                           help="Specifies repositories location to index OPTIONAL",
                           )
         self.parser.add_option('--index-only',
                           action='store',
                           dest='repo_list',
                           help="Specifies a comma separated list of repositores "
                                 "to build index on. If not given all repositories "
                                 "are scanned for indexing. OPTIONAL",
                           )
         self.parser.add_option('--update-only',
                           action='store',
                           dest='repo_update_list',
                           help="Specifies a comma separated list of repositores "
                                 "to re-build index on. OPTIONAL",
                           )
         self.parser.add_option('-f',
                           action='store_true',
                           dest='full_index',
                           help="Specifies that index should be made full i.e"
                                 " destroy old and build from scratch",
                           default=False)

rhodecode/lib/paster_commands/make_rcextensions.py

@@ -1,83 +1,83 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 rhodecode.lib.paster_commands.make_rcextensions
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 make-rcext paster command for RhodeCode
 
 :created_on: Mar 6, 2012
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH.
 :license: GPLv3, see LICENSE for more details.
 
 """
 
 from __future__ import with_statement
 
 import os
 import sys
 import logging
 import pkg_resources
 
 from rhodecode.lib.utils import BasePasterCommand, ask_ok
 
-# fix rhodecode import
+# Add location of top level folder to sys.path
 from os.path import dirname as dn
 rc_path = dn(dn(dn(os.path.realpath(__file__))))
 sys.path.append(rc_path)
 
 log = logging.getLogger(__name__)
 
 
 class Command(BasePasterCommand):
 
     max_args = 1
     min_args = 1
 
     usage = "CONFIG_FILE"
     group_name = "RhodeCode"
     takes_config_file = -1
     parser = BasePasterCommand.standard_parser(verbose=True)
     summary = "Creates additional extensions for rhodecode"
 
     def command(self):
         logging.config.fileConfig(self.path_to_ini_file)
         from pylons import config
 
         def _make_file(ext_file, tmpl):
             bdir = os.path.split(ext_file)[0]
             if not os.path.isdir(bdir):
                 os.makedirs(bdir)
             with open(ext_file, 'wb') as f:
                 f.write(tmpl)
                 log.info('Writen new extensions file to %s' % ext_file)
 
         here = config['here']
         tmpl = pkg_resources.resource_string(
             'rhodecode', os.path.join('config', 'rcextensions', '__init__.py')
         )
         ext_file = os.path.join(here, 'rcextensions', '__init__.py')
         if os.path.exists(ext_file):
             msg = ('Extension file already exists, do you want '
                    'to overwrite it ? [y/n]')
             if ask_ok(msg):
                 _make_file(ext_file, tmpl)
             else:
                 log.info('nothing done...')
         else:
             _make_file(ext_file, tmpl)
 
     def update_parser(self):
         pass

rhodecode/lib/paster_commands/repo_scan.py

@@ -1,71 +1,71 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 rhodecode.lib.paster_commands.make_rcextensions
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 repo-scan paster command for RhodeCode
 
 :created_on: Feb 9, 2013
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH.
 :license: GPLv3, see LICENSE for more details.
 """
 
 from __future__ import with_statement
 
 import os
 import sys
 import logging
 
 from rhodecode.model.scm import ScmModel
 from rhodecode.lib.utils import BasePasterCommand, repo2db_mapper
 
-# fix rhodecode import
+# Add location of top level folder to sys.path
 from os.path import dirname as dn
 rc_path = dn(dn(dn(os.path.realpath(__file__))))
 sys.path.append(rc_path)
 
 log = logging.getLogger(__name__)
 
 
 class Command(BasePasterCommand):
 
     max_args = 1
     min_args = 1
 
     usage = "CONFIG_FILE"
     group_name = "RhodeCode"
     takes_config_file = -1
     parser = BasePasterCommand.standard_parser(verbose=True)
     summary = "Rescan default location for new repositories"
 
     def command(self):
         #get SqlAlchemy session
         self._init_session()
         rm_obsolete = self.options.delete_obsolete
         log.info('Now scanning root location for new repos...')
         added, removed = repo2db_mapper(ScmModel().repo_scan(),
                                         remove_obsolete=rm_obsolete)
         added = ', '.join(added) or '-'
         removed = ', '.join(removed) or '-'
         log.info('Scan completed added: %s removed: %s' % (added, removed))
 
     def update_parser(self):
         self.parser.add_option(
             '--delete-obsolete',
             action='store_true',
             help="Use this flag do delete repositories that are "
                  "present in RhodeCode database but not on the filesystem",
         )

rhodecode/lib/paster_commands/setup_rhodecode.py

@@ -1,110 +1,110 b''
 import os
 import sys
 from paste.script.appinstall import AbstractInstallCommand
 from paste.script.command import BadCommand
 from paste.deploy import appconfig
 
-# fix rhodecode import
+# Add location of top level folder to sys.path
 from os.path import dirname as dn
 rc_path = dn(dn(dn(os.path.realpath(__file__))))
 sys.path.append(rc_path)
 
 
 class Command(AbstractInstallCommand):
 
     default_verbosity = 1
     max_args = 1
     min_args = 1
     summary = "Setup an application, given a config file"
     usage = "CONFIG_FILE"
     group_name = "RhodeCode"
 
     description = """\
 
     Setup RhodeCode according to its configuration file.  This is
     the second part of a two-phase web application installation
     process (the first phase is prepare-app).  The setup process
     consist of things like setting up databases, creating super user
     """
 
     parser = AbstractInstallCommand.standard_parser(
         simulate=True, quiet=True, interactive=True)
     parser.add_option('--user',
                       action='store',
                       dest='username',
                       default=None,
                       help='Admin Username')
     parser.add_option('--email',
                       action='store',
                       dest='email',
                       default=None,
                       help='Admin Email')
     parser.add_option('--password',
                       action='store',
                       dest='password',
                       default=None,
                       help='Admin password min 6 chars')
     parser.add_option('--repos',
                       action='store',
                       dest='repos_location',
                       default=None,
                       help='Absolute path to repositories location')
     parser.add_option('--name',
                       action='store',
                       dest='section_name',
                       default=None,
                       help='The name of the section to set up (default: app:main)')
     parser.add_option('--force-yes',
                        action='store_true',
                        dest='force_ask',
                        default=None,
                        help='Force yes to every question')
     parser.add_option('--force-no',
                        action='store_false',
                        dest='force_ask',
                        default=None,
                        help='Force no to every question')
     parser.add_option('--public-access',
                        action='store_true',
                        dest='public_access',
                        default=None,
                        help='Enable public access on this installation (default)')
     parser.add_option('--no-public-access',
                        action='store_false',
                        dest='public_access',
                        default=None,
                        help='Disable public access on this installation ')
     def command(self):
         config_spec = self.args[0]
         section = self.options.section_name
         if section is None:
             if '#' in config_spec:
                 config_spec, section = config_spec.split('#', 1)
             else:
                 section = 'main'
         if not ':' in section:
             plain_section = section
             section = 'app:' + section
         else:
             plain_section = section.split(':', 1)[0]
         if not config_spec.startswith('config:'):
             config_spec = 'config:' + config_spec
         if plain_section != 'main':
             config_spec += '#' + plain_section
         config_file = config_spec[len('config:'):].split('#', 1)[0]
         config_file = os.path.join(os.getcwd(), config_file)
         self.logging_file_config(config_file)
         conf = appconfig(config_spec, relative_to=os.getcwd())
         ep_name = conf.context.entry_point_name
         ep_group = conf.context.protocol
         dist = conf.context.distribution
         if dist is None:
             raise BadCommand(
                 "The section %r is not the application (probably a filter).  "
                 "You should add #section_name, where section_name is the "
                 "section that configures your application" % plain_section)
         installer = self.get_installer(dist, ep_group, ep_name)
         installer.setup_config(
             self, config_file, section, self.sysconfig_install_vars(installer))
         self.call_sysconfig_functions(
             'post_setup_hook', installer, config_file)

rhodecode/lib/paster_commands/update_repoinfo.py

@@ -1,88 +1,88 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 rhodecode.lib.paster_commands.make_rcextensions
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 uodate-repoinfo paster command for RhodeCode
 
 :created_on: Jul 14, 2012
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH.
 :license: GPLv3, see LICENSE for more details.
 """
 
 from __future__ import with_statement
 
 import os
 import sys
 import logging
 import string
 
 from rhodecode.lib.utils import BasePasterCommand
 from rhodecode.model.db import Repository
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.meta import Session
 
-# fix rhodecode import
+# Add location of top level folder to sys.path
 from os.path import dirname as dn
 rc_path = dn(dn(dn(os.path.realpath(__file__))))
 sys.path.append(rc_path)
 
 log = logging.getLogger(__name__)
 
 
 class Command(BasePasterCommand):
 
     max_args = 1
     min_args = 1
 
     usage = "CONFIG_FILE"
     group_name = "RhodeCode"
     takes_config_file = -1
     parser = BasePasterCommand.standard_parser(verbose=True)
     summary = "Updates repositories caches for last changeset"
 
     def command(self):
         #get SqlAlchemy session
         self._init_session()
 
         repo_update_list = map(string.strip,
                                self.options.repo_update_list.split(',')) \
                                if self.options.repo_update_list else None
 
         if repo_update_list:
             repo_list = Repository.query()\
                 .filter(Repository.repo_name.in_(repo_update_list))
         else:
             repo_list = Repository.getAll()
         RepoModel.update_repoinfo(repositories=repo_list)
         Session().commit()
 
         if self.options.invalidate_cache:
             for r in repo_list:
                 r.set_invalidate()
         log.info('Updated cache for %s repositories' % (len(repo_list)))
 
     def update_parser(self):
         self.parser.add_option('--update-only',
                            action='store',
                            dest='repo_update_list',
                            help="Specifies a comma separated list of repositores "
                                 "to update last commit info for. OPTIONAL")
         self.parser.add_option('--invalidate-cache',
                            action='store_true',
                            dest='invalidate_cache',
                            help="Trigger cache invalidation event for repos. "
                                 "OPTIONAL")