Hacking for git support,and new faster repo scan

docs/changelog.rst

@@ -3,6 +3,12 b''
 Changelog
 =========
 
+1.1.0 (**XXXX-XX-XX**)
+----------------------
+- git support
+- performance upgrade for cached repos list
+
+
 1.0.0 (**2010-10-xx**)
 ----------------------
 

rhodecode/__init__.py

@@ -24,7 +24,7 b' versioning implementation: http://semver'
 @author: marcink
 """
 
-VERSION = (1, 0, 0, 'rc4')
+VERSION = (1, 1, 0, 'beta')
 
 __version__ = '.'.join((str(each) for each in VERSION[:4]))
 

rhodecode/config/environment.py

@@ -53,7 +53,7 b' def load_environment(global_conf, app_co'
     if test:
         from rhodecode.lib.utils import create_test_env, create_test_index
         create_test_env('/tmp', config)
-        create_test_index('/tmp/*', True)
+        create_test_index('/tmp', True)
         
     #MULTIPLE DB configs
     # Setup the SQLAlchemy database engine

rhodecode/lib/app_globals.py

@@ -28,4 +28,4 b' class Globals(object):'
     @LazyProperty
     def base_path(self):
         if self.baseui:
-            return self.paths[0][1].replace('*', '')            
+            return self.paths[0][1]

rhodecode/lib/celerylib/tasks.py

@@ -8,7 +8,11 b' from rhodecode.lib.smtp_mailer import Sm'
 from rhodecode.lib.utils import OrderedDict
 from time import mktime
 from vcs.backends.hg import MercurialRepository
+from vcs.backends.git import GitRepository
+import os
 import traceback
+from vcs.backends import get_repo
+from vcs.utils.helpers import get_scm
 
 try:
     import json
@@ -95,8 +99,9 b' def get_commits_stats(repo_name, ts_min_'
 
     commits_by_day_author_aggregate = {}
     commits_by_day_aggregate = {}
-    repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '')
-    repo = MercurialRepository(repos_path + repo_name)
+    repos_path = get_hg_ui_settings()['paths_root_path']
+    p = os.path.join(repos_path, repo_name)
+    repo = get_repo(get_scm(p)[0], p)
 
     skip_date_limit = True
     parse_limit = 250 #limit for single task changeset parsing optimal for
@@ -305,8 +310,10 b' def __get_codes_stats(repo_name):'
     's', 'sh', 'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt', 'yaws']
 
 
-    repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '')
-    repo = MercurialRepository(repos_path + repo_name)
+    repos_path = get_hg_ui_settings()['paths_root_path']
+    p = os.path.join(repos_path, repo_name)
+    repo = get_repo(get_scm(p)[0], p)
+
     tip = repo.get_changeset()
 
     code_stats = {}

rhodecode/lib/db_manage.py

@@ -162,7 +162,7 b' class DbManage(object):'
         paths = RhodeCodeUi()
         paths.ui_section = 'paths'
         paths.ui_key = '/'
-        paths.ui_value = os.path.join(path, '*')
+        paths.ui_value = path
 
 
         hgsettings1 = RhodeCodeSettings()

rhodecode/lib/helpers.py

@@ -323,7 +323,7 b' flash = _Flash()'
 from mercurial import util
 from mercurial.templatefilters import age as _age, person as _person
 
-age = lambda  x:_age(x)
+age = lambda  x:x
 capitalize = lambda x: x.capitalize()
 date = lambda x: util.datestr(x)
 email = util.email
@@ -333,8 +333,8 b' hgdate = lambda x: "%d %d" % x'
 isodate = lambda  x: util.datestr(x, '%Y-%m-%d %H:%M %1%2')
 isodatesec = lambda  x: util.datestr(x, '%Y-%m-%d %H:%M:%S %1%2')
 localdate = lambda  x: (x[0], util.makedate()[1])
-rfc822date = lambda  x: util.datestr(x, "%a, %d %b %Y %H:%M:%S %1%2")
-rfc822date_notz = lambda  x: util.datestr(x, "%a, %d %b %Y %H:%M:%S")
+rfc822date = lambda  x: x#util.datestr(x, "%a, %d %b %Y %H:%M:%S %1%2")
+rfc822date_notz = lambda  x: x#util.datestr(x, "%a, %d %b %Y %H:%M:%S")
 rfc3339date = lambda  x: util.datestr(x, "%Y-%m-%dT%H:%M:%S%1:%2")
 time_ago = lambda x: util.datestr(_age(x), "%a, %d %b %Y %H:%M:%S %1%2")
 

rhodecode/lib/indexers/__init__.py

@@ -1,4 +1,10 b''
+import os
+import sys
 from os.path import dirname as dn, join as jn
+
+#to get the rhodecode import
+sys.path.append(dn(dn(dn(os.path.realpath(__file__)))))
+
 from rhodecode.config.environment import load_environment
 from rhodecode.model.hg import HgModel
 from shutil import rmtree
@@ -11,13 +17,8 b' from whoosh.index import create_in, open'
 from whoosh.formats import Characters
 from whoosh.highlight import highlight, SimpleFragmenter, HtmlFormatter   
 
-import os
-import sys
 import traceback
 
-#to get the rhodecode import
-sys.path.append(dn(dn(dn(os.path.realpath(__file__)))))
-
 
 #LOCATION WE KEEP THE INDEX
 IDX_LOCATION = jn(dn(dn(dn(dn(os.path.abspath(__file__))))), 'data', 'index')
@@ -48,6 +49,59 b" IDX_NAME = 'HG_INDEX'"
 FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n') 
 FRAGMENTER = SimpleFragmenter(200)
                             
+from paste.script import command
+import ConfigParser
+
+class MakeIndex(command.Command):
+
+    max_args = 1
+    min_args = 1
+
+    usage = "CONFIG_FILE"
+    summary = "Creates index for full text search given configuration file"
+    group_name = "Whoosh indexing"
+
+    parser = command.Command.standard_parser(verbose=True)
+#    parser.add_option('--repo-location',
+#                      action='store',
+#                      dest='repo_location',
+#                      help="Specifies repositories location to index",
+#                      )
+    parser.add_option('-f',
+                      action='store_true',
+                      dest='full_index',
+                      help="Specifies that index should be made full i.e"
+                            " destroy old and build from scratch",
+                      default=False)
+    def command(self):
+        config_name = self.args[0]
+
+        p = config_name.split('/')
+        if len(p) == 1:
+            root = '.'
+        else:
+            root = '/'.join(p[:-1])
+        print root
+        config = ConfigParser.ConfigParser({'here':root})
+        config.read(config_name)
+        print dict(config.items('app:main'))['index_dir']
+        index_location = dict(config.items('app:main'))['index_dir']
+        #return
+
+        #=======================================================================
+        # WHOOSH DAEMON
+        #=======================================================================
+        from rhodecode.lib.pidlock import LockHeld, DaemonLock
+        from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
+        try:
+            l = DaemonLock()
+            WhooshIndexingDaemon(index_location=index_location)\
+                .run(full_index=self.options.full_index)
+            l.release()
+        except LockHeld:
+            sys.exit(1)
+
+
 class ResultWrapper(object):
     def __init__(self, search_type, searcher, matcher, highlight_items):
         self.search_type = search_type
@@ -115,8 +169,8 b' class ResultWrapper(object):'
         Smart function that implements chunking the content
         but not overlap chunks so it doesn't highlight the same
         close occurrences twice.
-        :param matcher:
-        :param size:
+        @param matcher:
+        @param size:
         """
         memory = [(0, 0)]
         for span in self.matcher.spans():

rhodecode/lib/indexers/daemon.py

@@ -32,12 +32,12 b' from os.path import join as jn'
 project_path = dn(dn(dn(dn(os.path.realpath(__file__)))))
 sys.path.append(project_path)
 
-from rhodecode.lib.pidlock import LockHeld, DaemonLock
+
 from rhodecode.model.hg import HgModel
 from rhodecode.lib.helpers import safe_unicode
 from whoosh.index import create_in, open_dir
 from shutil import rmtree
-from rhodecode.lib.indexers import INDEX_EXTENSIONS, IDX_LOCATION, SCHEMA, IDX_NAME
+from rhodecode.lib.indexers import INDEX_EXTENSIONS, SCHEMA, IDX_NAME
 
 from time import mktime
 from vcs.exceptions import ChangesetError, RepositoryError
@@ -61,21 +61,33 b' ch.setFormatter(formatter)'
 # add ch to logger
 log.addHandler(ch)
 
-def scan_paths(root_location):
-    return HgModel.repo_scan('/', root_location, None, True)
+def get_repos_location():
+    return HgModel.get_repos_location()
+
 
 class WhooshIndexingDaemon(object):
     """
     Deamon for atomic jobs
     """
 
-    def __init__(self, indexname='HG_INDEX', repo_location=None):
+    def __init__(self, indexname='HG_INDEX', index_location=None,
+                 repo_location=None):
         self.indexname = indexname
+
+        self.index_location = index_location
+        if not index_location:
+            raise Exception('You have to provide index location')
+
         self.repo_location = repo_location
-        self.repo_paths = scan_paths(self.repo_location)
+        if not repo_location:
+            raise Exception('You have to provide repositories location')
+
+
+
+        self.repo_paths = HgModel.repo_scan('/', self.repo_location, None, True)
         self.initial = False
-        if not os.path.isdir(IDX_LOCATION):
-            os.mkdir(IDX_LOCATION)
+        if not os.path.isdir(self.index_location):
+            os.mkdir(self.index_location)
             log.info('Cannot run incremental index since it does not'
                      ' yet exist running full build')
             self.initial = True
@@ -87,9 +99,7 b' class WhooshIndexingDaemon(object):'
         """
         index_paths_ = set()
         try:
-            tip = repo.get_changeset()
-            
-            for topnode, dirs, files in tip.walk('/'):
+            for topnode, dirs, files in repo.walk('/', 'tip'):
                 for f in files:
                     index_paths_.add(jn(repo.path, f.path))
                 for dir in dirs:
@@ -130,14 +140,14 b' class WhooshIndexingDaemon(object):'
 
     
     def build_index(self):
-        if os.path.exists(IDX_LOCATION):
+        if os.path.exists(self.index_location):
             log.debug('removing previous index')
-            rmtree(IDX_LOCATION)
+            rmtree(self.index_location)
             
-        if not os.path.exists(IDX_LOCATION):
-            os.mkdir(IDX_LOCATION)
+        if not os.path.exists(self.index_location):
+            os.mkdir(self.index_location)
         
-        idx = create_in(IDX_LOCATION, SCHEMA, indexname=IDX_NAME)
+        idx = create_in(self.index_location, SCHEMA, indexname=IDX_NAME)
         writer = idx.writer()
         
         for cnt, repo in enumerate(self.repo_paths.values()):
@@ -154,7 +164,7 b' class WhooshIndexingDaemon(object):'
     def update_index(self):
         log.debug('STARTING INCREMENTAL INDEXING UPDATE')
             
-        idx = open_dir(IDX_LOCATION, indexname=self.indexname)
+        idx = open_dir(self.index_location, indexname=self.indexname)
         # The set of all paths in the index
         indexed_paths = set()
         # The set of all paths we need to re-index
@@ -209,40 +219,3 b' class WhooshIndexingDaemon(object):'
             self.build_index()
         else:
             self.update_index()
-        
-if __name__ == "__main__":
-    arg = sys.argv[1:]
-    if len(arg) != 2:
-        sys.stderr.write('Please specify indexing type [full|incremental]' 
-                         'and path to repositories as script args \n')
-        sys.exit()
-    
-    
-    if arg[0] == 'full':
-        full_index = True
-    elif arg[0] == 'incremental':
-        # False means looking just for changes
-        full_index = False
-    else:
-        sys.stdout.write('Please use [full|incremental]' 
-                         ' as script first arg \n')
-        sys.exit()
-    
-    if not os.path.isdir(arg[1]):
-        sys.stderr.write('%s is not a valid path \n' % arg[1])
-        sys.exit()
-    else:
-        if arg[1].endswith('/'):
-            repo_location = arg[1] + '*'
-        else:
-            repo_location = arg[1] + '/*'
-    
-    try:
-        l = DaemonLock()
-        WhooshIndexingDaemon(repo_location=repo_location)\
-            .run(full_index=full_index)
-        l.release()
-        reload(logging)
-    except LockHeld:
-        sys.exit(1)
-

rhodecode/lib/utils.py

@@ -16,24 +16,28 b''
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
+from UserDict import DictMixin
+from mercurial import ui, config, hg
+from mercurial.error import RepoError
+from rhodecode.model import meta
+from rhodecode.model.caching_query import FromCache
+from rhodecode.model.db import Repository, User, RhodeCodeUi, RhodeCodeSettings, \
+    UserLog
+from rhodecode.model.repo import RepoModel
+from rhodecode.model.user import UserModel
+from vcs.backends.base import BaseChangeset
+from vcs.backends.git import GitRepository
+from vcs.backends.hg import MercurialRepository
+from vcs.utils.lazy import LazyProperty
+import datetime
+import logging
+import os
 
 """
 Created on April 18, 2010
 Utilities for RhodeCode
 @author: marcink
 """
-from rhodecode.model.caching_query import FromCache
-from mercurial import ui, config, hg
-from mercurial.error import RepoError
-from rhodecode.model import meta
-from rhodecode.model.user import UserModel
-from rhodecode.model.repo import RepoModel
-from rhodecode.model.db import Repository, User, RhodeCodeUi, RhodeCodeSettings, UserLog
-from vcs.backends.base import BaseChangeset
-from vcs.utils.lazy import LazyProperty
-import logging
-import datetime
-import os
 
 log = logging.getLogger(__name__)
 
@@ -96,14 +100,30 b' def action_logger(user, action, repo, ip'
         sa.rollback()
         log.error('could not log user action:%s', str(e))
 
-def check_repo_dir(paths):
-    repos_path = paths[0][1].split('/')
-    if repos_path[-1] in ['*', '**']:
-        repos_path = repos_path[:-1]
-    if repos_path[0] != '/':
-        repos_path[0] = '/'
-    if not os.path.isdir(os.path.join(*repos_path)):
-        raise Exception('Not a valid repository in %s' % paths[0][1])
+def get_repos(path, recursive=False, initial=False):
+    """
+    Scans given path for repos and return (name,(type,path)) tuple 
+    :param prefix:
+    :param path:
+    :param recursive:
+    :param initial:
+    """
+    from vcs.utils.helpers import get_scm
+    from vcs.exceptions import VCSError
+    scm = get_scm(path)
+    if scm:
+        raise Exception('The given path %s should not be a repository got %s',
+                        path, scm)
+
+    for dirpath in os.listdir(path):
+        try:
+            yield dirpath, get_scm(os.path.join(path, dirpath))
+        except VCSError:
+            pass
+
+if __name__ == '__main__':
+    get_repos('', '/home/marcink/workspace-python')
+
 
 def check_repo_fast(repo_name, base_path):
     if os.path.isdir(os.path.join(base_path, repo_name)):return False
@@ -231,8 +251,6 b" def make_ui(read_from='file', path=None,"
             for k, v in cfg.items(section):
                 baseui.setconfig(section, k, v)
                 log.debug('settings ui from file[%s]%s:%s', section, k, v)
-        if checkpaths:check_repo_dir(cfg.items('paths'))
-
 
     elif read_from == 'db':
         hg_ui = get_hg_ui_cached()
@@ -284,7 +302,7 b' class EmptyChangeset(BaseChangeset):'
     @LazyProperty
     def raw_id(self):
         """
-        Returns raw string identifing this changeset, useful for web
+        Returns raw string identifying this changeset, useful for web
         representation.
         """
         return '0' * 40
@@ -308,16 +326,21 b' def repo2db_mapper(initial_repo_list, re'
     """
 
     sa = meta.Session()
+    rm = RepoModel(sa)
     user = sa.query(User).filter(User.admin == True).first()
 
-    rm = RepoModel()
+    for name, repo in initial_repo_list.items():
+        if not rm.get(name, cache=False):
+            log.info('repository %s not found creating default', name)
 
-    for name, repo in initial_repo_list.items():
-        if not RepoModel(sa).get(name, cache=False):
-            log.info('repository %s not found creating default', name)
+            if isinstance(repo, MercurialRepository):
+                repo_type = 'hg'
+            if isinstance(repo, GitRepository):
+                repo_type = 'git'
 
             form_data = {
                          'repo_name':name,
+                         'repo_type':repo_type,
                          'description':repo.description if repo.description != 'unknown' else \
                                         'auto description for %s' % name,
                          'private':False
@@ -335,7 +358,6 b' def repo2db_mapper(initial_repo_list, re'
 
     meta.Session.remove()
 
-from UserDict import DictMixin
 
 class OrderedDict(dict, DictMixin):
 

rhodecode/model/db.py

@@ -81,6 +81,7 b' class Repository(Base):'
     __table_args__ = (UniqueConstraint('repo_name'), {'useexisting':True},)
     repo_id = Column("repo_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
     repo_name = Column("repo_name", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=False, unique=True, default=None)
+    repo_type = Column("repo_type", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=False, unique=False, default=None)
     user_id = Column("user_id", INTEGER(), ForeignKey(u'users.user_id'), nullable=False, unique=False, default=None)
     private = Column("private", BOOLEAN(), nullable=True, unique=None, default=None)
     description = Column("description", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)

rhodecode/model/forms.py

@@ -194,16 +194,12 b' class ValidSettings(formencode.validator'
 
 class ValidPath(formencode.validators.FancyValidator):
     def to_python(self, value, state):
-        isdir = os.path.isdir(value.replace('*', ''))
-        if (value.endswith('/*') or value.endswith('/**')) and isdir:
-            return value
-        elif not isdir:
+
+        if not os.path.isdir(value):
             msg = _('This is not a valid path')
-        else:
-            msg = _('You need to specify * or ** at the end of path (ie. /tmp/*)')
-
         raise formencode.Invalid(msg, value, state,
                                      error_dict={'paths_root_path':msg})
+        return value
 
 def UniqSystemEmail(old_data):
     class _UniqSystemEmail(formencode.validators.FancyValidator):

rhodecode/model/hg.py

@@ -24,7 +24,6 b' Model for RhodeCode'
 """
 from beaker.cache import cache_region
 from mercurial import ui
-from mercurial.hgweb.hgwebdir_mod import findrepos
 from rhodecode.lib import helpers as h
 from rhodecode.lib.utils import invalidate_cache
 from rhodecode.lib.auth import HasRepoPermissionAny
@@ -33,12 +32,12 b' from rhodecode.model.db import Repositor'
 from sqlalchemy.orm import joinedload
 from vcs.exceptions import RepositoryError, VCSError
 import logging
-import os
 import sys
 log = logging.getLogger(__name__)
 
 try:
     from vcs.backends.hg import MercurialRepository
+    from vcs.backends.git import GitRepository
 except ImportError:
     sys.stderr.write('You have to import vcs module')
     raise Exception('Unable to import vcs')
@@ -47,7 +46,7 b' def _get_repos_cached_initial(app_global'
     """return cached dict with repos
     """
     g = app_globals
-    return HgModel.repo_scan(g.paths[0][0], g.paths[0][1], g.baseui, initial)
+    return HgModel().repo_scan(g.paths[0][1], g.baseui, initial)
 
 @cache_region('long_term', 'cached_repo_list')
 def _get_repos_cached():
@@ -55,7 +54,7 b' def _get_repos_cached():'
     """
     log.info('getting all repositories list')
     from pylons import app_globals as g
-    return HgModel.repo_scan(g.paths[0][0], g.paths[0][1], g.baseui)
+    return HgModel().repo_scan(g.paths[0][1], g.baseui)
 
 @cache_region('super_short_term', 'cached_repos_switcher_list')
 def _get_repos_switcher_cached(cached_repo_list):
@@ -73,42 +72,34 b' def _full_changelog_cached(repo_name):'
     return list(reversed(list(HgModel().get_repo(repo_name))))
 
 class HgModel(object):
-    """Mercurial Model
+    """
+    Mercurial Model
     """
 
-    def __init__(self):
-        pass
+    def __init__(self, sa=None):
+        if not sa:
+            self.sa = meta.Session()
+        else:
+            self.sa = sa
 
-    @staticmethod
-    def repo_scan(repos_prefix, repos_path, baseui, initial=False):
+    def repo_scan(self, repos_path, baseui, initial=False):
         """
         Listing of repositories in given path. This path should not be a 
         repository itself. Return a dictionary of repository objects
-        :param repos_path: path to directory it could take syntax with 
-        * or ** for deep recursive displaying repositories
-        """
-        sa = meta.Session()
-        def check_repo_dir(path):
-            """Checks the repository
-            :param path:
+        
+        :param repos_path: path to directory containing repositories
+        :param baseui
+        :param initial: initial scann
             """
-            repos_path = path.split('/')
-            if repos_path[-1] in ['*', '**']:
-                repos_path = repos_path[:-1]
-            if repos_path[0] != '/':
-                repos_path[0] = '/'
-            if not os.path.isdir(os.path.join(*repos_path)):
-                raise RepositoryError('Not a valid repository in %s' % path)
-        if not repos_path.endswith('*'):
-            raise VCSError('You need to specify * or ** at the end of path '
-                            'for recursive scanning')
+        log.info('scanning for repositories in %s', repos_path)
 
-        check_repo_dir(repos_path)
-        log.info('scanning for repositories in %s', repos_path)
-        repos = findrepos([(repos_prefix, repos_path)])
         if not isinstance(baseui, ui.ui):
             baseui = ui.ui()
 
+        from rhodecode.lib.utils import get_repos
+        repos = get_repos(repos_path)
+
+
         repos_list = {}
         for name, path in repos:
             try:
@@ -117,15 +108,19 b' class HgModel(object):'
                     raise RepositoryError('Duplicate repository name %s found in'
                                     ' %s' % (name, path))
                 else:
+                    if path[0] == 'hg':
+                        repos_list[name] = MercurialRepository(path[1], baseui=baseui)
+                        repos_list[name].name = name
 
-                    repos_list[name] = MercurialRepository(path, baseui=baseui)
+                    if path[0] == 'git':
+                        repos_list[name] = GitRepository(path[1])
                     repos_list[name].name = name
 
                     dbrepo = None
                     if not initial:
                         #for initial scann on application first run we don't
                         #have db repos yet.
-                        dbrepo = sa.query(Repository)\
+                        dbrepo = self.sa.query(Repository)\
                             .options(joinedload(Repository.fork))\
                             .filter(Repository.repo_name == name)\
                             .scalar()
@@ -137,16 +132,17 b' class HgModel(object):'
                         if dbrepo.user:
                             repos_list[name].contact = dbrepo.user.full_contact
                         else:
-                            repos_list[name].contact = sa.query(User)\
+                            repos_list[name].contact = self.sa.query(User)\
                             .filter(User.admin == True).first().full_contact
             except OSError:
                 continue
-        meta.Session.remove()
+
         return repos_list
 
     def get_repos(self):
         for name, repo in _get_repos_cached().items():
-            if repo._get_hidden():
+
+            if isinstance(repo, MercurialRepository) and repo._get_hidden():
                 #skip hidden web repository
                 continue
 

rhodecode/templates/shortlog/shortlog_data.html

@@ -13,7 +13,7 b''
 	</tr>
 %for cnt,cs in enumerate(c.repo_changesets):
 	<tr class="parity${cnt%2}">
-		<td>${h.age(cs._ctx.date())} - ${h.rfc822date_notz(cs._ctx.date())} </td>
+		<td>${h.age(cs.date)} - ${h.rfc822date_notz(cs.date)} </td>
 		<td title="${cs.author}">${h.person(cs.author)}</td>
 		<td>r${cs.revision}:${cs.short_id}</td>
 		<td>