Reimplemented way of caching repos list, hg model now get's repos objects right from cached dict, this way we skipp creating instances of MercurialRepository and gain performance. Some import cleanup

pylons_app/controllers/branches.py

@@ -1,15 +1,11 b''
+from pylons import tmpl_context as c, app_globals as g
+from pylons_app.lib.auth import LoginRequired
+from pylons_app.lib.base import BaseController, render
+from pylons_app.model.hg_model import HgModel
 import logging
 
-from pylons import tmpl_context as c, app_globals as g, session, request, config, url
-from pylons.controllers.util import abort, redirect
-
-from pylons_app.lib.base import BaseController, render
-from pylons_app.lib.utils import get_repo_slug
-from pylons_app.model.hg_model import HgModel
-from pylons_app.lib.auth import LoginRequired
 log = logging.getLogger(__name__)
 
-
 class BranchesController(BaseController):
     
     @LoginRequired()

pylons_app/controllers/changelog.py

@@ -1,11 +1,10 b''
 from mercurial.graphmod import revisions as graph_rev, colored, CHANGESET
 from mercurial.node import short
-from pylons import request, response, session, tmpl_context as c, url, config, \
-    app_globals as g
-from pylons.controllers.util import abort, redirect
+from pylons import request, session, tmpl_context as c
 from pylons_app.lib.auth import LoginRequired
-from pylons_app.lib.base import BaseController, render, _full_changelog_cached
+from pylons_app.lib.base import BaseController, render
 from pylons_app.lib.filters import age as _age, person
+from pylons_app.model.hg_model import _full_changelog_cached
 from simplejson import dumps
 from webhelpers.paginate import Page
 import logging

pylons_app/controllers/changeset.py

@@ -1,6 +1,4 b''
-from pylons import request, response, session, tmpl_context as c, url, config, \
-    app_globals as g
-from pylons.controllers.util import abort, redirect
+from pylons import tmpl_context as c
 from pylons_app.lib.auth import LoginRequired
 from pylons_app.lib.base import BaseController, render
 from pylons_app.model.hg_model import HgModel

pylons_app/controllers/feed.py

@@ -1,13 +1,10 b''
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
+from pylons import tmpl_context as c, url, response
+from pylons_app.lib.base import BaseController, render
+from pylons_app.model.hg_model import _full_changelog_cached
+from webhelpers.feedgenerator import Atom1Feed, Rss201rev2Feed
 import logging
-from operator import itemgetter
-from pylons import tmpl_context as c, request, config, url, response
-from pylons_app.lib.base import BaseController, render, _full_changelog_cached
-from pylons_app.lib.utils import get_repo_slug
-from pylons_app.model.hg_model import HgModel
-from pylons_app.lib.auth import LoginRequired
-from webhelpers.feedgenerator import Atom1Feed, Rss201rev2Feed
 log = logging.getLogger(__name__)
 
 class FeedController(BaseController):
@@ -35,8 +32,9 b' class FeedController(BaseController):'
             if cnt > self.feed_nr:
                 break
             feed.add_item(title=cs.message,
-                          link=url('changeset_home', repo_name=repo_name, revision=cs.raw_id, qualified=True),
-                          description=str(cs.date))
+                          link=url('changeset_home', repo_name=repo_name,
+                                   revision=cs.raw_id, qualified=True),
+                                   description=str(cs.date))
         
         response.content_type = feed.mime_type
         return feed.writeString('utf-8')

pylons_app/controllers/files.py

@@ -1,15 +1,12 b''
-import tempfile
-from pylons import request, response, session, tmpl_context as c, url, config, \
-    app_globals as g
-from pylons.controllers.util import abort, redirect
+from mercurial import archival
+from pylons import request, response, session, tmpl_context as c, url
 from pylons_app.lib.auth import LoginRequired
 from pylons_app.lib.base import BaseController, render
-from pylons_app.lib.utils import get_repo_slug
 from pylons_app.model.hg_model import HgModel
 from vcs.exceptions import RepositoryError, ChangesetError
 from vcs.utils import diffs as differ
 import logging
-from mercurial import archival
+import tempfile
 
         
 log = logging.getLogger(__name__)

pylons_app/controllers/hg.py

@@ -1,10 +1,11 b''
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
-import logging
 from operator import itemgetter
 from pylons import tmpl_context as c, request, config
+from pylons_app.lib.auth import LoginRequired
 from pylons_app.lib.base import BaseController, render
-from pylons_app.lib.auth import LoginRequired
+from pylons_app.model.hg_model import HgModel
+import logging
 log = logging.getLogger(__name__)
 
 class HgController(BaseController):
@@ -18,12 +19,12 b' class HgController(BaseController):'
         cs = c.current_sort
         c.cs_slug = cs.replace('-', '')
         sortables = ['name', 'description', 'last_change', 'tip', 'contact']
-        
+        cached_repo_list = HgModel().get_repos()
         if cs and c.cs_slug in sortables:
             sort_key = c.cs_slug + '_sort'
             if cs.startswith('-'):
-                c.repos_list = sorted(c.cached_repo_list, key=itemgetter(sort_key), reverse=True)
+                c.repos_list = sorted(cached_repo_list, key=itemgetter(sort_key), reverse=True)
             else:
-                c.repos_list = sorted(c.cached_repo_list, key=itemgetter(sort_key), reverse=False)
+                c.repos_list = sorted(cached_repo_list, key=itemgetter(sort_key), reverse=False)
             
         return render('/index.html')

pylons_app/controllers/shortlog.py

@@ -1,13 +1,9 b''
-import logging
-
-from pylons import tmpl_context as c, app_globals as g, session, request, config, url
-from pylons.controllers.util import abort, redirect
-
+from pylons import tmpl_context as c, request
+from pylons_app.lib.auth import LoginRequired
 from pylons_app.lib.base import BaseController, render
-from pylons_app.lib.utils import get_repo_slug
 from pylons_app.model.hg_model import HgModel
 from webhelpers.paginate import Page
-from pylons_app.lib.auth import LoginRequired
+import logging
 
 log = logging.getLogger(__name__)
 

pylons_app/controllers/summary.py

@@ -1,7 +1,7 b''
 from pylons import tmpl_context as c, request
 from pylons_app.lib.auth import LoginRequired
-from pylons_app.lib.base import BaseController, render, _full_changelog_cached
-from pylons_app.model.hg_model import HgModel
+from pylons_app.lib.base import BaseController, render
+from pylons_app.model.hg_model import HgModel, _full_changelog_cached
 import logging
 
 log = logging.getLogger(__name__)
@@ -16,7 +16,6 b' class SummaryController(BaseController):'
         hg_model = HgModel()
         c.repo_info = hg_model.get_repo(c.repo_name)
         c.repo_changesets = _full_changelog_cached(c.repo_name)[:10]
-        
         e = request.environ
         uri = u'%(protocol)s://%(user)s@%(host)s/%(repo_name)s' % {
                                         'protocol': e.get('wsgi.url_scheme'),

pylons_app/controllers/tags.py

@@ -1,15 +1,11 b''
+from pylons import tmpl_context as c
+from pylons_app.lib.auth import LoginRequired
+from pylons_app.lib.base import BaseController, render
+from pylons_app.model.hg_model import HgModel
 import logging
 
-from pylons import tmpl_context as c, app_globals as g, session, request, config, url
-from pylons.controllers.util import abort, redirect
-
-from pylons_app.lib.base import BaseController, render
-from pylons_app.lib.utils import get_repo_slug
-from pylons_app.model.hg_model import HgModel
-from pylons_app.lib.auth import LoginRequired
 log = logging.getLogger(__name__)
 
-
 class TagsController(BaseController):
     
     @LoginRequired()

pylons_app/controllers/users.py

@@ -1,9 +1,8 b''
 from formencode import htmlfill
-from pylons import request, response, session, tmpl_context as c, url, \
-    app_globals as g
+from pylons import request, session, tmpl_context as c, url
+from pylons.controllers.util import abort, redirect
 from pylons.i18n.translation import _
-from pylons_app.lib import helpers as h    
-from pylons.controllers.util import abort, redirect
+from pylons_app.lib import helpers as h
 from pylons_app.lib.auth import LoginRequired, CheckPermissionAll
 from pylons_app.lib.base import BaseController, render
 from pylons_app.model.db import User, UserLog
@@ -12,8 +11,6 b' from pylons_app.model.user_model import '
 import formencode
 import logging
 
-
-
 log = logging.getLogger(__name__)
 
 class UsersController(BaseController):

pylons_app/lib/base.py

@@ -2,24 +2,15 b''
 
 Provides the BaseController class for subclassing.
 """
-from beaker.cache import cache_region
 from pylons import config, tmpl_context as c, request, session
 from pylons.controllers import WSGIController
 from pylons.templating import render_mako as render
 from pylons_app.lib.auth import LoginRequired, AuthUser
 from pylons_app.lib.utils import get_repo_slug
 from pylons_app.model import meta
-from pylons_app.model.hg_model import HgModel
+from pylons_app.model.hg_model import _get_repos_cached
 from pylons_app import __version__
 
-@cache_region('long_term', 'cached_repo_list')
-def _get_repos_cached():
-    return [rep for rep in HgModel().get_repos()]
-
-@cache_region('long_term', 'full_changelog')
-def _full_changelog_cached(repo_name):
-    return list(reversed(list(HgModel().get_repo(repo_name))))  
-
 class BaseController(WSGIController):
     
     def __before__(self):

pylons_app/lib/utils.py

@@ -107,11 +107,11 b' def invalidate_cache(name, *args):'
     args = tuple(tmp)
     
     if name == 'cached_repo_list':
-        from pylons_app.lib.base import _get_repos_cached
+        from pylons_app.model.hg_model import _get_repos_cached
         region_invalidate(_get_repos_cached, None, *args)
         
     if name == 'full_changelog':
-        from pylons_app.lib.base import _full_changelog_cached
+        from pylons_app.model.hg_model import _full_changelog_cached
         region_invalidate(_full_changelog_cached, None, *args)
         
 from vcs.backends.base import BaseChangeset
@@ -128,3 +128,17 b' class EmptyChangeset(BaseChangeset):'
         """
         return '0' * 12
 
+
+def repo2db_mapper():
+    """
+    put !
+    """
+    pass
+    #scann all dirs for .hgdbid
+    #if some dir doesn't have one generate one.
+    #
+    
+    
+    
+    
+    

pylons_app/model/hg_model.py

@@ -3,65 +3,118 b''
 #
 # Copyright (c) 2010 marcink.  All rights reserved.
 #
-from vcs.exceptions import RepositoryError
 '''
 Created on Apr 9, 2010
 
 @author: marcink
 '''
+
+from beaker.cache import cache_region
+from mercurial import ui
+from mercurial.hgweb.hgwebdir_mod import findrepos
+from pylons import app_globals as g
+from vcs.exceptions import RepositoryError, VCSError
+import logging
 import os
-from pylons import tmpl_context as c, app_globals as g, session, request, config
-from pylons.controllers.util import abort
 import sys
+log = logging.getLogger(__name__)
+
 try:
-    from vcs.backends.hg import get_repositories, MercurialRepository
+    from vcs.backends.hg import MercurialRepository
 except ImportError:
     sys.stderr.write('You have to import vcs module')
     raise Exception('Unable to import vcs')
 
+
+@cache_region('long_term', 'cached_repo_list')
+def _get_repos_cached():
+    """
+    return cached dict with repos
+    """
+    return HgModel.repo_scan(g.paths[0][0], g.paths[0][1], g.baseui)
+
+@cache_region('long_term', 'full_changelog')
+def _full_changelog_cached(repo_name):
+    log.info('getting full changelog for %s', repo_name)
+    return list(reversed(list(HgModel().get_repo(repo_name))))
+
 class HgModel(object):
     """
     Mercurial Model
     """
 
-
     def __init__(self):
         """
         Constructor
         """
         pass
-
+    
+    @staticmethod
+    def repo_scan(repos_prefix, repos_path, baseui):
+        """
+        Listing of repositories in given path. This path should not be a 
+        repository itself. Return a dictionary of repository objects
+        :param repos_path: path to directory it could take syntax with 
+        * or ** for deep recursive displaying repositories
+        """
+        def check_repo_dir(path):
+            """
+            Checks the repository
+            :param path:
+            """
+            repos_path = path.split('/')
+            if repos_path[-1] in ['*', '**']:
+                repos_path = repos_path[:-1]
+            if repos_path[0] != '/':
+                repos_path[0] = '/'
+            if not os.path.isdir(os.path.join(*repos_path)):
+                raise RepositoryError('Not a valid repository in %s' % path[0][1])        
+        if not repos_path.endswith('*'):
+            raise VCSError('You need to specify * or ** at the end of path '
+                            'for recursive scanning')
+            
+        check_repo_dir(repos_path)
+        log.info('scanning for repositories in %s', repos_path)
+        repos = findrepos([(repos_prefix, repos_path)])
+        if not isinstance(baseui, ui.ui):
+            baseui = ui.ui()
+    
+        repos_list = {}
+        for name, path in repos:
+            try:
+                repos_list[name] = MercurialRepository(path, baseui=baseui)
+            except OSError:
+                continue
+        return repos_list
+        
     def get_repos(self):
-        for mercurial_repo in get_repositories(g.paths[0][0], g.paths[0][1], g.baseui):
-            
-            if mercurial_repo._get_hidden():
+        for name, repo in _get_repos_cached().items():
+            if repo._get_hidden():
                 #skip hidden web repository
                 continue
             
-            last_change = mercurial_repo.last_change
+            last_change = repo.last_change
             try:
-                tip = mercurial_repo.get_changeset('tip')
+                tip = repo.get_changeset('tip')
             except RepositoryError:
                 from pylons_app.lib.utils import EmptyChangeset
                 tip = EmptyChangeset()
                 
             tmp_d = {}
-            tmp_d['name'] = mercurial_repo.name
+            tmp_d['name'] = repo.name
             tmp_d['name_sort'] = tmp_d['name'].lower()
-            tmp_d['description'] = mercurial_repo.description
+            tmp_d['description'] = repo.description
             tmp_d['description_sort'] = tmp_d['description']
             tmp_d['last_change'] = last_change
             tmp_d['last_change_sort'] = last_change[1] - last_change[0]
             tmp_d['tip'] = tip.raw_id
             tmp_d['tip_sort'] = tip.revision 
             tmp_d['rev'] = tip.revision
-            tmp_d['contact'] = mercurial_repo.contact
+            tmp_d['contact'] = repo.contact
             tmp_d['contact_sort'] = tmp_d['contact']
-            tmp_d['repo_archives'] = list(mercurial_repo._get_archives())
+            tmp_d['repo_archives'] = list(repo._get_archives())
             
             yield tmp_d
 
     def get_repo(self, repo_name):
-        path = g.paths[0][1].replace('*', '')
-        repo = MercurialRepository(os.path.join(path, repo_name), baseui=g.baseui)
-        return repo
+        return _get_repos_cached()[repo_name]

pylons_app/templates/base/base.html

@@ -92,8 +92,8 b' def is_current(selected):'
 					<a id="repo_switcher" title="${_('Switch repository')}" href="#">&darr;</a>
 					<div id="switch_repos" style="display:none;position: absolute;width: 150px;height: 25px">
 						<select id="repos_list" size="=10">
-						%for repo in c.cached_repo_list:
-							<option value="${repo['name']}">${repo['name']}</option>
+						%for repo in c.cached_repo_list.values():
+							<option value="${repo.name}">${repo.name}</option>
 						%endfor
 						</select>
 					</div>