#50 on point cache invalidation changes.

Created cacheInvalidation table
cleaned up sa sessions from models, since it wasn't really needed.

rhodecode/controllers/admin/permissions.py

@@ -31,7 +31,7 b' from rhodecode.lib import helpers as h'
 from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator
 from rhodecode.lib.base import BaseController, render
 from rhodecode.model.forms import UserForm, DefaultPermissionsForm
-from rhodecode.model.permission_model import PermissionModel
+from rhodecode.model.permission import PermissionModel
 from rhodecode.model.user import UserModel
 import formencode
 import logging

rhodecode/controllers/admin/repos.py

@@ -133,7 +133,7 b' class ReposController(BaseController):'
             form_result = _form.to_python(dict(request.POST))
             repo_model.update(repo_name, form_result)
             invalidate_cache('get_repo_cached_%s' % repo_name)
-            h.flash(_('Repository %s updated succesfully' % repo_name),
+            h.flash(_('Repository %s updated successfully' % repo_name),
                     category='success')
             changed_name = form_result['repo_name']
             action_logger(self.rhodecode_user, 'admin_updated_repo',
@@ -152,7 +152,7 b' class ReposController(BaseController):'
 
         except Exception:
             log.error(traceback.format_exc())
-            h.flash(_('error occured during update of repository %s') \
+            h.flash(_('error occurred during update of repository %s') \
                     % repo_name, category='error')
 
         return redirect(url('edit_repo', repo_name=changed_name))

rhodecode/controllers/admin/settings.py

@@ -248,7 +248,7 b' class SettingsController(BaseController)'
         """
 
         # url('admin_settings_my_account')
-        c.user = UserModel(self.sa).get(c.rhodecode_user.user_id, cache=False)
+        c.user = UserModel().get(c.rhodecode_user.user_id, cache=False)
         all_repos = self.sa.query(Repository)\
             .filter(Repository.user_id == c.user.user_id)\
             .order_by(func.lower(Repository.repo_name))\
@@ -289,7 +289,7 b' class SettingsController(BaseController)'
 
         except formencode.Invalid, errors:
             c.user = user_model.get(c.rhodecode_user.user_id, cache=False)
-            c.user = UserModel(self.sa).get(c.rhodecode_user.user_id, cache=False)
+            c.user = UserModel().get(c.rhodecode_user.user_id, cache=False)
             all_repos = self.sa.query(Repository)\
                 .filter(Repository.user_id == c.user.user_id)\
                 .order_by(func.lower(Repository.repo_name))\

rhodecode/controllers/admin/users.py

@@ -17,7 +17,6 b''
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
-from rhodecode.lib.utils import action_logger
 """
 Created on April 4, 2010
 users controller for pylons

rhodecode/lib/auth.py

@@ -143,7 +143,7 b' def fill_perms(user):'
     #===========================================================================
     # fetch default permissions
     #===========================================================================
-    default_user = UserModel(sa).get_by_username('default', cache=True)
+    default_user = UserModel().get_by_username('default', cache=True)
 
     default_perms = sa.query(RepoToPerm, Repository, Permission)\
         .join((Repository, RepoToPerm.repository_id == Repository.repo_id))\

rhodecode/lib/celerylib/tasks.py

@@ -1,16 +1,18 b''
 from celery.decorators import task
 
+import os
+import traceback
+from time import mktime
+
 from operator import itemgetter
 from pylons.i18n.translation import _
 from rhodecode.lib.celerylib import run_task, locked_task
 from rhodecode.lib.helpers import person
 from rhodecode.lib.smtp_mailer import SmtpMailer
 from rhodecode.lib.utils import OrderedDict
-from time import mktime
-import os
-import traceback
 from vcs.backends import get_repo
-from rhodecode.model.scm import ScmModel
+from rhodecode.model.db import RhodeCodeUi
+
 try:
     import json
 except ImportError:
@@ -44,6 +46,11 b' def get_session():'
 
     return sa
 
+def get_repos_path():
+    sa = get_session()
+    q = sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()
+    return q.ui_value
+
 @task
 @locked_task
 def whoosh_index(repo_location, full_index):
@@ -62,7 +69,7 b' def get_commits_stats(repo_name, ts_min_'
 
     commits_by_day_author_aggregate = {}
     commits_by_day_aggregate = {}
-    repos_path = ScmModel().repos_path
+    repos_path = get_repos_path()
     p = os.path.join(repos_path, repo_name)
     repo = get_repo(p)
 
@@ -271,10 +278,10 b' def create_repo_fork(form_data, cur_user'
     from rhodecode.model.repo import RepoModel
     from vcs import get_backend
     log = create_repo_fork.get_logger()
-    repo_model = RepoModel(get_session())
+    repo_model = RepoModel()
     repo_model.create(form_data, cur_user, just_db=True, fork=True)
     repo_name = form_data['repo_name']
-    repos_path = ScmModel().repos_path
+    repos_path = get_repos_path()
     repo_path = os.path.join(repos_path, repo_name)
     repo_fork_path = os.path.join(repos_path, form_data['fork_name'])
     alias = form_data['repo_type']
@@ -291,7 +298,7 b' def __get_codes_stats(repo_name):'
     's', 'sh', 'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt', 'yaws']
 
 
-    repos_path = ScmModel().repos_path
+    repos_path = get_repos_path()
     p = os.path.join(repos_path, repo_name)
     repo = get_repo(p)
     tip = repo.get_changeset()

rhodecode/lib/utils.py

@@ -89,7 +89,7 b' def action_logger(user, action, repo, ip'
         if hasattr(user, 'user_id'):
             user_obj = user
         elif isinstance(user, basestring):
-            user_obj = UserModel(sa).get_by_username(user, cache=False)
+            user_obj = UserModel().get_by_username(user, cache=False)
         else:
             raise Exception('You have to provide user object or username')
 
@@ -97,7 +97,7 b' def action_logger(user, action, repo, ip'
         if repo:
             repo_name = repo.lstrip('/')
 
-            repository = RepoModel(sa).get(repo_name, cache=False)
+            repository = RepoModel().get(repo_name, cache=False)
             if not repository:
                 raise Exception('You have to provide valid repository')
         else:
@@ -293,12 +293,16 b' def set_rhodecode_config(config):'
     for k, v in hgsettings.items():
         config[k] = v
 
-def invalidate_cache(name, *args):
+def invalidate_cache(cache_key, *args):
     """
     Puts cache invalidation task into db for 
     further global cache invalidation
     """
-    pass
+    from rhodecode.model.scm import ScmModel
+
+    if cache_key.startswith('get_repo_cached_'):
+        name = cache_key.split('get_repo_cached_')[-1]
+        ScmModel().mark_for_invalidation(name)
 
 class EmptyChangeset(BaseChangeset):
     """
@@ -340,7 +344,7 b' def repo2db_mapper(initial_repo_list, re'
     """
 
     sa = meta.Session()
-    rm = RepoModel(sa)
+    rm = RepoModel()
     user = sa.query(User).filter(User.admin == True).first()
 
     for name, repo in initial_repo_list.items():

rhodecode/model/db.py

@@ -90,9 +90,8 b' class Repository(Base):'
     repo_to_perm = relation('RepoToPerm', cascade='all')
     stats = relation('Statistics', cascade='all')
 
-
     def __repr__(self):
-        return "<Repository('id:%s:%s')>" % (self.repo_id, self.repo_name)
+        return "<Repository('%s:%s')>" % (self.repo_id, self.repo_name)
 
 class Permission(Base):
     __tablename__ = 'permissions'
@@ -140,10 +139,17 b' class Statistics(Base):'
 
 class CacheInvalidation(Base):
     __tablename__ = 'cache_invalidation'
-    __table_args__ = {'useexisting':True}
+    __table_args__ = (UniqueConstraint('cache_key'), {'useexisting':True})
     cache_id = Column("cache_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
     cache_key = Column("cache_key", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     cache_args = Column("cache_args", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
-    cache_active = Column("cache_active", BOOLEAN(), nullable=True, unique=None, default=None)
+    cache_active = Column("cache_active", BOOLEAN(), nullable=True, unique=None, default=False)
 
 
+    def __init__(self, cache_key, cache_args=''):
+        self.cache_key = cache_key
+        self.cache_args = cache_args
+        self.cache_active = False
+
+    def __repr__(self):
+        return "<CacheInvaidation('%s:%s')>" % (self.cache_id, self.cache_key)

rhodecode/model/permission.py

@@ -33,11 +33,8 b' log = logging.getLogger(__name__)'
 
 class PermissionModel(object):
 
-    def __init__(self, sa=None):
-        if not sa:
-            self.sa = Session()
-        else:
-            self.sa = sa
+    def __init__(self):
+        self.sa = Session()
 
     def get_permission(self, permission_id, cache=False):
         perm = self.sa.query(Permission)

rhodecode/model/repo.py

@@ -36,11 +36,8 b' log = logging.getLogger(__name__)'
 
 class RepoModel(object):
 
-    def __init__(self, sa=None):
-        if not sa:
-            self.sa = Session()
-        else:
-            self.sa = sa
+    def __init__(self):
+        self.sa = Session()
 
     def get(self, repo_id, cache=False):
         repo = self.sa.query(Repository)\
@@ -67,7 +64,7 b' class RepoModel(object):'
             #update permissions
             for username, perm in form_data['perms_updates']:
                 r2p = self.sa.query(RepoToPerm)\
-                        .filter(RepoToPerm.user == UserModel(self.sa).get_by_username(username, cache=False))\
+                        .filter(RepoToPerm.user == UserModel().get_by_username(username, cache=False))\
                         .filter(RepoToPerm.repository == self.get(repo_name))\
                         .one()
 
@@ -80,7 +77,7 b' class RepoModel(object):'
             for username, perm in form_data['perms_new']:
                 r2p = RepoToPerm()
                 r2p.repository = self.get(repo_name)
-                r2p.user = UserModel(self.sa).get_by_username(username, cache=False)
+                r2p.user = UserModel().get_by_username(username, cache=False)
 
                 r2p.permission_id = self.sa.query(Permission).filter(
                                         Permission.permission_name == perm)\
@@ -134,7 +131,7 b' class RepoModel(object):'
             #create default permission
             repo_to_perm = RepoToPerm()
             default = 'repository.read'
-            for p in UserModel(self.sa).get_by_username('default', cache=False).user_perms:
+            for p in UserModel().get_by_username('default', cache=False).user_perms:
                 if p.permission.permission_name.startswith('repository.'):
                     default = p.permission.permission_name
                     break
@@ -146,7 +143,7 b' class RepoModel(object):'
                     .one().permission_id
 
             repo_to_perm.repository_id = new_repo.repo_id
-            repo_to_perm.user_id = UserModel(self.sa).get_by_username('default', cache=False).user_id
+            repo_to_perm.user_id = UserModel().get_by_username('default', cache=False).user_id
 
             self.sa.add(repo_to_perm)
             self.sa.commit()

rhodecode/model/scm.py

@@ -28,12 +28,15 b' from rhodecode.lib import helpers as h'
 from rhodecode.lib.auth import HasRepoPermissionAny
 from rhodecode.lib.utils import get_repos
 from rhodecode.model import meta
-from rhodecode.model.db import Repository, User, RhodeCodeUi
+from rhodecode.model.db import Repository, User, RhodeCodeUi, CacheInvalidation
+from rhodecode.model.caching_query import FromCache
 from sqlalchemy.orm import joinedload
+from sqlalchemy.orm.session import make_transient
 from vcs import get_backend
 from vcs.utils.helpers import get_scm
 from vcs.exceptions import RepositoryError, VCSError
 from vcs.utils.lazy import LazyProperty
+import traceback
 import logging
 import os
 import time
@@ -45,12 +48,8 b' class ScmModel(object):'
     Mercurial Model
     """
 
-    def __init__(self, sa=None):
-        if not sa:
-            self.sa = meta.Session()
-        else:
-            self.sa = sa
-
+    def __init__(self):
+        self.sa = meta.Session()
 
     @LazyProperty
     def repos_path(self):
@@ -143,7 +142,7 b' class ScmModel(object):'
                             'repository.admin')(repo_name, 'get repo check'):
             return
 
-        @cache_region('long_term', 'get_repo_cached_%s' % repo_name)
+        @cache_region('long_term')
         def _get_repo(repo_name):
 
             repo_path = os.path.join(self.repos_path, repo_name)
@@ -165,13 +164,76 b' class ScmModel(object):'
                 .options(joinedload(Repository.user))\
                 .filter(Repository.repo_name == repo_name)\
                 .scalar()
+            make_transient(dbrepo)
             repo.dbrepo = dbrepo
             return repo
 
-        invalidate = False
+        invalidate = self._should_invalidate(repo_name)
         if invalidate:
-            log.info('INVALIDATING CACHE FOR %s', repo_name)
+            log.info('invalidating cache for repository %s', repo_name)
             region_invalidate(_get_repo, None, repo_name)
+            self._mark_invalidated(invalidate)
 
         return _get_repo(repo_name)
 
+
+
+    def mark_for_invalidation(self, repo_name):
+        """
+        Puts cache invalidation task into db for 
+        further global cache invalidation
+        
+        :param repo_name: this repo that should invalidation take place
+        """
+        log.debug('marking %s for invalidation', repo_name)
+        cache = self.sa.query(CacheInvalidation)\
+            .filter(CacheInvalidation.cache_key == repo_name).scalar()
+
+        if cache:
+            #mark this cache as inactive
+            cache.cache_active = False
+        else:
+            log.debug('cache key not found in invalidation db -> creating one')
+            cache = CacheInvalidation(repo_name)
+
+        try:
+            self.sa.add(cache)
+            self.sa.commit()
+        except:
+            log.error(traceback.format_exc())
+            self.sa.rollback()
+
+
+
+
+
+    def _should_invalidate(self, repo_name):
+        """
+        Looks up database for invalidation signals for this repo_name
+        :param repo_name:
+        """
+
+        ret = self.sa.query(CacheInvalidation)\
+            .options(FromCache('sql_cache_short',
+                           'get_invalidation_%s' % repo_name))\
+            .filter(CacheInvalidation.cache_key == repo_name)\
+            .filter(CacheInvalidation.cache_active == False)\
+            .scalar()
+
+        return ret
+
+    def _mark_invalidated(self, cache_key):
+        """
+        Marks all occurences of cache to invaldation as already invalidated
+        @param repo_name:
+        """
+        if cache_key:
+            log.debug('marking %s as already invalidated', cache_key)
+        try:
+            cache_key.cache_active = True
+            self.sa.add(cache_key)
+            self.sa.commit()
+        except:
+            log.error(traceback.format_exc())
+            self.sa.rollback()
+

rhodecode/model/user.py

@@ -36,11 +36,8 b' class DefaultUserException(Exception):pa'
 
 class UserModel(object):
 
-    def __init__(self, sa=None):
-        if not sa:
-            self.sa = Session()
-        else:
-            self.sa = sa
+    def __init__(self):
+        self.sa = Session()
 
     def get(self, user_id, cache=False):
         user = self.sa.query(User)