implemented cache for repeated queries in simplehg mercurial requests

development.ini

@@ -44,11 +44,13 b' cache_dir = %(here)s/data'
 ####################################
 beaker.cache.data_dir=/%(here)s/data/cache/data
 beaker.cache.lock_dir=/%(here)s/data/cache/lock
-beaker.cache.regions=short_term,long_term
+beaker.cache.regions=super_short_term,short_term,long_term
 beaker.cache.long_term.type=memory
 beaker.cache.long_term.expire=36000
 beaker.cache.short_term.type=memory
 beaker.cache.short_term.expire=60
+beaker.cache.super_short_term.type=memory
+beaker.cache.super_short_term.expire=10
 
 ################################################################################
 ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##

production.ini

@@ -39,17 +39,18 b' static_files = false'
 lang=en
 cache_dir = %(here)s/data
 
-
 ####################################
 ###         BEAKER CACHE        ####
 ####################################
 beaker.cache.data_dir=/%(here)s/data/cache/data
 beaker.cache.lock_dir=/%(here)s/data/cache/lock
-beaker.cache.regions=short_term,long_term
+beaker.cache.regions=super_short_term,short_term,long_term
 beaker.cache.long_term.type=memory
 beaker.cache.long_term.expire=36000
 beaker.cache.short_term.type=memory
 beaker.cache.short_term.expire=60
+beaker.cache.super_short_term.type=memory
+beaker.cache.super_short_term.expire=10
     
 ################################################################################
 ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##

pylons_app/lib/auth.py

@@ -22,18 +22,18 b' Created on April 4, 2010'
 
 @author: marcink
 """
-
+from beaker.cache import cache_region
 from functools import wraps
-from pylons import session, url, request
+from pylons import config, session, url, request
 from pylons.controllers.util import abort, redirect
+from pylons_app.lib.utils import get_repo_slug
 from pylons_app.model import meta
 from pylons_app.model.db import User, Repo2Perm, Repository, Permission
-from pylons_app.lib.utils import get_repo_slug
 from sqlalchemy.exc import OperationalError
 from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
 import crypt
 import logging
-from pylons import config
+
 log = logging.getLogger(__name__) 
 
 def get_crypt_password(password):
@@ -43,11 +43,17 b' def get_crypt_password(password):'
     """
     return crypt.crypt(password, '6a')
 
+
+@cache_region('super_short_term', 'cached_user')
+def get_user_cached(username):
+    sa = meta.Session
+    user = sa.query(User).filter(User.username == username).one()
+    return user
+
 def authfunc(environ, username, password):
-    sa = meta.Session
     password_crypt = get_crypt_password(password)
     try:
-        user = sa.query(User).filter(User.username == username).one()
+        user = get_user_cached(username)
     except (NoResultFound, MultipleResultsFound, OperationalError) as e:
         log.error(e)
         user = None

pylons_app/lib/middleware/simplehg.py

@@ -2,7 +2,7 b''
 # encoding: utf-8
 # middleware to handle mercurial api calls
 # Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
- 
+
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
@@ -27,21 +27,23 b" It's implemented with basic auth functio"
 """
 from datetime import datetime
 from itertools import chain
+from mercurial.error import RepoError
 from mercurial.hgweb import hgweb
 from mercurial.hgweb.request import wsgiapplication
-from mercurial.error import RepoError
 from paste.auth.basic import AuthBasicAuthenticator
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
-from pylons_app.lib.auth import authfunc, HasPermissionAnyMiddleware
+from pylons_app.lib.auth import authfunc, HasPermissionAnyMiddleware, \
+    get_user_cached
 from pylons_app.lib.utils import is_mercurial, make_ui, invalidate_cache, \
     check_repo_fast
 from pylons_app.model import meta
 from pylons_app.model.db import UserLog, User
-import pylons_app.lib.helpers as h
 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError
 import logging
 import os
+import pylons_app.lib.helpers as h
 import traceback
+ 
 log = logging.getLogger(__name__)
 
 class SimpleHg(object):
@@ -56,86 +58,84 b' class SimpleHg(object):'
     def __call__(self, environ, start_response):
         if not is_mercurial(environ):
             return self.application(environ, start_response)
-        else:
-            #===================================================================
-            # AUTHENTICATE THIS MERCURIAL REQUEST
-            #===================================================================
-            username = REMOTE_USER(environ)
-            if not username:
-                result = self.authenticate(environ)
-                if isinstance(result, str):
-                    AUTH_TYPE.update(environ, 'basic')
-                    REMOTE_USER.update(environ, result)
-                else:
-                    return result.wsgi_application(environ, start_response)
-            
+
+        #===================================================================
+        # AUTHENTICATE THIS MERCURIAL REQUEST
+        #===================================================================
+        username = REMOTE_USER(environ)
+        if not username:
+            result = self.authenticate(environ)
+            if isinstance(result, str):
+                AUTH_TYPE.update(environ, 'basic')
+                REMOTE_USER.update(environ, result)
+            else:
+                return result.wsgi_application(environ, start_response)
+        
+        try:
+            repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:])
+        except:
+            log.error(traceback.format_exc())
+            return HTTPInternalServerError()(environ, start_response)
+        
+        #===================================================================
+        # CHECK PERMISSIONS FOR THIS REQUEST
+        #===================================================================
+        action = self.__get_action(environ)
+        if action:
+            username = self.__get_environ_user(environ)
             try:
-                repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:])
+                user = self.__get_user(username)
             except:
                 log.error(traceback.format_exc())
                 return HTTPInternalServerError()(environ, start_response)
+            #check permissions for this repository
+            if action == 'pull':
+                if not HasPermissionAnyMiddleware('repository.read',
+                                                  'repository.write',
+                                                  'repository.admin')\
+                                                    (user, repo_name):
+                    return HTTPForbidden()(environ, start_response)
+            if action == 'push':
+                if not HasPermissionAnyMiddleware('repository.write',
+                                                  'repository.admin')\
+                                                    (user, repo_name):
+                    return HTTPForbidden()(environ, start_response)
             
-            #===================================================================
-            # CHECK PERMISSIONS FOR THIS REQUEST
-            #===================================================================
-            action = self.__get_action(environ)
-            if action:
-                username = self.__get_environ_user(environ)
-                try:
-                    sa = meta.Session
-                    user = sa.query(User)\
-                        .filter(User.username == username).one()
-                except:
-                    log.error(traceback.format_exc())
-                    return HTTPInternalServerError()(environ, start_response)
-                #check permissions for this repository
-                if action == 'pull':
-                    if not HasPermissionAnyMiddleware('repository.read',
-                                                      'repository.write',
-                                                      'repository.admin')\
-                                                        (user, repo_name):
-                        return HTTPForbidden()(environ, start_response)
-                if action == 'push':
-                    if not HasPermissionAnyMiddleware('repository.write',
-                                                      'repository.admin')\
-                                                        (user, repo_name):
-                        return HTTPForbidden()(environ, start_response)
-                
-                #log action    
-                proxy_key = 'HTTP_X_REAL_IP'
-                def_key = 'REMOTE_ADDR'
-                ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0'))
-                self.__log_user_action(user, action, repo_name, ipaddr)            
-            
-            #===================================================================
-            # MERCURIAL REQUEST HANDLING
-            #===================================================================
-            environ['PATH_INFO'] = '/'#since we wrap into hgweb, reset the path
-            self.baseui = make_ui('db')
-            self.basepath = self.config['base_path']
-            self.repo_path = os.path.join(self.basepath, repo_name)
+            #log action    
+            proxy_key = 'HTTP_X_REAL_IP'
+            def_key = 'REMOTE_ADDR'
+            ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0'))
+            self.__log_user_action(user, action, repo_name, ipaddr)            
+        
+        #===================================================================
+        # MERCURIAL REQUEST HANDLING
+        #===================================================================
+        environ['PATH_INFO'] = '/'#since we wrap into hgweb, reset the path
+        self.baseui = make_ui('db')
+        self.basepath = self.config['base_path']
+        self.repo_path = os.path.join(self.basepath, repo_name)
 
-            #quick check if that dir exists...
-            if check_repo_fast(repo_name, self.basepath):
+        #quick check if that dir exists...
+        if check_repo_fast(repo_name, self.basepath):
+            return HTTPNotFound()(environ, start_response)
+        try:
+            app = wsgiapplication(self.__make_app)
+        except RepoError as e:
+            if str(e).find('not found') != -1:
                 return HTTPNotFound()(environ, start_response)
-            try:
-                app = wsgiapplication(self.__make_app)
-            except RepoError as e:
-                if str(e).find('not found') != -1:
-                    return HTTPNotFound()(environ, start_response)
-            except Exception:
-                log.error(traceback.format_exc())
-                return HTTPInternalServerError()(environ, start_response)
-            
-            #invalidate cache on push
-            if action == 'push':
-                self.__invalidate_cache(repo_name)
-                messages = []
-                messages.append('thank you for using hg-app')
-            
-                return self.msg_wrapper(app, environ, start_response, messages)
-            else:
-                return app(environ, start_response)           
+        except Exception:
+            log.error(traceback.format_exc())
+            return HTTPInternalServerError()(environ, start_response)
+        
+        #invalidate cache on push
+        if action == 'push':
+            self.__invalidate_cache(repo_name)
+            messages = []
+            messages.append('thank you for using hg-app')
+        
+            return self.msg_wrapper(app, environ, start_response, messages)
+        else:
+            return app(environ, start_response)           
 
 
     def msg_wrapper(self, app, environ, start_response, messages=[]):
@@ -160,6 +160,11 b' class SimpleHg(object):'
     def __get_environ_user(self, environ):
         return environ.get('REMOTE_USER')
     
+    def __get_user(self, username):
+        return get_user_cached(username)
+        
+        
+                        
     def __get_size(self, repo_path, content_size):
         size = int(content_size)
         for path, dirs, files in os.walk(repo_path):

pylons_app/lib/utils.py

@@ -16,6 +16,7 b''
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
+from beaker.cache import cache_region
 
 """
 Created on April 18, 2010
@@ -75,6 +76,13 b' def check_repo(repo_name, base_path, ver'
         log.info('%s repo is free for creation', repo_name)
         return True
 
+
+@cache_region('super_short_term', 'cached_hg_ui')
+def get_hg_ui_cached():
+    from pylons_app.model.meta import Session
+    sa = Session()
+    return sa.query(HgAppUi).all()    
+
 def make_ui(read_from='file', path=None, checkpaths=True):        
     """
     A function that will read python rc files or database
@@ -112,10 +120,7 b" def make_ui(read_from='file', path=None,"
               
         
     elif read_from == 'db':
-        from pylons_app.model.meta import Session
-        sa = Session()
-            
-        hg_ui = sa.query(HgAppUi).all()
+        hg_ui = get_hg_ui_cached()
         for ui_ in hg_ui:
             baseui.setconfig(ui_.ui_section, ui_.ui_key, ui_.ui_value)