implemented cache for repeated queries in simplehg mercurial requests

development.ini

@@ -1,133 +1,135 b''
 ################################################################################
 ################################################################################
 # pylons_app - Pylons environment configuration                                #
 #                                                                              # 
 # The %(here)s variable will be replaced with the parent directory of this file#
 ################################################################################
 
 [DEFAULT]
 debug = true
 ############################################
 ## Uncomment and replace with the address ##
 ## which should receive any error reports ##
 ############################################
 #email_to = admin@localhost
 #smtp_server = mail.server.com
 #error_email_from = paste_error@localhost
 #smtp_username = 
 #smtp_password = 
 #error_message = 'mercurial crash !'
 
 [server:main]
 ##nr of threads to spawn
 threadpool_workers = 5
 
 ##max request before
 threadpool_max_requests = 2
 
 ##option to use threads of process
 use_threadpool = true
 
 use = egg:Paste#http
 host = 127.0.0.1
 port = 5000
 
 [app:main]
 use = egg:pylons_app
 full_stack = true
 static_files = true
 lang=en
 cache_dir = %(here)s/data
 
 ####################################
 ###         BEAKER CACHE        ####
 ####################################
 beaker.cache.data_dir=/%(here)s/data/cache/data
 beaker.cache.lock_dir=/%(here)s/data/cache/lock
-beaker.cache.regions=short_term,long_term
+beaker.cache.regions=super_short_term,short_term,long_term
 beaker.cache.long_term.type=memory
 beaker.cache.long_term.expire=36000
 beaker.cache.short_term.type=memory
 beaker.cache.short_term.expire=60
+beaker.cache.super_short_term.type=memory
+beaker.cache.super_short_term.expire=10
 
 ################################################################################
 ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 ## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 ## execute malicious code after an exception is raised.                       ##
 ################################################################################
 #set debug = false
 
 ##################################
 ###       LOGVIEW CONFIG       ###
 ##################################
 logview.sqlalchemy = #faa
 logview.pylons.templating = #bfb
 logview.pylons.util = #eee
 
 #########################################################
 ### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###
 #########################################################
 sqlalchemy.db1.url = sqlite:///%(here)s/hg_app.db
 #sqlalchemy.db1.echo = False
 #sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.convert_unicode = true
 
 ################################
 ### LOGGING CONFIGURATION   ####
 ################################
 [loggers]
 keys = root, routes, pylons_app, sqlalchemy
 
 [handlers]
 keys = console
 
 [formatters]
 keys = generic,color_formatter
 
 #############
 ## LOGGERS ##
 #############
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_routes]
 level = DEBUG
 handlers = console
 qualname = routes.middleware
 # "level = DEBUG" logs the route matched and routing variables.
 
 [logger_pylons_app]
 level = DEBUG
 handlers = console
 qualname = pylons_app
 propagate = 0
 
 [logger_sqlalchemy]
 level = ERROR
 handlers = console
 qualname = sqlalchemy.engine
 propagate = 0
 
 ##############
 ## HANDLERS ##
 ##############
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
 level = NOTSET
 formatter = color_formatter
 
 ################
 ## FORMATTERS ##
 ################
 
 [formatter_generic]
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class=pylons_app.lib.colored_formatter.ColorFormatter
 format= %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
  No newline at end of file

production.ini

@@ -1,134 +1,135 b''
 ################################################################################
 ################################################################################
 # pylons_app - Pylons environment configuration                                #
 #                                                                              # 
 # The %(here)s variable will be replaced with the parent directory of this file#
 ################################################################################
 
 [DEFAULT]
 debug = true
 ############################################
 ## Uncomment and replace with the address ##
 ## which should receive any error reports ##
 ############################################
 #email_to = admin@localhost
 #smtp_server = mail.server.com
 #error_email_from = paste_error@localhost
 #smtp_username = 
 #smtp_password = 
 #error_message = 'mercurial crash !'
 
 [server:main]
 ##nr of threads to spawn
 threadpool_workers = 5
 
 ##max request before
 threadpool_max_requests = 2
 
 ##option to use threads of process
 use_threadpool = true
 
 use = egg:Paste#http
 host = 127.0.0.1
 port = 8001
 
 [app:main]
 use = egg:pylons_app
 full_stack = true
 static_files = false
 lang=en
 cache_dir = %(here)s/data
 
-
 ####################################
 ###         BEAKER CACHE        ####
 ####################################
 beaker.cache.data_dir=/%(here)s/data/cache/data
 beaker.cache.lock_dir=/%(here)s/data/cache/lock
-beaker.cache.regions=short_term,long_term
+beaker.cache.regions=super_short_term,short_term,long_term
 beaker.cache.long_term.type=memory
 beaker.cache.long_term.expire=36000
 beaker.cache.short_term.type=memory
 beaker.cache.short_term.expire=60
+beaker.cache.super_short_term.type=memory
+beaker.cache.super_short_term.expire=10
     
 ################################################################################
 ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 ## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 ## execute malicious code after an exception is raised.                       ##
 ################################################################################
 set debug = false
 
 ##################################
 ###       LOGVIEW CONFIG       ###
 ##################################
 logview.sqlalchemy = #faa
 logview.pylons.templating = #bfb
 logview.pylons.util = #eee
 
 #########################################################
 ### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###
 #########################################################
 sqlalchemy.db1.url = sqlite:///%(here)s/hg_app.db
 #sqlalchemy.db1.echo = False
 #sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.convert_unicode = true
 
 ################################
 ### LOGGING CONFIGURATION   ####
 ################################
 [loggers]
 keys = root, routes, pylons_app, sqlalchemy
 
 [handlers]
 keys = console
 
 [formatters]
 keys = generic,color_formatter
 
 #############
 ## LOGGERS ##
 #############
 [logger_root]
 level = INFO
 handlers = console
 
 [logger_routes]
 level = INFO
 handlers = console
 qualname = routes.middleware
 # "level = DEBUG" logs the route matched and routing variables.
 
 [logger_pylons_app]
 level = DEBUG
 handlers = console
 qualname = pylons_app
 propagate = 0
 
 [logger_sqlalchemy]
 level = ERROR
 handlers = console
 qualname = sqlalchemy.engine
 propagate = 0
 
 ##############
 ## HANDLERS ##
 ##############
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
 level = NOTSET
 formatter = color_formatter
 
 ################
 ## FORMATTERS ##
 ################
 
 [formatter_generic]
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class=pylons_app.lib.colored_formatter.ColorFormatter
 format= %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
  No newline at end of file

pylons_app/lib/auth.py

@@ -1,399 +1,405 b''
 #!/usr/bin/env python
 # encoding: utf-8
 # authentication and permission libraries
 # Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
 
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 """
 Created on April 4, 2010
 
 @author: marcink
 """
-
+from beaker.cache import cache_region
 from functools import wraps
-from pylons import session, url, request
+from pylons import config, session, url, request
 from pylons.controllers.util import abort, redirect
+from pylons_app.lib.utils import get_repo_slug
 from pylons_app.model import meta
 from pylons_app.model.db import User, Repo2Perm, Repository, Permission
-from pylons_app.lib.utils import get_repo_slug
 from sqlalchemy.exc import OperationalError
 from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
 import crypt
 import logging
-from pylons import config
+
 log = logging.getLogger(__name__) 
 
 def get_crypt_password(password):
     """
     Cryptographic function used for password hashing
     @param password: password to hash
     """
     return crypt.crypt(password, '6a')
 
+
+@cache_region('super_short_term', 'cached_user')
+def get_user_cached(username):
+    sa = meta.Session
+    user = sa.query(User).filter(User.username == username).one()
+    return user
+
 def authfunc(environ, username, password):
-    sa = meta.Session
     password_crypt = get_crypt_password(password)
     try:
-        user = sa.query(User).filter(User.username == username).one()
+        user = get_user_cached(username)
     except (NoResultFound, MultipleResultsFound, OperationalError) as e:
         log.error(e)
         user = None
         
     if user:
         if user.active:
             if user.username == username and user.password == password_crypt:
                 log.info('user %s authenticated correctly', username)
                 return True
         else:
             log.error('user %s is disabled', username)
             
     return False
 
 class  AuthUser(object):
     """
     A simple object that handles a mercurial username for authentication
     """
     def __init__(self):
         self.username = 'None'
         self.user_id = None
         self.is_authenticated = False
         self.is_admin = False
         self.permissions = {}
 
 
 def set_available_permissions(config):
     """
     This function will propagate pylons globals with all available defined
     permission given in db. We don't wannt to check each time from db for new 
     permissions since adding a new permission also requires application restart
     ie. to decorate new views with the newly created permission
     @param config:
     """
     log.info('getting information about all available permissions')
     sa = meta.Session
     all_perms = sa.query(Permission).all()
     config['available_permissions'] = [x.permission_name for x in all_perms]
 
 def set_base_path(config):
     config['base_path'] = config['pylons.app_globals'].base_path
         
 def fill_perms(user):
     sa = meta.Session
     user.permissions['repositories'] = {}
     
     #first fetch default permissions
     default_perms = sa.query(Repo2Perm, Repository, Permission)\
         .join((Repository, Repo2Perm.repository == Repository.repo_name))\
         .join((Permission, Repo2Perm.permission_id == Permission.permission_id))\
         .filter(Repo2Perm.user_id == sa.query(User).filter(User.username == 
                                             'default').one().user_id).all()
 
     if user.is_admin:
         user.permissions['global'] = set(['hg.admin'])
         #admin have all rights full
         for perm in default_perms:
             p = 'repository.admin'
             user.permissions['repositories'][perm.Repo2Perm.repository] = p
     
     else:
         user.permissions['global'] = set()
         for perm in default_perms:
             if perm.Repository.private:
                 #disable defaults for private repos,
                 p = 'repository.none'
             elif perm.Repository.user_id == user.user_id:
                 #set admin if owner
                 p = 'repository.admin'
             else:
                 p = perm.Permission.permission_name
                 
             user.permissions['repositories'][perm.Repo2Perm.repository] = p
                                                 
         
         user_perms = sa.query(Repo2Perm, Permission, Repository)\
             .join((Repository, Repo2Perm.repository == Repository.repo_name))\
             .join((Permission, Repo2Perm.permission_id == Permission.permission_id))\
             .filter(Repo2Perm.user_id == user.user_id).all()
         #overwrite userpermissions with defaults
         for perm in user_perms:
             #set write if owner
             if perm.Repository.user_id == user.user_id:
                 p = 'repository.write'
             else:
                 p = perm.Permission.permission_name
             user.permissions['repositories'][perm.Repo2Perm.repository] = p            
     return user
     
 def get_user(session):
     """
     Gets user from session, and wraps permissions into user
     @param session:
     """
     user = session.get('hg_app_user', AuthUser())
         
     if user.is_authenticated:
         user = fill_perms(user)
 
     session['hg_app_user'] = user
     session.save()
     return user
         
 #===============================================================================
 # CHECK DECORATORS
 #===============================================================================
 class LoginRequired(object):
     """
     Must be logged in to execute this function else redirect to login page
     """
    
     def __call__(self, func):
         @wraps(func)
         def _wrapper(*fargs, **fkwargs):
             user = session.get('hg_app_user', AuthUser())
             log.debug('Checking login required for user:%s', user.username)
             if user.is_authenticated:
                 log.debug('user %s is authenticated', user.username)
                 func(*fargs)
             else:
                 log.warn('user %s not authenticated', user.username)
                 log.debug('redirecting to login page')
                 return redirect(url('login_home'))
 
         return _wrapper
 
 class PermsDecorator(object):
     """
     Base class for decorators
     """
     
     def __init__(self, *required_perms):
         available_perms = config['available_permissions']
         for perm in required_perms:
             if perm not in available_perms:
                 raise Exception("'%s' permission is not defined" % perm)
         self.required_perms = set(required_perms)
         self.user_perms = None
         
     def __call__(self, func):
         @wraps(func)
         def _wrapper(*fargs, **fkwargs):
             self.user_perms = session.get('hg_app_user', AuthUser()).permissions
             log.debug('checking %s permissions %s for %s',
                self.__class__.__name__, self.required_perms, func.__name__)
             
             if self.check_permissions():
                 log.debug('Permission granted for %s', func.__name__)
                 return func(*fargs)
             
             else:
                 log.warning('Permission denied for %s', func.__name__)
                 #redirect with forbidden ret code
                 return abort(403)
         return _wrapper
         
         
     def check_permissions(self):
         """
         Dummy function for overriding
         """
         raise Exception('You have to write this function in child class')
 
 class HasPermissionAllDecorator(PermsDecorator):
     """
     Checks for access permission for all given predicates. All of them have to
     be meet in order to fulfill the request
     """
         
     def check_permissions(self):
         if self.required_perms.issubset(self.user_perms.get('global')):
             return True
         return False
             
 
 class HasPermissionAnyDecorator(PermsDecorator):
     """
     Checks for access permission for any of given predicates. In order to 
     fulfill the request any of predicates must be meet
     """
     
     def check_permissions(self):
         if self.required_perms.intersection(self.user_perms.get('global')):
             return True
         return False
 
 class HasRepoPermissionAllDecorator(PermsDecorator):
     """
     Checks for access permission for all given predicates for specific 
     repository. All of them have to be meet in order to fulfill the request
     """
             
     def check_permissions(self):
         repo_name = get_repo_slug(request)
         try:
             user_perms = set([self.user_perms['repositories'][repo_name]])
         except KeyError:
             return False
         if self.required_perms.issubset(user_perms):
             return True
         return False
             
 
 class HasRepoPermissionAnyDecorator(PermsDecorator):
     """
     Checks for access permission for any of given predicates for specific 
     repository. In order to fulfill the request any of predicates must be meet
     """
             
     def check_permissions(self):
         repo_name = get_repo_slug(request)
         
         try:
             user_perms = set([self.user_perms['repositories'][repo_name]])
         except KeyError:
             return False
         if self.required_perms.intersection(user_perms):
             return True
         return False
 #===============================================================================
 # CHECK FUNCTIONS
 #===============================================================================
 
 class PermsFunction(object):
     """
     Base function for other check functions
     """
     
     def __init__(self, *perms):
         available_perms = config['available_permissions']
         
         for perm in perms:
             if perm not in available_perms:
                 raise Exception("'%s' permission in not defined" % perm)
         self.required_perms = set(perms)
         self.user_perms = None
         self.granted_for = ''
         self.repo_name = None
         
     def __call__(self, check_Location=''):
         user = session.get('hg_app_user', False)
         if not user:
             return False
         self.user_perms = user.permissions
         self.granted_for = user.username        
         log.debug('checking %s %s', self.__class__.__name__, self.required_perms)            
         
         if self.check_permissions():
             log.debug('Permission granted for %s @%s', self.granted_for,
                       check_Location)
             return True
         
         else:
             log.warning('Permission denied for %s @%s', self.granted_for,
                         check_Location)
             return False 
     
     def check_permissions(self):
         """
         Dummy function for overriding
         """
         raise Exception('You have to write this function in child class')
         
 class HasPermissionAll(PermsFunction):
     def check_permissions(self):
         if self.required_perms.issubset(self.user_perms.get('global')):
             return True
         return False
 
 class HasPermissionAny(PermsFunction):
     def check_permissions(self):
         if self.required_perms.intersection(self.user_perms.get('global')):
             return True
         return False
 
 class HasRepoPermissionAll(PermsFunction):
     
     def __call__(self, repo_name=None, check_Location=''):
         self.repo_name = repo_name
         return super(HasRepoPermissionAll, self).__call__(check_Location)
             
     def check_permissions(self):
         if not self.repo_name:
             self.repo_name = get_repo_slug(request)
 
         try:
             self.user_perms = set([self.user_perms['repositories']\
                                    [self.repo_name]])
         except KeyError:
             return False
         self.granted_for = self.repo_name       
         if self.required_perms.issubset(self.user_perms):
             return True
         return False
             
 class HasRepoPermissionAny(PermsFunction):
     
     
     def __call__(self, repo_name=None, check_Location=''):
         self.repo_name = repo_name
         return super(HasRepoPermissionAny, self).__call__(check_Location)
         
     def check_permissions(self):
         if not self.repo_name:
             self.repo_name = get_repo_slug(request)
 
         try:
             self.user_perms = set([self.user_perms['repositories']\
                                    [self.repo_name]])
         except KeyError:
             return False
         self.granted_for = self.repo_name
         if self.required_perms.intersection(self.user_perms):
             return True
         return False
 
 #===============================================================================
 # SPECIAL VERSION TO HANDLE MIDDLEWARE AUTH
 #===============================================================================
 
 class HasPermissionAnyMiddleware(object):
     def __init__(self, *perms):
         self.required_perms = set(perms)
     
     def __call__(self, user, repo_name):
         usr = AuthUser()
         usr.user_id = user.user_id
         usr.username = user.username
         usr.is_admin = user.admin
         
         try:
             self.user_perms = set([fill_perms(usr)\
                                    .permissions['repositories'][repo_name]])
         except:
             self.user_perms = set()
         self.granted_for = ''
         self.username = user.username
         self.repo_name = repo_name        
         return self.check_permissions()
             
     def check_permissions(self):
         log.debug('checking mercurial protocol '
                   'permissions for user:%s repository:%s',
                                                 self.username, self.repo_name)
         if self.required_perms.intersection(self.user_perms):
             log.debug('permission granted')
             return True
         log.debug('permission denied')
         return False

pylons_app/lib/middleware/simplehg.py

@@ -1,226 +1,231 b''
 #!/usr/bin/env python
 # encoding: utf-8
 # middleware to handle mercurial api calls
 # Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
- 
+
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
 """
 Created on 2010-04-28
 
 @author: marcink
 SimpleHG middleware for handling mercurial protocol request (push/clone etc.)
 It's implemented with basic auth function
 """
 from datetime import datetime
 from itertools import chain
+from mercurial.error import RepoError
 from mercurial.hgweb import hgweb
 from mercurial.hgweb.request import wsgiapplication
-from mercurial.error import RepoError
 from paste.auth.basic import AuthBasicAuthenticator
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
-from pylons_app.lib.auth import authfunc, HasPermissionAnyMiddleware
+from pylons_app.lib.auth import authfunc, HasPermissionAnyMiddleware, \
+    get_user_cached
 from pylons_app.lib.utils import is_mercurial, make_ui, invalidate_cache, \
     check_repo_fast
 from pylons_app.model import meta
 from pylons_app.model.db import UserLog, User
-import pylons_app.lib.helpers as h
 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError
 import logging
 import os
+import pylons_app.lib.helpers as h
 import traceback
+ 
 log = logging.getLogger(__name__)
 
 class SimpleHg(object):
 
     def __init__(self, application, config):
         self.application = application
         self.config = config
         #authenticate this mercurial request using 
         realm = self.config['hg_app_auth_realm']
         self.authenticate = AuthBasicAuthenticator(realm, authfunc)
         
     def __call__(self, environ, start_response):
         if not is_mercurial(environ):
             return self.application(environ, start_response)
-        else:
-            #===================================================================
-            # AUTHENTICATE THIS MERCURIAL REQUEST
-            #===================================================================
-            username = REMOTE_USER(environ)
-            if not username:
-                result = self.authenticate(environ)
-                if isinstance(result, str):
-                    AUTH_TYPE.update(environ, 'basic')
-                    REMOTE_USER.update(environ, result)
-                else:
-                    return result.wsgi_application(environ, start_response)
-            
+
+        #===================================================================
+        # AUTHENTICATE THIS MERCURIAL REQUEST
+        #===================================================================
+        username = REMOTE_USER(environ)
+        if not username:
+            result = self.authenticate(environ)
+            if isinstance(result, str):
+                AUTH_TYPE.update(environ, 'basic')
+                REMOTE_USER.update(environ, result)
+            else:
+                return result.wsgi_application(environ, start_response)
+        
+        try:
+            repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:])
+        except:
+            log.error(traceback.format_exc())
+            return HTTPInternalServerError()(environ, start_response)
+        
+        #===================================================================
+        # CHECK PERMISSIONS FOR THIS REQUEST
+        #===================================================================
+        action = self.__get_action(environ)
+        if action:
+            username = self.__get_environ_user(environ)
             try:
-                repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:])
+                user = self.__get_user(username)
             except:
                 log.error(traceback.format_exc())
                 return HTTPInternalServerError()(environ, start_response)
+            #check permissions for this repository
+            if action == 'pull':
+                if not HasPermissionAnyMiddleware('repository.read',
+                                                  'repository.write',
+                                                  'repository.admin')\
+                                                    (user, repo_name):
+                    return HTTPForbidden()(environ, start_response)
+            if action == 'push':
+                if not HasPermissionAnyMiddleware('repository.write',
+                                                  'repository.admin')\
+                                                    (user, repo_name):
+                    return HTTPForbidden()(environ, start_response)
             
-            #===================================================================
-            # CHECK PERMISSIONS FOR THIS REQUEST
-            #===================================================================
-            action = self.__get_action(environ)
-            if action:
-                username = self.__get_environ_user(environ)
-                try:
-                    sa = meta.Session
-                    user = sa.query(User)\
-                        .filter(User.username == username).one()
-                except:
-                    log.error(traceback.format_exc())
-                    return HTTPInternalServerError()(environ, start_response)
-                #check permissions for this repository
-                if action == 'pull':
-                    if not HasPermissionAnyMiddleware('repository.read',
-                                                      'repository.write',
-                                                      'repository.admin')\
-                                                        (user, repo_name):
-                        return HTTPForbidden()(environ, start_response)
-                if action == 'push':
-                    if not HasPermissionAnyMiddleware('repository.write',
-                                                      'repository.admin')\
-                                                        (user, repo_name):
-                        return HTTPForbidden()(environ, start_response)
-                
-                #log action    
-                proxy_key = 'HTTP_X_REAL_IP'
-                def_key = 'REMOTE_ADDR'
-                ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0'))
-                self.__log_user_action(user, action, repo_name, ipaddr)            
-            
-            #===================================================================
-            # MERCURIAL REQUEST HANDLING
-            #===================================================================
-            environ['PATH_INFO'] = '/'#since we wrap into hgweb, reset the path
-            self.baseui = make_ui('db')
-            self.basepath = self.config['base_path']
-            self.repo_path = os.path.join(self.basepath, repo_name)
+            #log action    
+            proxy_key = 'HTTP_X_REAL_IP'
+            def_key = 'REMOTE_ADDR'
+            ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0'))
+            self.__log_user_action(user, action, repo_name, ipaddr)            
+        
+        #===================================================================
+        # MERCURIAL REQUEST HANDLING
+        #===================================================================
+        environ['PATH_INFO'] = '/'#since we wrap into hgweb, reset the path
+        self.baseui = make_ui('db')
+        self.basepath = self.config['base_path']
+        self.repo_path = os.path.join(self.basepath, repo_name)
 
-            #quick check if that dir exists...
-            if check_repo_fast(repo_name, self.basepath):
+        #quick check if that dir exists...
+        if check_repo_fast(repo_name, self.basepath):
+            return HTTPNotFound()(environ, start_response)
+        try:
+            app = wsgiapplication(self.__make_app)
+        except RepoError as e:
+            if str(e).find('not found') != -1:
                 return HTTPNotFound()(environ, start_response)
-            try:
-                app = wsgiapplication(self.__make_app)
-            except RepoError as e:
-                if str(e).find('not found') != -1:
-                    return HTTPNotFound()(environ, start_response)
-            except Exception:
-                log.error(traceback.format_exc())
-                return HTTPInternalServerError()(environ, start_response)
-            
-            #invalidate cache on push
-            if action == 'push':
-                self.__invalidate_cache(repo_name)
-                messages = []
-                messages.append('thank you for using hg-app')
-            
-                return self.msg_wrapper(app, environ, start_response, messages)
-            else:
-                return app(environ, start_response)           
+        except Exception:
+            log.error(traceback.format_exc())
+            return HTTPInternalServerError()(environ, start_response)
+        
+        #invalidate cache on push
+        if action == 'push':
+            self.__invalidate_cache(repo_name)
+            messages = []
+            messages.append('thank you for using hg-app')
+        
+            return self.msg_wrapper(app, environ, start_response, messages)
+        else:
+            return app(environ, start_response)           
 
 
     def msg_wrapper(self, app, environ, start_response, messages=[]):
         """
         Wrapper for custom messages that come out of mercurial respond messages
         is a list of messages that the user will see at the end of response 
         from merurial protocol actions that involves remote answers
         @param app:
         @param environ:
         @param start_response:
         """
         def custom_messages(msg_list):
             for msg in msg_list:
                 yield msg + '\n'
         org_response = app(environ, start_response)
         return chain(org_response, custom_messages(messages))
 
     def __make_app(self):
         hgserve = hgweb(str(self.repo_path), baseui=self.baseui)
         return  self.__load_web_settings(hgserve)
     
     def __get_environ_user(self, environ):
         return environ.get('REMOTE_USER')
     
+    def __get_user(self, username):
+        return get_user_cached(username)
+        
+        
+                        
     def __get_size(self, repo_path, content_size):
         size = int(content_size)
         for path, dirs, files in os.walk(repo_path):
             if path.find('.hg') == -1:
                 for f in files:
                     size += os.path.getsize(os.path.join(path, f))
         return size
         return h.format_byte_size(size)
         
     def __get_action(self, environ):
         """
         Maps mercurial request commands into a pull or push command.
         @param environ:
         """
         mapping = {'changegroup': 'pull',
                    'changegroupsubset': 'pull',
                    'stream_out': 'pull',
                    'listkeys': 'pull',
                    'unbundle': 'push',
                    'pushkey': 'push', }
         
         for qry in environ['QUERY_STRING'].split('&'):
             if qry.startswith('cmd'):
                 cmd = qry.split('=')[-1]
                 if mapping.has_key(cmd):
                     return mapping[cmd]
     
     def __log_user_action(self, user, action, repo, ipaddr):
         sa = meta.Session
         try:
             user_log = UserLog()
             user_log.user_id = user.user_id
             user_log.action = action
             user_log.repository = repo.replace('/', '')
             user_log.action_date = datetime.now()
             user_log.user_ip = ipaddr
             sa.add(user_log)
             sa.commit()
             log.info('Adding user %s, action %s on %s',
                                             user.username, action, repo)
         except Exception as e:
             sa.rollback()
             log.error('could not log user action:%s', str(e))
     
     def __invalidate_cache(self, repo_name):
         """we know that some change was made to repositories and we should
         invalidate the cache to see the changes right away but only for
         push requests"""
         invalidate_cache('cached_repo_list')
         invalidate_cache('full_changelog', repo_name)
            
                    
     def __load_web_settings(self, hgserve):
         #set the global ui for hgserve
         hgserve.repo.ui = self.baseui
         
         hgrc = os.path.join(self.repo_path, '.hg', 'hgrc')
         repoui = make_ui('file', hgrc, False)
         
         if repoui:
             #set the repository based config
             hgserve.repo.ui = repoui
             
         return hgserve

pylons_app/lib/utils.py

@@ -1,189 +1,194 b''
 #!/usr/bin/env python
 # encoding: utf-8
 # Utilities for hg app
 # Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
+from beaker.cache import cache_region
 
 """
 Created on April 18, 2010
 Utilities for hg app
 @author: marcink
 """
 
 import os
 import logging
 from mercurial import ui, config, hg
 from mercurial.error import RepoError
 from pylons_app.model.db import Repository, User, HgAppUi
 log = logging.getLogger(__name__)
 
 
 def get_repo_slug(request):
     return request.environ['pylons.routes_dict'].get('repo_name')
 
 def is_mercurial(environ):
     """
     Returns True if request's target is mercurial server - header
     ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
     """
     http_accept = environ.get('HTTP_ACCEPT')
     if http_accept and http_accept.startswith('application/mercurial'):
         return True
     return False
 
 def check_repo_dir(paths):
     repos_path = paths[0][1].split('/')
     if repos_path[-1] in ['*', '**']:
         repos_path = repos_path[:-1]
     if repos_path[0] != '/':
         repos_path[0] = '/'
     if not os.path.isdir(os.path.join(*repos_path)):
         raise Exception('Not a valid repository in %s' % paths[0][1])
 
 def check_repo_fast(repo_name, base_path):
     if os.path.isdir(os.path.join(base_path, repo_name)):return False
     return True
 
 def check_repo(repo_name, base_path, verify=True):
 
     repo_path = os.path.join(base_path, repo_name)
 
     try:
         if not check_repo_fast(repo_name, base_path):
             return False
         r = hg.repository(ui.ui(), repo_path)
         if verify:
             hg.verify(r)
         #here we hnow that repo exists it was verified
         log.info('%s repo is already created', repo_name)
         return False
     except RepoError:
         #it means that there is no valid repo there...
         log.info('%s repo is free for creation', repo_name)
         return True
 
+
+@cache_region('super_short_term', 'cached_hg_ui')
+def get_hg_ui_cached():
+    from pylons_app.model.meta import Session
+    sa = Session()
+    return sa.query(HgAppUi).all()    
+
 def make_ui(read_from='file', path=None, checkpaths=True):        
     """
     A function that will read python rc files or database
     and make an mercurial ui object from read options
     
     @param path: path to mercurial config file
     @param checkpaths: check the path
     @param read_from: read from 'file' or 'db'
     """
     #propagated from mercurial documentation
     sections = ['alias', 'auth',
                 'decode/encode', 'defaults',
                 'diff', 'email',
                 'extensions', 'format',
                 'merge-patterns', 'merge-tools',
                 'hooks', 'http_proxy',
                 'smtp', 'patch',
                 'paths', 'profiling',
                 'server', 'trusted',
                 'ui', 'web', ]
     baseui = ui.ui()
 
                 
     if read_from == 'file':
         if not os.path.isfile(path):
             log.warning('Unable to read config file %s' % path)
             return False
         
         cfg = config.config()
         cfg.read(path)
         for section in sections:
             for k, v in cfg.items(section):
                 baseui.setconfig(section, k, v)
         if checkpaths:check_repo_dir(cfg.items('paths'))                
               
         
     elif read_from == 'db':
-        from pylons_app.model.meta import Session
-        sa = Session()
-            
-        hg_ui = sa.query(HgAppUi).all()
+        hg_ui = get_hg_ui_cached()
         for ui_ in hg_ui:
             baseui.setconfig(ui_.ui_section, ui_.ui_key, ui_.ui_value)
         
     
     return baseui
 
 
 def set_hg_app_config(config):
     config['hg_app_auth_realm'] = 'realm'
     config['hg_app_name'] = 'app name'
 
 def invalidate_cache(name, *args):
     """Invalidates given name cache"""
     
     from beaker.cache import region_invalidate
     log.info('INVALIDATING CACHE FOR %s', name)
     
     """propagate our arguments to make sure invalidation works. First
     argument has to be the name of cached func name give to cache decorator
     without that the invalidation would not work"""
     tmp = [name]
     tmp.extend(args)
     args = tuple(tmp)
     
     if name == 'cached_repo_list':
         from pylons_app.model.hg_model import _get_repos_cached
         region_invalidate(_get_repos_cached, None, *args)
         
     if name == 'full_changelog':
         from pylons_app.model.hg_model import _full_changelog_cached
         region_invalidate(_full_changelog_cached, None, *args)
         
 from vcs.backends.base import BaseChangeset
 from vcs.utils.lazy import LazyProperty
 class EmptyChangeset(BaseChangeset):
     
     revision = -1
     message = ''
     
     @LazyProperty
     def raw_id(self):
         """
         Returns raw string identifing this changeset, useful for web
         representation.
         """
         return '0' * 12
 
 
 def repo2db_mapper(initial_repo_list):
     """
     maps all found repositories into db
     """
     from pylons_app.model.meta import Session
     from pylons_app.model.repo_model import RepoModel
     
     sa = Session()
     user = sa.query(User).filter(User.admin == True).first()
     
     rm = RepoModel()
     
     for name, repo in initial_repo_list.items():
         if not sa.query(Repository).get(name):
             log.info('repository %s not found creating default', name)
                 
             form_data = {
                          'repo_name':name,
                          'description':repo.description if repo.description != 'unknown' else \
                                         'auto description for %s' % name,
                          'private':False
                          }
             rm.create(form_data, user, just_db=True)