Made repos path config configurable from pylons app configs. update Readme

README.txt

@@ -1,18 +1,24 b''
 Pylons based replacement for hgwebdir. Fully customizable, 
 with authentication, permissions. Based on vcs library.
 - has it's own middleware to handle mercurial protocol request each request can
   be logged and authenticated +threaded performance unlikely to hgweb
 - mako templates let's you cusmotize look and feel of appplication.
 - diffs annotations and source code all colored by pygments.
 - admin interface for performing user/permission managments as well as repository
   managment
 - added cache with invalidation on push/repo managment for high performance and
   always upto date data.
 - rss /atom feed customizable
 - future support for git
 - based on pylons 1.0 / sqlalchemy 0.6
 
 ===
 This software is still in beta mode. I don't guarantee that it'll work.
 I started this project since i was tired of sad looks, and zero controll over
 our company regular hgwebdir.
+
+
+== INSTALATION
+run dbmanage.py from pylons_app/lib it should create all needed table and
+an admin account, Edit file repositories.config and change the path for you 
+mercurial repositories, remember about permissions.
  No newline at end of file

development.ini

@@ -1,135 +1,136 b''
 ################################################################################
 ################################################################################
 # pylons_app - Pylons environment configuration                                #
 #                                                                              # 
 # The %(here)s variable will be replaced with the parent directory of this file#
 ################################################################################
 
 [DEFAULT]
 debug = true
 ############################################
 ## Uncomment and replace with the address ##
 ## which should receive any error reports ##
 ############################################
 #email_to = [e-mail]
 #smtp_server = mail.etelko.pl
 #error_email_from = paste_error@localhost
 #smtp_username = 
 #smtp_password = 
 #error_message = 'mercurial crash !'
 
 [server:main]
 ##nr of threads to spawn
 threadpool_workers = 5
 
 ##max request before
 threadpool_max_requests = 2
 
 ##option to use threads of process
 use_threadpool = true
 
 use = egg:Paste#http
 host = 127.0.0.1
 port = 5000
 
 [app:main]
 use = egg:pylons_app
 full_stack = true
 static_files = false
 lang=en
 cache_dir = %(here)s/data
 ##a name for our application
 hg_app_name = Python-works
+hg_app_repo_conf = repositories.config
 
 ####################################
 ###         BEAKER CACHE        ####
 ####################################
 beaker.cache.data_dir=/tmp/cache/data
 beaker.cache.lock_dir=/tmp/cache/lock
 beaker.cache.regions=short_term,long_term
 beaker.cache.long_term.type=memory
 beaker.cache.long_term.expire=36000
 beaker.cache.short_term.type=memory
 beaker.cache.short_term.expire=60
 
 ################################################################################
 ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 ## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 ## execute malicious code after an exception is raised.                       ##
 ################################################################################
 #set debug = false
 
 ##################################
 ###       LOGVIEW CONFIG       ###
 ##################################
 logview.sqlalchemy = #faa
 logview.pylons.templating = #bfb
 logview.pylons.util = #eee
 
 #########################################################
 ### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###
 #########################################################
 sqlalchemy.db1.url = sqlite:///%(here)s/hg_app.db
 #sqlalchemy.db1.echo = False
 #sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.convert_unicode = true
 
 ################################
 ### LOGGING CONFIGURATION   ####
 ################################
 [loggers]
 keys = root, routes, pylons_app, sqlalchemy
 
 [handlers]
 keys = console
 
 [formatters]
 keys = generic,color_formatter
 
 #############
 ## LOGGERS ##
 #############
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_routes]
 level = DEBUG
 handlers = console
 qualname = routes.middleware
 # "level = DEBUG" logs the route matched and routing variables.
 
 [logger_pylons_app]
 level = DEBUG
 handlers = console
 qualname = pylons_app
 propagate = 0
 
 [logger_sqlalchemy]
 level = ERROR
 handlers = console
 qualname = sqlalchemy.engine
 propagate = 0
 
 ##############
 ## HANDLERS ##
 ##############
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
 level = NOTSET
 formatter = color_formatter
 
 ################
 ## FORMATTERS ##
 ################
 
 [formatter_generic]
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class=pylons_app.lib.colored_formatter.ColorFormatter
 format= %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
  No newline at end of file

production.ini

@@ -1,135 +1,136 b''
 ################################################################################
 ################################################################################
 # pylons_app - Pylons environment configuration                                #
 #                                                                              # 
 # The %(here)s variable will be replaced with the parent directory of this file#
 ################################################################################
 
 [DEFAULT]
 debug = true
 ############################################
 ## Uncomment and replace with the address ##
 ## which should receive any error reports ##
 ############################################
 #email_to = [e-mail]
 #smtp_server = mail.etelko.pl
 #error_email_from = paste_error@localhost
 #smtp_username = 
 #smtp_password = 
 #error_message = 'mercurial crash !'
 
 [server:main]
 ##nr of threads to spawn
 threadpool_workers = 5
 
 ##max request before
 threadpool_max_requests = 2
 
 ##option to use threads of process
 use_threadpool = true
 
 use = egg:Paste#http
 host = 127.0.0.1
 port = 8001
 
 [app:main]
 use = egg:pylons_app
 full_stack = true
 static_files = false
 lang=en
 cache_dir = %(here)s/data
 ##a name for our application
 hg_app_name = Python-works
+hg_app_repo_conf = repositories.config
 
 ####################################
 ###         BEAKER CACHE        ####
 ####################################
 beaker.cache.data_dir=/tmp/cache/data
 beaker.cache.lock_dir=/tmp/cache/lock
 beaker.cache.regions=short_term,long_term
 beaker.cache.long_term.type=memory
 beaker.cache.long_term.expire=36000
 beaker.cache.short_term.type=memory
 beaker.cache.short_term.expire=60
     
 ################################################################################
 ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 ## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 ## execute malicious code after an exception is raised.                       ##
 ################################################################################
 set debug = false
 
 ##################################
 ###       LOGVIEW CONFIG       ###
 ##################################
 logview.sqlalchemy = #faa
 logview.pylons.templating = #bfb
 logview.pylons.util = #eee
 
 #########################################################
 ### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###
 #########################################################
 sqlalchemy.db1.url = sqlite:///%(here)s/hg_app.db
 #sqlalchemy.db1.echo = False
 #sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.convert_unicode = true
 
 ################################
 ### LOGGING CONFIGURATION   ####
 ################################
 [loggers]
 keys = root, routes, pylons_app, sqlalchemy
 
 [handlers]
 keys = console
 
 [formatters]
 keys = generic,color_formatter
 
 #############
 ## LOGGERS ##
 #############
 [logger_root]
 level = INFO
 handlers = console
 
 [logger_routes]
 level = INFO
 handlers = console
 qualname = routes.middleware
 # "level = DEBUG" logs the route matched and routing variables.
 
 [logger_pylons_app]
 level = DEBUG
 handlers = console
 qualname = pylons_app
 propagate = 0
 
 [logger_sqlalchemy]
 level = ERROR
 handlers = console
 qualname = sqlalchemy.engine
 propagate = 0
 
 ##############
 ## HANDLERS ##
 ##############
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
 level = NOTSET
 formatter = color_formatter
 
 ################
 ## FORMATTERS ##
 ################
 
 [formatter_generic]
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class=pylons_app.lib.colored_formatter.ColorFormatter
 format= %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
  No newline at end of file

pylons_app/lib/app_globals.py

@@ -1,25 +1,25 b''
 """The application's Globals object"""
 
 from beaker.cache import CacheManager
 from beaker.util import parse_cache_config_options
 from pylons_app.lib.utils import make_ui
 
 class Globals(object):
 
     """Globals acts as a container for objects available throughout the
     life of the application
 
     """
 
     def __init__(self, config):
         """One instance of Globals is created during application
         initialization and is available during requests via the
         'app_globals' variable
 
         """
         self.cache = CacheManager(**parse_cache_config_options(config))
-        self.baseui = make_ui('hgwebdir.config')
+        self.baseui = make_ui(config['hg_app_repo_conf'])
         self.paths = self.baseui.configitems('paths')
         self.base_path = self.paths[0][1].replace('*', '')
         self.changeset_annotation_colors = {}
         self.available_permissions = None # propagated after init_model

pylons_app/lib/backup_manager.py

@@ -1,88 +1,88 b''
 '''BACKUP MANAGER'''
 import logging
 from mercurial import config
 import tarfile
 import os
 import datetime
 import sys
 import subprocess
 logging.basicConfig(level=logging.DEBUG,
                     format="%(asctime)s %(levelname)-5.5s %(message)s")
 
 class BackupManager(object):
-    def __init__(self):
+    def __init__(self, id_rsa_path, repo_conf):
         self.repos_path = None
         self.backup_file_name = None
-        self.id_rsa_path = '/home/pylons/id_rsa'
+        self.id_rsa_path = id_rsa_path
         self.check_id_rsa()
         cur_dir = os.path.realpath(__file__)
         dn = os.path.dirname
         self.backup_file_path = os.path.join(dn(dn(dn(cur_dir))), 'data')
         cfg = config.config()
         try:
-            cfg.read(os.path.join(dn(dn(dn(cur_dir))), 'hgwebdir.config'))
+            cfg.read(os.path.join(dn(dn(dn(cur_dir))), repo_conf))
         except IOError:
-            logging.error('Could not read hgwebdir.config')
+            logging.error('Could not read %s', repo_conf)
             sys.exit()
         self.set_repos_path(cfg.items('paths'))
         logging.info('starting backup for %s', self.repos_path)
         logging.info('backup target %s', self.backup_file_path)
 
         if not os.path.isdir(self.repos_path):
             raise Exception('Not a valid directory in %s' % self.repos_path)
 
     def check_id_rsa(self):
         if not os.path.isfile(self.id_rsa_path):
             logging.error('Could not load id_rsa key file in %s',
                           self.id_rsa_path)
             sys.exit()
 
     def set_repos_path(self, paths):
         repos_path = paths[0][1].split('/')
         if repos_path[-1] in ['*', '**']:
             repos_path = repos_path[:-1]
         if repos_path[0] != '/':
             repos_path[0] = '/'
         self.repos_path = os.path.join(*repos_path)
 
     def backup_repos(self):
         today = datetime.datetime.now().weekday() + 1
         self.backup_file_name = "mercurial_repos.%s.tar.gz" % today
         bckp_file = os.path.join(self.backup_file_path, self.backup_file_name)
         tar = tarfile.open(bckp_file, "w:gz")
 
         for dir_name in os.listdir(self.repos_path):
             logging.info('backing up %s', dir_name)
             tar.add(os.path.join(self.repos_path, dir_name), dir_name)
         tar.close()
         logging.info('finished backup of mercurial repositories')
 
 
 
     def transfer_files(self):
         params = {
                   'id_rsa_key': self.id_rsa_path,
                   'backup_file_path':self.backup_file_path,
                   'backup_file_name':self.backup_file_name,
                   }
         cmd = ['scp', '-l', '40000', '-i', '%(id_rsa_key)s' % params,
                '%(backup_file_path)s/%(backup_file_name)s' % params,
                'root@192.168.2.102:/backups/mercurial' % params]
 
         subprocess.call(cmd)
         logging.info('Transfered file %s to %s', self.backup_file_name, cmd[4])
         
     
     def rm_file(self):
         logging.info('Removing file %s', self.backup_file_name)
         os.remove(os.path.join(self.backup_file_path, self.backup_file_name))
     
 
 
 if __name__ == "__main__":
-    B_MANAGER = BackupManager()
+    B_MANAGER = BackupManager('/home/pylons/id_rsa', 'repositories.config')
     B_MANAGER.backup_repos()
     B_MANAGER.transfer_files()
     B_MANAGER.rm_file()
 
 

pylons_app/lib/middleware/simplehg.py

@@ -1,136 +1,136 b''
 #!/usr/bin/env python
 # encoding: utf-8
 #
 # Copyright (c) 2010 marcink.  All rights reserved.
 #
 """
 Created on 2010-04-28
 
 @author: marcink
 SimpleHG middleware for handling mercurial protocol request (push/clone etc.)
 It's implemented with basic auth function
 """
 from datetime import datetime
 from mercurial.hgweb import hgweb
 from mercurial.hgweb.request import wsgiapplication
 from paste.auth.basic import AuthBasicAuthenticator
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 from pylons_app.lib.auth import authfunc
 from pylons_app.lib.utils import is_mercurial, make_ui, invalidate_cache
 from pylons_app.model import meta
 from pylons_app.model.db import UserLog, User
 from webob.exc import HTTPNotFound
 import logging
 import os
 log = logging.getLogger(__name__)
 
 class SimpleHg(object):
 
     def __init__(self, application, config):
         self.application = application
         self.config = config
         #authenticate this mercurial request using 
         realm = '%s %s' % (config['hg_app_name'], 'mercurial repository')
         self.authenticate = AuthBasicAuthenticator(realm, authfunc)
         
     def __call__(self, environ, start_response):
         if not is_mercurial(environ):
             return self.application(environ, start_response)
         else:
             #===================================================================
             # AUTHENTICATE THIS MERCURIAL REQUEST
             #===================================================================
             username = REMOTE_USER(environ)
             if not username:
                 result = self.authenticate(environ)
                 if isinstance(result, str):
                     AUTH_TYPE.update(environ, 'basic')
                     REMOTE_USER.update(environ, result)
                 else:
                     return result.wsgi_application(environ, start_response)
             
             try:
                 repo_name = environ['PATH_INFO'].split('/')[1]
             except:
                 return HTTPNotFound()(environ, start_response)
             
             #since we wrap into hgweb, just reset the path
             environ['PATH_INFO'] = '/'
-            self.baseui = make_ui()
+            self.baseui = make_ui(self.config['hg_app_repo_conf'])
             self.basepath = self.baseui.configitems('paths')[0][1]\
                                                             .replace('*', '')
             self.repo_path = os.path.join(self.basepath, repo_name)
             try:
                 app = wsgiapplication(self.__make_app)
             except Exception as e:
                 return HTTPNotFound()(environ, start_response)
             action = self.__get_action(environ)            
             #invalidate cache on push
             if action == 'push':
                 self.__invalidate_cache(repo_name)
             
             if action:
                 username = self.__get_environ_user(environ)
                 self.__log_user_action(username, action, repo_name)
                          
             return app(environ, start_response)            
 
     def __make_app(self):
         hgserve = hgweb(self.repo_path)
         return  self.__load_web_settings(hgserve)
     
     def __get_environ_user(self, environ):
         return environ.get('REMOTE_USER')
         
     def __get_action(self, environ):
         """
         Maps mercurial request commands into a pull or push command.
         @param environ:
         """
         mapping = {
             'changegroup': 'pull',
             'changegroupsubset': 'pull',
             'unbundle': 'push',
             'stream_out': 'pull',
         }                    
         for qry in environ['QUERY_STRING'].split('&'):
             if qry.startswith('cmd'):
                 cmd = qry.split('=')[-1]
                 if mapping.has_key(cmd):
                     return mapping[cmd]
     
     def __log_user_action(self, username, action, repo):
         sa = meta.Session
         try:
             user = sa.query(User).filter(User.username == username).one()
             user_log = UserLog()
             user_log.user_id = user.user_id
             user_log.action = action
             user_log.repository = repo.replace('/', '')
             user_log.action_date = datetime.now()
             sa.add(user_log)
             sa.commit()
             log.info('Adding user %s, action %s on %s',
                                             username, action, repo)
         except Exception as e:
             sa.rollback()
             log.error('could not log user action:%s', str(e))
     
     def __invalidate_cache(self, repo_name):
         """we know that some change was made to repositories and we should
         invalidate the cache to see the changes right away but only for
         push requests"""
         invalidate_cache('cached_repo_list')
         invalidate_cache('full_changelog', repo_name)
            
                    
     def __load_web_settings(self, hgserve):
         repoui = make_ui(os.path.join(self.repo_path, '.hg', 'hgrc'), False)
         #set the global ui for hgserve
         hgserve.repo.ui = self.baseui
         
         if repoui:
             #set the repository based config
             hgserve.repo.ui = repoui
             
         return hgserve

pylons_app/lib/utils.py

@@ -1,127 +1,130 b''
 import os
 import logging
 from mercurial import ui, config, hg
 from mercurial.error import RepoError
 log = logging.getLogger(__name__)
 
 
 def get_repo_slug(request):
     path_info = request.environ.get('PATH_INFO')
     uri_lst = path_info.split('/')   
     repo_name = uri_lst[1]
     return repo_name
 
 def is_mercurial(environ):
     """
     Returns True if request's target is mercurial server - header
     ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
     """
     http_accept = environ.get('HTTP_ACCEPT')
     if http_accept and http_accept.startswith('application/mercurial'):
         return True
     return False
 
 def check_repo_dir(paths):
     repos_path = paths[0][1].split('/')
     if repos_path[-1] in ['*', '**']:
         repos_path = repos_path[:-1]
     if repos_path[0] != '/':
         repos_path[0] = '/'
     if not os.path.isdir(os.path.join(*repos_path)):
         raise Exception('Not a valid repository in %s' % paths[0][1])
 
 def check_repo(repo_name, base_path):
 
     repo_path = os.path.join(base_path, repo_name)
 
     try:
         r = hg.repository(ui.ui(), repo_path)
         hg.verify(r)
         #here we hnow that repo exists it was verified
         log.info('%s repo is already created', repo_name)
         return False
         #raise Exception('Repo exists')
     except RepoError:
         log.info('%s repo is free for creation', repo_name)
         #it means that there is no valid repo there...
         return True
                 
-def make_ui(path='hgwebdir.config', checkpaths=True):        
+def make_ui(path=None, checkpaths=True):        
     """
     A funcion that will read python rc files and make an ui from read options
     
     @param path: path to mercurial config file
     """
+    if not path:
+        log.error('repos config path is empty !')
+    
     if not os.path.isfile(path):
         log.warning('Unable to read config file %s' % path)
         return False
     #propagated from mercurial documentation
     sections = [
                 'alias',
                 'auth',
                 'decode/encode',
                 'defaults',
                 'diff',
                 'email',
                 'extensions',
                 'format',
                 'merge-patterns',
                 'merge-tools',
                 'hooks',
                 'http_proxy',
                 'smtp',
                 'patch',
                 'paths',
                 'profiling',
                 'server',
                 'trusted',
                 'ui',
                 'web',
                 ]
 
     baseui = ui.ui()
     cfg = config.config()
     cfg.read(path)
     if checkpaths:check_repo_dir(cfg.items('paths'))
 
     for section in sections:
         for k, v in cfg.items(section):
             baseui.setconfig(section, k, v)
     
     return baseui
 
 def invalidate_cache(name, *args):
     """Invalidates given name cache"""
     
     from beaker.cache import region_invalidate
     log.info('INVALIDATING CACHE FOR %s', name)
     
     """propagate our arguments to make sure invalidation works. First
     argument has to be the name of cached func name give to cache decorator
     without that the invalidation would not work"""
     tmp = [name]
     tmp.extend(args)
     args = tuple(tmp)
     
     if name == 'cached_repo_list':
         from pylons_app.lib.base import _get_repos_cached
         region_invalidate(_get_repos_cached, None, *args)
         
     if name == 'full_changelog':
         from pylons_app.lib.base import _full_changelog_cached
         region_invalidate(_full_changelog_cached, None, *args)
         
 from vcs.backends.base import BaseChangeset
 from vcs.utils.lazy import LazyProperty
 class EmptyChangeset(BaseChangeset):
     
     revision = -1
 
     @LazyProperty
     def raw_id(self):
         """
         Returns raw string identifing this changeset, useful for web
         representation.
         """
         return '0' * 12