project refactoring, cleaned up lib.utils from rarly used functions, and place them

in proper controllers
moves is_git is_hg functions to the middleware classes

rhodecode/controllers/admin/settings.py

@@ -1,314 +1,335 b''
 #!/usr/bin/env python
 # encoding: utf-8
 # settings controller for pylons
 # Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 """
 Created on July 14, 2010
 settings controller for pylons
 @author: marcink
 """
 from formencode import htmlfill
 from pylons import request, session, tmpl_context as c, url, app_globals as g, \
     config
 from pylons.controllers.util import abort, redirect
 from pylons.i18n.translation import _
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator, \
     HasPermissionAnyDecorator
 from rhodecode.lib.base import BaseController, render
 from rhodecode.lib.celerylib import tasks, run_task
 from rhodecode.lib.utils import repo2db_mapper, invalidate_cache, \
-    set_rhodecode_config, get_hg_settings, get_hg_ui_settings
+    set_rhodecode_config
 from rhodecode.model.db import RhodeCodeUi, Repository
 from rhodecode.model.forms import UserForm, ApplicationSettingsForm, \
     ApplicationUiSettingsForm
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.settings import SettingsModel
 from rhodecode.model.user import UserModel
 from sqlalchemy import func
 import formencode
 import logging
 import traceback
 
 log = logging.getLogger(__name__)
 
 
 class SettingsController(BaseController):
     """REST Controller styled on the Atom Publishing Protocol"""
     # To properly map this controller, ensure your config/routing.py
     # file has a resource setup:
     #     map.resource('setting', 'settings', controller='admin/settings', 
     #         path_prefix='/admin', name_prefix='admin_')
 
 
     @LoginRequired()
     def __before__(self):
         c.admin_user = session.get('admin_user')
         c.admin_username = session.get('admin_username')
         super(SettingsController, self).__before__()
 
 
     @HasPermissionAllDecorator('hg.admin')
     def index(self, format='html'):
         """GET /admin/settings: All items in the collection"""
         # url('admin_settings')
 
-        defaults = get_hg_settings()
-        defaults.update(get_hg_ui_settings())
+        defaults = SettingsModel().get_app_settings()
+        defaults.update(self.get_hg_ui_settings())
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
 
     @HasPermissionAllDecorator('hg.admin')
     def create(self):
         """POST /admin/settings: Create a new item"""
         # url('admin_settings')
 
     @HasPermissionAllDecorator('hg.admin')
     def new(self, format='html'):
         """GET /admin/settings/new: Form to create a new item"""
         # url('admin_new_setting')
 
     @HasPermissionAllDecorator('hg.admin')
     def update(self, setting_id):
         """PUT /admin/settings/setting_id: Update an existing item"""
         # Forms posted to this method should contain a hidden field:
         #    <input type="hidden" name="_method" value="PUT" />
         # Or using helpers:
         #    h.form(url('admin_setting', setting_id=ID),
         #           method='put')
         # url('admin_setting', setting_id=ID)
         if setting_id == 'mapping':
             rm_obsolete = request.POST.get('destroy', False)
             log.debug('Rescanning directories with destroy=%s', rm_obsolete)
 
             initial = ScmModel().repo_scan(g.paths[0][1], g.baseui)
             for repo_name in initial.keys():
                 invalidate_cache('get_repo_cached_%s' % repo_name)
 
             repo2db_mapper(initial, rm_obsolete)
 
             h.flash(_('Repositories successfully rescanned'), category='success')
 
         if setting_id == 'whoosh':
-            repo_location = get_hg_ui_settings()['paths_root_path']
+            repo_location = self.get_hg_ui_settings()['paths_root_path']
             full_index = request.POST.get('full_index', False)
             task = run_task(tasks.whoosh_index, repo_location, full_index)
 
             h.flash(_('Whoosh reindex task scheduled'), category='success')
         if setting_id == 'global':
 
             application_form = ApplicationSettingsForm()()
             try:
                 form_result = application_form.to_python(dict(request.POST))
                 settings_model = SettingsModel()
                 try:
                     hgsettings1 = settings_model.get('title')
                     hgsettings1.app_settings_value = form_result['rhodecode_title']
 
                     hgsettings2 = settings_model.get('realm')
                     hgsettings2.app_settings_value = form_result['rhodecode_realm']
 
 
                     self.sa.add(hgsettings1)
                     self.sa.add(hgsettings2)
                     self.sa.commit()
                     set_rhodecode_config(config)
                     h.flash(_('Updated application settings'),
                             category='success')
 
                 except:
                     log.error(traceback.format_exc())
                     h.flash(_('error occurred during updating application settings'),
                             category='error')
 
                     self.sa.rollback()
 
 
             except formencode.Invalid, errors:
                 return htmlfill.render(
                      render('admin/settings/settings.html'),
                      defaults=errors.value,
                      errors=errors.error_dict or {},
                      prefix_error=False,
                      encoding="UTF-8")
 
         if setting_id == 'mercurial':
             application_form = ApplicationUiSettingsForm()()
             try:
                 form_result = application_form.to_python(dict(request.POST))
 
                 try:
 
                     hgsettings1 = self.sa.query(RhodeCodeUi)\
                     .filter(RhodeCodeUi.ui_key == 'push_ssl').one()
                     hgsettings1.ui_value = form_result['web_push_ssl']
 
                     hgsettings2 = self.sa.query(RhodeCodeUi)\
                     .filter(RhodeCodeUi.ui_key == '/').one()
                     hgsettings2.ui_value = form_result['paths_root_path']
 
 
                     #HOOKS
                     hgsettings3 = self.sa.query(RhodeCodeUi)\
                     .filter(RhodeCodeUi.ui_key == 'changegroup.update').one()
                     hgsettings3.ui_active = bool(form_result['hooks_changegroup_update'])
 
                     hgsettings4 = self.sa.query(RhodeCodeUi)\
                     .filter(RhodeCodeUi.ui_key == 'changegroup.repo_size').one()
                     hgsettings4.ui_active = bool(form_result['hooks_changegroup_repo_size'])
 
                     hgsettings5 = self.sa.query(RhodeCodeUi)\
                     .filter(RhodeCodeUi.ui_key == 'pretxnchangegroup.push_logger').one()
                     hgsettings5.ui_active = bool(form_result['hooks_pretxnchangegroup_push_logger'])
 
                     hgsettings6 = self.sa.query(RhodeCodeUi)\
                     .filter(RhodeCodeUi.ui_key == 'preoutgoing.pull_logger').one()
                     hgsettings6.ui_active = bool(form_result['hooks_preoutgoing_pull_logger'])
 
 
                     self.sa.add(hgsettings1)
                     self.sa.add(hgsettings2)
                     self.sa.add(hgsettings3)
                     self.sa.add(hgsettings4)
                     self.sa.add(hgsettings5)
                     self.sa.add(hgsettings6)
                     self.sa.commit()
 
                     h.flash(_('Updated mercurial settings'),
                             category='success')
 
                 except:
                     log.error(traceback.format_exc())
                     h.flash(_('error occurred during updating application settings'),
                             category='error')
 
                     self.sa.rollback()
 
 
             except formencode.Invalid, errors:
                 return htmlfill.render(
                      render('admin/settings/settings.html'),
                      defaults=errors.value,
                      errors=errors.error_dict or {},
                      prefix_error=False,
                      encoding="UTF-8")
 
 
 
         return redirect(url('admin_settings'))
 
     @HasPermissionAllDecorator('hg.admin')
     def delete(self, setting_id):
         """DELETE /admin/settings/setting_id: Delete an existing item"""
         # Forms posted to this method should contain a hidden field:
         #    <input type="hidden" name="_method" value="DELETE" />
         # Or using helpers:
         #    h.form(url('admin_setting', setting_id=ID),
         #           method='delete')
         # url('admin_setting', setting_id=ID)
 
     @HasPermissionAllDecorator('hg.admin')
     def show(self, setting_id, format='html'):
         """GET /admin/settings/setting_id: Show a specific item"""
         # url('admin_setting', setting_id=ID)
 
     @HasPermissionAllDecorator('hg.admin')
     def edit(self, setting_id, format='html'):
         """GET /admin/settings/setting_id/edit: Form to edit an existing item"""
         # url('admin_edit_setting', setting_id=ID)
 
 
     def my_account(self):
         """
         GET /_admin/my_account Displays info about my account 
         """
 
         # url('admin_settings_my_account')
         c.user = UserModel().get(c.rhodecode_user.user_id, cache=False)
         all_repos = self.sa.query(Repository)\
             .filter(Repository.user_id == c.user.user_id)\
             .order_by(func.lower(Repository.repo_name))\
             .all()
         c.user_repos = ScmModel().get_repos(all_repos)
 
         if c.user.username == 'default':
             h.flash(_("You can't edit this user since it's"
               " crucial for entire application"), category='warning')
             return redirect(url('users'))
 
         defaults = c.user.__dict__
         return htmlfill.render(
             render('admin/users/user_edit_my_account.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
 
     def my_account_update(self):
         """PUT /_admin/my_account_update: Update an existing item"""
         # Forms posted to this method should contain a hidden field:
         #    <input type="hidden" name="_method" value="PUT" />
         # Or using helpers:
         #    h.form(url('admin_settings_my_account_update'),
         #           method='put')
         # url('admin_settings_my_account_update', id=ID)
         user_model = UserModel()
         uid = c.rhodecode_user.user_id
         _form = UserForm(edit=True, old_data={'user_id':uid,
                                               'email':c.rhodecode_user.email})()
         form_result = {}
         try:
             form_result = _form.to_python(dict(request.POST))
             user_model.update_my_account(uid, form_result)
             h.flash(_('Your account was updated succesfully'),
                     category='success')
 
         except formencode.Invalid, errors:
             c.user = user_model.get(c.rhodecode_user.user_id, cache=False)
             c.user = UserModel().get(c.rhodecode_user.user_id, cache=False)
             all_repos = self.sa.query(Repository)\
                 .filter(Repository.user_id == c.user.user_id)\
                 .order_by(func.lower(Repository.repo_name))\
                 .all()
             c.user_repos = ScmModel().get_repos(all_repos)
 
             return htmlfill.render(
                 render('admin/users/user_edit_my_account.html'),
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8")
         except Exception:
             log.error(traceback.format_exc())
             h.flash(_('error occured during update of user %s') \
                     % form_result.get('username'), category='error')
 
         return redirect(url('my_account'))
 
     @HasPermissionAnyDecorator('hg.admin', 'hg.create.repository')
     def create_repository(self):
         """GET /_admin/create_repository: Form to create a new item"""
         new_repo = request.GET.get('repo', '')
         c.new_repo = h.repo_name_slug(new_repo)
 
         return render('admin/repos/repo_add_create_repository.html')
 
+    def get_hg_ui_settings(self):
+        ret = self.sa.query(RhodeCodeUi).all()
+
+        if not ret:
+            raise Exception('Could not get application ui settings !')
+        settings = {}
+        for each in ret:
+            k = each.ui_key
+            v = each.ui_value
+            if k == '/':
+                k = 'root_path'
+
+            if k.find('.') != -1:
+                k = k.replace('.', '_')
+
+            if each.ui_section == 'hooks':
+                v = each.ui_active
+
+            settings[each.ui_section + '_' + k] = v
+
+        return settings

rhodecode/lib/middleware/simplegit.py

@@ -1,209 +1,221 b''
 #!/usr/bin/env python
 # encoding: utf-8
 # middleware to handle git api calls
 # Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 """
 Created on 2010-04-28
 
 @author: marcink
 SimpleGit middleware for handling git protocol request (push/clone etc.)
 It's implemented with basic auth function
 """
 
 from dulwich import server as dulserver
 
 class SimpleGitUploadPackHandler(dulserver.UploadPackHandler):
 
     def handle(self):
         write = lambda x: self.proto.write_sideband(1, x)
 
         graph_walker = dulserver.ProtocolGraphWalker(self, self.repo.object_store,
             self.repo.get_peeled)
         objects_iter = self.repo.fetch_objects(
           graph_walker.determine_wants, graph_walker, self.progress,
           get_tagged=self.get_tagged)
 
         # Do they want any objects?
         if len(objects_iter) == 0:
             return
 
         self.progress("counting objects: %d, done.\n" % len(objects_iter))
         dulserver.write_pack_data(dulserver.ProtocolFile(None, write), objects_iter,
                         len(objects_iter))
         messages = []
         messages.append('thank you for using rhodecode')
 
         for msg in messages:
             self.progress(msg + "\n")
         # we are done
         self.proto.write("0000")
 
 dulserver.DEFAULT_HANDLERS = {
   'git-upload-pack': SimpleGitUploadPackHandler,
   'git-receive-pack': dulserver.ReceivePackHandler,
 }
 
 from dulwich.repo import Repo
 from dulwich.web import HTTPGitApplication
 from paste.auth.basic import AuthBasicAuthenticator
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 from rhodecode.lib.auth import authfunc, HasPermissionAnyMiddleware
-from rhodecode.lib.utils import is_git, invalidate_cache, check_repo_fast
+from rhodecode.lib.utils import invalidate_cache, check_repo_fast
 from rhodecode.model.user import UserModel
 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError
 import logging
 import os
 import traceback
 
 log = logging.getLogger(__name__)
 
+def is_git(environ):
+    """
+    Returns True if request's target is git server. ``HTTP_USER_AGENT`` would
+    then have git client version given.
+    
+    :param environ:
+    """
+    http_user_agent = environ.get('HTTP_USER_AGENT')
+    if http_user_agent and http_user_agent.startswith('git'):
+        return True
+    return False
+
 class SimpleGit(object):
 
     def __init__(self, application, config):
         self.application = application
         self.config = config
         #authenticate this git request using 
         self.authenticate = AuthBasicAuthenticator('', authfunc)
         self.ipaddr = '0.0.0.0'
         self.repository = None
         self.username = None
         self.action = None
 
     def __call__(self, environ, start_response):
         if not is_git(environ):
             return self.application(environ, start_response)
 
         proxy_key = 'HTTP_X_REAL_IP'
         def_key = 'REMOTE_ADDR'
         self.ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0'))
 
         #===================================================================
         # AUTHENTICATE THIS GIT REQUEST
         #===================================================================
         username = REMOTE_USER(environ)
         if not username:
             self.authenticate.realm = self.config['rhodecode_realm']
             result = self.authenticate(environ)
             if isinstance(result, str):
                 AUTH_TYPE.update(environ, 'basic')
                 REMOTE_USER.update(environ, result)
             else:
                 return result.wsgi_application(environ, start_response)
 
         #=======================================================================
         # GET REPOSITORY
         #=======================================================================
         try:
             repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:])
             if repo_name.endswith('/'):
                 repo_name = repo_name.rstrip('/')
             self.repository = repo_name
         except:
             log.error(traceback.format_exc())
             return HTTPInternalServerError()(environ, start_response)
 
         #===================================================================
         # CHECK PERMISSIONS FOR THIS REQUEST
         #===================================================================
         self.action = self.__get_action(environ)
         if self.action:
             username = self.__get_environ_user(environ)
             try:
                 user = self.__get_user(username)
                 self.username = user.username
             except:
                 log.error(traceback.format_exc())
                 return HTTPInternalServerError()(environ, start_response)
 
             #check permissions for this repository
             if self.action == 'push':
                 if not HasPermissionAnyMiddleware('repository.write',
                                                   'repository.admin')\
                                                     (user, repo_name):
                     return HTTPForbidden()(environ, start_response)
 
             else:
                 #any other action need at least read permission
                 if not HasPermissionAnyMiddleware('repository.read',
                                                   'repository.write',
                                                   'repository.admin')\
                                                     (user, repo_name):
                     return HTTPForbidden()(environ, start_response)
 
         self.extras = {'ip':self.ipaddr,
                        'username':self.username,
                        'action':self.action,
                        'repository':self.repository}
 
         #===================================================================
         # GIT REQUEST HANDLING
         #===================================================================
         self.basepath = self.config['base_path']
         self.repo_path = os.path.join(self.basepath, self.repo_name)
         #quick check if that dir exists...
         if check_repo_fast(self.repo_name, self.basepath):
             return HTTPNotFound()(environ, start_response)
         try:
             app = self.__make_app()
         except:
             log.error(traceback.format_exc())
             return HTTPInternalServerError()(environ, start_response)
 
         #invalidate cache on push
         if self.action == 'push':
             self.__invalidate_cache(self.repo_name)
             messages = []
             messages.append('thank you for using rhodecode')
             return app(environ, start_response)
         else:
             return app(environ, start_response)
 
 
     def __make_app(self):
         backend = dulserver.DictBackend({'/' + self.repo_name: Repo(self.repo_path)})
         gitserve = HTTPGitApplication(backend)
 
         return gitserve
 
     def __get_environ_user(self, environ):
         return environ.get('REMOTE_USER')
 
     def __get_user(self, username):
         return UserModel().get_by_username(username, cache=True)
 
     def __get_action(self, environ):
         """
         Maps git request commands into a pull or push command.
         :param environ:
         """
         service = environ['QUERY_STRING'].split('=')
         if len(service) > 1:
             service_cmd = service[1]
             mapping = {'git-receive-pack': 'push',
                        'git-upload-pack': 'pull',
                        }
 
             return mapping.get(service_cmd, service_cmd if service_cmd else 'other')
         else:
             return 'other'
 
     def __invalidate_cache(self, repo_name):
         """we know that some change was made to repositories and we should
         invalidate the cache to see the changes right away but only for
         push requests"""
         invalidate_cache('get_repo_cached_%s' % repo_name)

rhodecode/lib/middleware/simplehg.py

@@ -1,220 +1,229 b''
 #!/usr/bin/env python
 # encoding: utf-8
 # middleware to handle mercurial api calls
 # Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 """
 Created on 2010-04-28
 
 @author: marcink
 SimpleHG middleware for handling mercurial protocol request (push/clone etc.)
 It's implemented with basic auth function
 """
-from itertools import chain
 from mercurial.error import RepoError
 from mercurial.hgweb import hgweb
 from mercurial.hgweb.request import wsgiapplication
 from paste.auth.basic import AuthBasicAuthenticator
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 from rhodecode.lib.auth import authfunc, HasPermissionAnyMiddleware
-from rhodecode.lib.utils import is_mercurial, make_ui, invalidate_cache, \
+from rhodecode.lib.utils import make_ui, invalidate_cache, \
     check_repo_fast, ui_sections
 from rhodecode.model.user import UserModel
 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError
 import logging
 import os
 import traceback
 
 log = logging.getLogger(__name__)
 
+def is_mercurial(environ):
+    """
+    Returns True if request's target is mercurial server - header
+    ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
+    """
+    http_accept = environ.get('HTTP_ACCEPT')
+    if http_accept and http_accept.startswith('application/mercurial'):
+        return True
+    return False
+
 class SimpleHg(object):
 
     def __init__(self, application, config):
         self.application = application
         self.config = config
         #authenticate this mercurial request using authfunc
         self.authenticate = AuthBasicAuthenticator('', authfunc)
         self.ipaddr = '0.0.0.0'
         self.repository = None
         self.username = None
         self.action = None
 
     def __call__(self, environ, start_response):
         if not is_mercurial(environ):
             return self.application(environ, start_response)
 
         proxy_key = 'HTTP_X_REAL_IP'
         def_key = 'REMOTE_ADDR'
         self.ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0'))
 
         #===================================================================
         # AUTHENTICATE THIS MERCURIAL REQUEST
         #===================================================================
         username = REMOTE_USER(environ)
 
         if not username:
             self.authenticate.realm = self.config['rhodecode_realm']
             result = self.authenticate(environ)
             if isinstance(result, str):
                 AUTH_TYPE.update(environ, 'basic')
                 REMOTE_USER.update(environ, result)
             else:
                 return result.wsgi_application(environ, start_response)
 
         #=======================================================================
         # GET REPOSITORY
         #=======================================================================
         try:
             repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:])
             if repo_name.endswith('/'):
                 repo_name = repo_name.rstrip('/')
             self.repository = repo_name
         except:
             log.error(traceback.format_exc())
             return HTTPInternalServerError()(environ, start_response)
 
         #===================================================================
         # CHECK PERMISSIONS FOR THIS REQUEST
         #===================================================================
         self.action = self.__get_action(environ)
         if self.action:
             username = self.__get_environ_user(environ)
             try:
                 user = self.__get_user(username)
                 self.username = user.username
             except:
                 log.error(traceback.format_exc())
                 return HTTPInternalServerError()(environ, start_response)
 
             #check permissions for this repository
             if self.action == 'push':
                 if not HasPermissionAnyMiddleware('repository.write',
                                                   'repository.admin')\
                                                     (user, repo_name):
                     return HTTPForbidden()(environ, start_response)
 
             else:
                 #any other action need at least read permission
                 if not HasPermissionAnyMiddleware('repository.read',
                                                   'repository.write',
                                                   'repository.admin')\
                                                     (user, repo_name):
                     return HTTPForbidden()(environ, start_response)
 
         self.extras = {'ip':self.ipaddr,
                        'username':self.username,
                        'action':self.action,
                        'repository':self.repository}
 
         #===================================================================
         # MERCURIAL REQUEST HANDLING
         #===================================================================
         environ['PATH_INFO'] = '/'#since we wrap into hgweb, reset the path
         self.baseui = make_ui('db')
         self.basepath = self.config['base_path']
         self.repo_path = os.path.join(self.basepath, repo_name)
 
         #quick check if that dir exists...
         if check_repo_fast(repo_name, self.basepath):
             return HTTPNotFound()(environ, start_response)
         try:
             app = wsgiapplication(self.__make_app)
         except RepoError, e:
             if str(e).find('not found') != -1:
                 return HTTPNotFound()(environ, start_response)
         except Exception:
             log.error(traceback.format_exc())
             return HTTPInternalServerError()(environ, start_response)
 
         #invalidate cache on push
         if self.action == 'push':
             self.__invalidate_cache(repo_name)
-        
+
         return app(environ, start_response)
 
 
     def __make_app(self):
         hgserve = hgweb(str(self.repo_path), baseui=self.baseui)
         return  self.__load_web_settings(hgserve, self.extras)
 
     def __get_environ_user(self, environ):
         return environ.get('REMOTE_USER')
 
     def __get_user(self, username):
         return UserModel().get_by_username(username, cache=True)
 
     def __get_action(self, environ):
         """
         Maps mercurial request commands into a clone,pull or push command.
         This should always return a valid command string
         :param environ:
         """
         mapping = {'changegroup': 'pull',
                    'changegroupsubset': 'pull',
                    'stream_out': 'pull',
                    'listkeys': 'pull',
                    'unbundle': 'push',
                    'pushkey': 'push', }
         for qry in environ['QUERY_STRING'].split('&'):
             if qry.startswith('cmd'):
                 cmd = qry.split('=')[-1]
                 if mapping.has_key(cmd):
                     return mapping[cmd]
                 else:
                     return cmd
 
     def __invalidate_cache(self, repo_name):
         """we know that some change was made to repositories and we should
         invalidate the cache to see the changes right away but only for
         push requests"""
         invalidate_cache('get_repo_cached_%s' % repo_name)
 
 
     def __load_web_settings(self, hgserve, extras={}):
         #set the global ui for hgserve instance passed
         hgserve.repo.ui = self.baseui
 
         hgrc = os.path.join(self.repo_path, '.hg', 'hgrc')
 
         #inject some additional parameters that will be available in ui
         #for hooks
         for k, v in extras.items():
             hgserve.repo.ui.setconfig('rhodecode_extras', k, v)
 
         repoui = make_ui('file', hgrc, False)
 
         if repoui:
             #overwrite our ui instance with the section from hgrc file
             for section in ui_sections:
                 for k, v in repoui.configitems(section):
                     hgserve.repo.ui.setconfig(section, k, v)
 
         return hgserve
 
 
 
 
 
 
 
 
 
 
 
 
 
 

rhodecode/lib/utils.py

@@ -1,585 +1,508 b''
 #!/usr/bin/env python
 # encoding: utf-8
 # Utilities for RhodeCode
 # Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 """
 Created on April 18, 2010
 Utilities for RhodeCode
 @author: marcink
 """
 
 from UserDict import DictMixin
 from mercurial import ui, config, hg
 from mercurial.error import RepoError
 from rhodecode.model import meta
 from rhodecode.model.caching_query import FromCache
-from rhodecode.model.db import Repository, User, RhodeCodeUi, RhodeCodeSettings, \
-    UserLog
+from rhodecode.model.db import Repository, User, RhodeCodeUi, UserLog
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.user import UserModel
+
 from vcs.backends.base import BaseChangeset
 from paste.script import command
 import ConfigParser
 from vcs.utils.lazy import LazyProperty
 import traceback
 import datetime
 import logging
 import os
 
 log = logging.getLogger(__name__)
 
 
 def get_repo_slug(request):
     return request.environ['pylons.routes_dict'].get('repo_name')
 
-def is_mercurial(environ):
-    """
-    Returns True if request's target is mercurial server - header
-    ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
-    """
-    http_accept = environ.get('HTTP_ACCEPT')
-    if http_accept and http_accept.startswith('application/mercurial'):
-        return True
-    return False
-
-def is_git(environ):
-    """
-    Returns True if request's target is git server. ``HTTP_USER_AGENT`` would
-    then have git client version given.
-    
-    :param environ:
-    """
-    http_user_agent = environ.get('HTTP_USER_AGENT')
-    if http_user_agent and http_user_agent.startswith('git'):
-        return True
-    return False
-
 def action_logger(user, action, repo, ipaddr='', sa=None):
     """
     Action logger for various actions made by users
     
     :param user: user that made this action, can be a unique username string or
         object containing user_id attribute
     :param action: action to log, should be on of predefined unique actions for
         easy translations
     :param repo: string name of repository or object containing repo_id,
         that action was made on
     :param ipaddr: optional ip address from what the action was made
     :param sa: optional sqlalchemy session
     
     """
 
     if not sa:
         sa = meta.Session()
 
     try:
         um = UserModel()
         if hasattr(user, 'user_id'):
             user_obj = user
         elif isinstance(user, basestring):
             user_obj = um.get_by_username(user, cache=False)
         else:
             raise Exception('You have to provide user object or username')
 
 
         rm = RepoModel()
         if hasattr(repo, 'repo_id'):
             repo_obj = rm.get(repo.repo_id, cache=False)
             repo_name = repo_obj.repo_name
         elif  isinstance(repo, basestring):
             repo_name = repo.lstrip('/')
             repo_obj = rm.get_by_repo_name(repo_name, cache=False)
         else:
             raise Exception('You have to provide repository to action logger')
 
 
         user_log = UserLog()
         user_log.user_id = user_obj.user_id
         user_log.action = action
-        
+
         user_log.repository_id = repo_obj.repo_id
         user_log.repository_name = repo_name
-        
+
         user_log.action_date = datetime.datetime.now()
         user_log.user_ip = ipaddr
         sa.add(user_log)
         sa.commit()
 
-        log.info('Adding user %s, action %s on %s',
-                                        user_obj.username, action, repo)
+        log.info('Adding user %s, action %s on %s', user_obj, action, repo)
     except:
         log.error(traceback.format_exc())
         sa.rollback()
 
 def get_repos(path, recursive=False, initial=False):
     """
     Scans given path for repos and return (name,(type,path)) tuple 
     :param prefix:
     :param path:
     :param recursive:
     :param initial:
     """
     from vcs.utils.helpers import get_scm
     from vcs.exceptions import VCSError
 
     try:
         scm = get_scm(path)
     except:
         pass
     else:
         raise Exception('The given path %s should not be a repository got %s',
                         path, scm)
 
     for dirpath in os.listdir(path):
         try:
             yield dirpath, get_scm(os.path.join(path, dirpath))
         except VCSError:
             pass
 
-if __name__ == '__main__':
-    get_repos('', '/home/marcink/workspace-python')
-
-
 def check_repo_fast(repo_name, base_path):
     if os.path.isdir(os.path.join(base_path, repo_name)):return False
     return True
 
 def check_repo(repo_name, base_path, verify=True):
 
     repo_path = os.path.join(base_path, repo_name)
 
     try:
         if not check_repo_fast(repo_name, base_path):
             return False
         r = hg.repository(ui.ui(), repo_path)
         if verify:
             hg.verify(r)
         #here we hnow that repo exists it was verified
         log.info('%s repo is already created', repo_name)
         return False
     except RepoError:
         #it means that there is no valid repo there...
         log.info('%s repo is free for creation', repo_name)
         return True
 
 def ask_ok(prompt, retries=4, complaint='Yes or no, please!'):
     while True:
         ok = raw_input(prompt)
         if ok in ('y', 'ye', 'yes'): return True
         if ok in ('n', 'no', 'nop', 'nope'): return False
         retries = retries - 1
         if retries < 0: raise IOError
         print complaint
 
-def get_hg_ui_cached():
-    try:
-        sa = meta.Session
-        ret = sa.query(RhodeCodeUi)\
-        .options(FromCache("sql_cache_short", "get_hg_ui_settings"))\
-        .all()
-    except:
-        pass
-    finally:
-        meta.Session.remove()
-    return ret
-
-
-def get_hg_settings():
-    try:
-        sa = meta.Session()
-        ret = sa.query(RhodeCodeSettings)\
-        .options(FromCache("sql_cache_short", "get_hg_settings"))\
-        .all()
-    except:
-        pass
-    finally:
-        meta.Session.remove()
-
-    if not ret:
-        raise Exception('Could not get application settings !')
-    settings = {}
-    for each in ret:
-        settings['rhodecode_' + each.app_settings_name] = each.app_settings_value
-
-    return settings
-
-def get_hg_ui_settings():
-    try:
-        sa = meta.Session()
-        ret = sa.query(RhodeCodeUi).all()
-    except:
-        pass
-    finally:
-        meta.Session.remove()
-
-    if not ret:
-        raise Exception('Could not get application ui settings !')
-    settings = {}
-    for each in ret:
-        k = each.ui_key
-        v = each.ui_value
-        if k == '/':
-            k = 'root_path'
-
-        if k.find('.') != -1:
-            k = k.replace('.', '_')
-
-        if each.ui_section == 'hooks':
-            v = each.ui_active
-
-        settings[each.ui_section + '_' + k] = v
-
-    return settings
-
 #propagated from mercurial documentation
 ui_sections = ['alias', 'auth',
                 'decode/encode', 'defaults',
                 'diff', 'email',
                 'extensions', 'format',
                 'merge-patterns', 'merge-tools',
                 'hooks', 'http_proxy',
                 'smtp', 'patch',
                 'paths', 'profiling',
                 'server', 'trusted',
                 'ui', 'web', ]
 
 def make_ui(read_from='file', path=None, checkpaths=True):
     """
     A function that will read python rc files or database
     and make an mercurial ui object from read options
     
     :param path: path to mercurial config file
     :param checkpaths: check the path
     :param read_from: read from 'file' or 'db'
     """
 
     baseui = ui.ui()
 
     #clean the baseui object
     baseui._ocfg = config.config()
     baseui._ucfg = config.config()
     baseui._tcfg = config.config()
 
     if read_from == 'file':
         if not os.path.isfile(path):
             log.warning('Unable to read config file %s' % path)
             return False
         log.debug('reading hgrc from %s', path)
         cfg = config.config()
         cfg.read(path)
         for section in ui_sections:
             for k, v in cfg.items(section):
                 log.debug('settings ui from file[%s]%s:%s', section, k, v)
                 baseui.setconfig(section, k, v)
 
 
     elif read_from == 'db':
-        hg_ui = get_hg_ui_cached()
+        sa = meta.Session()
+        ret = sa.query(RhodeCodeUi)\
+            .options(FromCache("sql_cache_short",
+                               "get_hg_ui_settings")).all()
+        meta.Session.remove()
+        hg_ui = ret
         for ui_ in hg_ui:
             if ui_.ui_active:
                 log.debug('settings ui from db[%s]%s:%s', ui_.ui_section, ui_.ui_key, ui_.ui_value)
                 baseui.setconfig(ui_.ui_section, ui_.ui_key, ui_.ui_value)
     return baseui
 
 
 def set_rhodecode_config(config):
-    hgsettings = get_hg_settings()
+    """
+    Updates pylons config with new settings from database
+    :param config:
+    """
+    from rhodecode.model.settings import SettingsModel
+    hgsettings = SettingsModel().get_app_settings()
 
     for k, v in hgsettings.items():
         config[k] = v
 
 def invalidate_cache(cache_key, *args):
     """
     Puts cache invalidation task into db for 
     further global cache invalidation
     """
     from rhodecode.model.scm import ScmModel
 
     if cache_key.startswith('get_repo_cached_'):
         name = cache_key.split('get_repo_cached_')[-1]
         ScmModel().mark_for_invalidation(name)
 
 class EmptyChangeset(BaseChangeset):
     """
     An dummy empty changeset. It's possible to pass hash when creating
     an EmptyChangeset
     """
 
     def __init__(self, cs='0' * 40):
         self._empty_cs = cs
         self.revision = -1
         self.message = ''
         self.author = ''
         self.date = ''
 
     @LazyProperty
     def raw_id(self):
         """
         Returns raw string identifying this changeset, useful for web
         representation.
         """
         return self._empty_cs
 
     @LazyProperty
     def short_id(self):
         return self.raw_id[:12]
 
     def get_file_changeset(self, path):
         return self
 
     def get_file_content(self, path):
         return u''
 
     def get_file_size(self, path):
         return 0
 
 def repo2db_mapper(initial_repo_list, remove_obsolete=False):
     """
     maps all found repositories into db
     """
 
     sa = meta.Session()
     rm = RepoModel()
     user = sa.query(User).filter(User.admin == True).first()
 
     for name, repo in initial_repo_list.items():
         if not rm.get_by_repo_name(name, cache=False):
             log.info('repository %s not found creating default', name)
 
             form_data = {
                          'repo_name':name,
                          'repo_type':repo.alias,
                          'description':repo.description \
                             if repo.description != 'unknown' else \
                                         '%s repository' % name,
                          'private':False
                          }
             rm.create(form_data, user, just_db=True)
 
     if remove_obsolete:
         #remove from database those repositories that are not in the filesystem
         for repo in sa.query(Repository).all():
             if repo.repo_name not in initial_repo_list.keys():
                 sa.delete(repo)
                 sa.commit()
 
 class OrderedDict(dict, DictMixin):
 
     def __init__(self, *args, **kwds):
         if len(args) > 1:
             raise TypeError('expected at most 1 arguments, got %d' % len(args))
         try:
             self.__end
         except AttributeError:
             self.clear()
         self.update(*args, **kwds)
 
     def clear(self):
         self.__end = end = []
         end += [None, end, end]         # sentinel node for doubly linked list
         self.__map = {}                 # key --> [key, prev, next]
         dict.clear(self)
 
     def __setitem__(self, key, value):
         if key not in self:
             end = self.__end
             curr = end[1]
             curr[2] = end[1] = self.__map[key] = [key, curr, end]
         dict.__setitem__(self, key, value)
 
     def __delitem__(self, key):
         dict.__delitem__(self, key)
         key, prev, next = self.__map.pop(key)
         prev[2] = next
         next[1] = prev
 
     def __iter__(self):
         end = self.__end
         curr = end[2]
         while curr is not end:
             yield curr[0]
             curr = curr[2]
 
     def __reversed__(self):
         end = self.__end
         curr = end[1]
         while curr is not end:
             yield curr[0]
             curr = curr[1]
 
     def popitem(self, last=True):
         if not self:
             raise KeyError('dictionary is empty')
         if last:
             key = reversed(self).next()
         else:
             key = iter(self).next()
         value = self.pop(key)
         return key, value
 
     def __reduce__(self):
         items = [[k, self[k]] for k in self]
         tmp = self.__map, self.__end
         del self.__map, self.__end
         inst_dict = vars(self).copy()
         self.__map, self.__end = tmp
         if inst_dict:
             return (self.__class__, (items,), inst_dict)
         return self.__class__, (items,)
 
     def keys(self):
         return list(self)
 
     setdefault = DictMixin.setdefault
     update = DictMixin.update
     pop = DictMixin.pop
     values = DictMixin.values
     items = DictMixin.items
     iterkeys = DictMixin.iterkeys
     itervalues = DictMixin.itervalues
     iteritems = DictMixin.iteritems
 
     def __repr__(self):
         if not self:
             return '%s()' % (self.__class__.__name__,)
         return '%s(%r)' % (self.__class__.__name__, self.items())
 
     def copy(self):
         return self.__class__(self)
 
     @classmethod
     def fromkeys(cls, iterable, value=None):
         d = cls()
         for key in iterable:
             d[key] = value
         return d
 
     def __eq__(self, other):
         if isinstance(other, OrderedDict):
             return len(self) == len(other) and self.items() == other.items()
         return dict.__eq__(self, other)
 
     def __ne__(self, other):
         return not self == other
 
 
 #===============================================================================
 # TEST FUNCTIONS AND CREATORS
 #===============================================================================
 def create_test_index(repo_location, full_index):
     """Makes default test index
     :param repo_location:
     :param full_index:
     """
     from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
     from rhodecode.lib.pidlock import DaemonLock, LockHeld
     import shutil
 
     index_location = os.path.join(repo_location, 'index')
     if os.path.exists(index_location):
         shutil.rmtree(index_location)
 
     try:
         l = DaemonLock()
         WhooshIndexingDaemon(index_location=index_location,
                              repo_location=repo_location)\
             .run(full_index=full_index)
         l.release()
     except LockHeld:
         pass
 
 def create_test_env(repos_test_path, config):
     """Makes a fresh database and 
     install test repository into tmp dir
     """
     from rhodecode.lib.db_manage import DbManage
     from rhodecode.tests import HG_REPO, GIT_REPO, NEW_HG_REPO, NEW_GIT_REPO, \
         HG_FORK, GIT_FORK, TESTS_TMP_PATH
     import tarfile
     import shutil
     from os.path import dirname as dn, join as jn, abspath
 
     log = logging.getLogger('TestEnvCreator')
     # create logger
     log.setLevel(logging.DEBUG)
     log.propagate = True
     # create console handler and set level to debug
     ch = logging.StreamHandler()
     ch.setLevel(logging.DEBUG)
 
     # create formatter
     formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
 
     # add formatter to ch
     ch.setFormatter(formatter)
 
     # add ch to logger
     log.addHandler(ch)
 
     #PART ONE create db
     dbname = config['sqlalchemy.db1.url'].split('/')[-1]
     log.debug('making test db %s', dbname)
 
     dbmanage = DbManage(log_sql=True, dbname=dbname, root=config['here'],
                         tests=True)
     dbmanage.create_tables(override=True)
     dbmanage.config_prompt(repos_test_path)
     dbmanage.create_default_user()
     dbmanage.admin_prompt()
     dbmanage.create_permissions()
     dbmanage.populate_default_permissions()
 
     #PART TWO make test repo
     log.debug('making test vcs repositories')
 
     #remove old one from previos tests
     for r in [HG_REPO, GIT_REPO, NEW_HG_REPO, NEW_GIT_REPO, HG_FORK, GIT_FORK]:
 
         if os.path.isdir(jn(TESTS_TMP_PATH, r)):
             log.debug('removing %s', r)
             shutil.rmtree(jn(TESTS_TMP_PATH, r))
 
     #CREATE DEFAULT HG REPOSITORY
     cur_dir = dn(dn(abspath(__file__)))
     tar = tarfile.open(jn(cur_dir, 'tests', "vcs_test_hg.tar.gz"))
     tar.extractall(jn(TESTS_TMP_PATH, HG_REPO))
     tar.close()
 
 class UpgradeDb(command.Command):
     """Command used for paster to upgrade our database to newer version
     """
 
     max_args = 1
     min_args = 1
 
     usage = "CONFIG_FILE"
     summary = "Upgrades current db to newer version given configuration file"
     group_name = "RhodeCode"
 
     parser = command.Command.standard_parser(verbose=True)
 
     parser.add_option('--sql',
                       action='store_true',
                       dest='just_sql',
                       help="Prints upgrade sql for further investigation",
                       default=False)
     def command(self):
         config_name = self.args[0]
         p = config_name.split('/')
         root = '.' if len(p) == 1 else '/'.join(p[:-1])
         config = ConfigParser.ConfigParser({'here':root})
         config.read(config_name)

rhodecode/model/settings.py

@@ -1,76 +1,87 b''
 #!/usr/bin/env python
 # encoding: utf-8
 # Model for RhodeCode settings
 # Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
 # 
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 """
 Created on Nov 17, 2010
 Model for RhodeCode
 :author: marcink
 """
 
 from rhodecode.lib import helpers as h
 from rhodecode.model import BaseModel
 from rhodecode.model.caching_query import FromCache
 from rhodecode.model.db import  RhodeCodeSettings
 from sqlalchemy.orm import joinedload
-from sqlalchemy.orm.session import make_transient
 import logging
 
 log = logging.getLogger(__name__)
 
 class SettingsModel(BaseModel):
     """
     Settings model
     """
 
     def get(self, settings_key, cache=False):
         r = self.sa.query(RhodeCodeSettings)\
             .filter(RhodeCodeSettings.app_settings_name == settings_key).scalar()
         if cache:
             r = r.options(FromCache("sql_cache_short",
                                           "get_setting_%s" % settings_key))
         return r
 
+    def get_app_settings(self):
+        ret = self.sa.query(RhodeCodeSettings)\
+            .options(FromCache("sql_cache_short",
+                           "get_hg_settings")).all()
+
+        if not ret:
+            raise Exception('Could not get application settings !')
+        settings = {}
+        for each in ret:
+            settings['rhodecode_' + each.app_settings_name] = each.app_settings_value
+
+        return settings
 
     def get_ldap_settings(self):
         """
         Returns ldap settings from database
         :returns:
         ldap_active
         ldap_host
         ldap_port 
         ldap_ldaps
         ldap_dn_user 
         ldap_dn_pass 
         ldap_base_dn
         """
 
         r = self.sa.query(RhodeCodeSettings)\
                 .filter(RhodeCodeSettings.app_settings_name\
                         .startswith('ldap_'))\
                 .all()
 
         fd = {}
 
         for row in r:
             v = row.app_settings_value
             if v in ['0', '1']:
                 v = v == '1'
             fd.update({row.app_settings_name:v})
 
         return fd