# HG changeset patch # User Marcin Kuzminski # Date 2010-10-20 23:38:14 # Node ID 7e536d1af60d3ea018abd0c7ae625d05d31e7b88 # Parent dd06bdf974c8d40b802123b9b905b5694460ad4c Code refactoring,models renames cleaned up sqlalchemy sessions, added cache support to most queries in models fixed test.ini file diff --git a/rhodecode/config/environment.py b/rhodecode/config/environment.py --- a/rhodecode/config/environment.py +++ b/rhodecode/config/environment.py @@ -6,7 +6,7 @@ from rhodecode.config.routing import mak from rhodecode.lib.auth import set_available_permissions, set_base_path from rhodecode.lib.utils import repo2db_mapper, make_ui, set_rhodecode_config from rhodecode.model import init_model -from rhodecode.model.hg_model import _get_repos_cached_initial +from rhodecode.model.hg import _get_repos_cached_initial from sqlalchemy import engine_from_config import logging import os diff --git a/rhodecode/controllers/admin/permissions.py b/rhodecode/controllers/admin/permissions.py --- a/rhodecode/controllers/admin/permissions.py +++ b/rhodecode/controllers/admin/permissions.py @@ -30,10 +30,9 @@ from pylons.i18n.translation import _ from rhodecode.lib import helpers as h from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator from rhodecode.lib.base import BaseController, render -from rhodecode.model.db import User, UserLog from rhodecode.model.forms import UserForm, DefaultPermissionsForm from rhodecode.model.permission_model import PermissionModel -from rhodecode.model.user_model import UserModel +from rhodecode.model.user import UserModel import formencode import logging import traceback @@ -45,14 +44,14 @@ class PermissionsController(BaseControll # To properly map this controller, ensure your config/routing.py # file has a resource setup: # map.resource('permission', 'permissions') - + @LoginRequired() @HasPermissionAllDecorator('hg.admin') def __before__(self): c.admin_user = session.get('admin_user') c.admin_username = session.get('admin_username') super(PermissionsController, self).__before__() - + self.perms_choices = [('repository.none', _('None'),), ('repository.read', _('Read'),), ('repository.write', _('Write'),), @@ -63,11 +62,11 @@ class PermissionsController(BaseControll _('allowed with manual account activation')), ('hg.register.auto_activate', _('allowed with automatic account activation')), ] - + self.create_choices = [('hg.create.none', _('Disabled')), - ('hg.create.repository', _('Enabled'))] + ('hg.create.repository', _('Enabled'))] - + def index(self, format='html'): """GET /permissions: All items in the collection""" # url('permissions') @@ -88,38 +87,38 @@ class PermissionsController(BaseControll # h.form(url('permission', id=ID), # method='put') # url('permission', id=ID) - + permission_model = PermissionModel() - + _form = DefaultPermissionsForm([x[0] for x in self.perms_choices], [x[0] for x in self.register_choices], [x[0] for x in self.create_choices])() - + try: form_result = _form.to_python(dict(request.POST)) form_result.update({'perm_user_name':id}) permission_model.update(form_result) h.flash(_('Default permissions updated succesfully'), category='success') - + except formencode.Invalid, errors: c.perms_choices = self.perms_choices c.register_choices = self.register_choices c.create_choices = self.create_choices - + return htmlfill.render( render('admin/permissions/permissions.html'), defaults=errors.value, errors=errors.error_dict or {}, prefix_error=False, - encoding="UTF-8") + encoding="UTF-8") except Exception: log.error(traceback.format_exc()) h.flash(_('error occured during update of permissions'), category='error') - + return redirect(url('edit_permission', id=id)) - + def delete(self, id): @@ -141,23 +140,23 @@ class PermissionsController(BaseControll c.perms_choices = self.perms_choices c.register_choices = self.register_choices c.create_choices = self.create_choices - + if id == 'default': defaults = {'_method':'put'} for p in UserModel().get_default().user_perms: if p.permission.permission_name.startswith('repository.'): - defaults['default_perm'] = p.permission.permission_name - + defaults['default_perm'] = p.permission.permission_name + if p.permission.permission_name.startswith('hg.register.'): defaults['default_register'] = p.permission.permission_name - + if p.permission.permission_name.startswith('hg.create.'): defaults['default_create'] = p.permission.permission_name - + return htmlfill.render( render('admin/permissions/permissions.html'), defaults=defaults, encoding="UTF-8", - force_defaults=True,) + force_defaults=True,) else: return redirect(url('admin_home')) diff --git a/rhodecode/controllers/admin/repos.py b/rhodecode/controllers/admin/repos.py --- a/rhodecode/controllers/admin/repos.py +++ b/rhodecode/controllers/admin/repos.py @@ -35,8 +35,8 @@ from rhodecode.lib.base import BaseContr from rhodecode.lib.utils import invalidate_cache, action_logger from rhodecode.model.db import User from rhodecode.model.forms import RepoForm -from rhodecode.model.hg_model import HgModel -from rhodecode.model.repo_model import RepoModel +from rhodecode.model.hg import HgModel +from rhodecode.model.repo import RepoModel import formencode import logging import traceback diff --git a/rhodecode/controllers/admin/settings.py b/rhodecode/controllers/admin/settings.py --- a/rhodecode/controllers/admin/settings.py +++ b/rhodecode/controllers/admin/settings.py @@ -32,17 +32,17 @@ from rhodecode.lib.auth import LoginRequ HasPermissionAnyDecorator from rhodecode.lib.base import BaseController, render from rhodecode.lib.utils import repo2db_mapper, invalidate_cache, \ - set_rhodecode_config, get_hg_settings, get_hg_ui_settings, make_ui -from rhodecode.model.db import User, UserLog, RhodeCodeSettings, RhodeCodeUi + set_rhodecode_config, get_hg_settings, get_hg_ui_settings +from rhodecode.model.db import RhodeCodeSettings, RhodeCodeUi from rhodecode.model.forms import UserForm, ApplicationSettingsForm, \ ApplicationUiSettingsForm -from rhodecode.model.hg_model import HgModel -from rhodecode.model.user_model import UserModel +from rhodecode.model.hg import HgModel +from rhodecode.model.user import UserModel from rhodecode.lib.celerylib import tasks, run_task import formencode import logging import traceback - + log = logging.getLogger(__name__) @@ -59,9 +59,9 @@ class SettingsController(BaseController) c.admin_user = session.get('admin_user') c.admin_username = session.get('admin_username') super(SettingsController, self).__before__() - - - @HasPermissionAllDecorator('hg.admin') + + + @HasPermissionAllDecorator('hg.admin') def index(self, format='html'): """GET /admin/settings: All items in the collection""" # url('admin_settings') @@ -73,18 +73,18 @@ class SettingsController(BaseController) defaults=defaults, encoding="UTF-8", force_defaults=False - ) - + ) + @HasPermissionAllDecorator('hg.admin') def create(self): """POST /admin/settings: Create a new item""" # url('admin_settings') - + @HasPermissionAllDecorator('hg.admin') def new(self, format='html'): """GET /admin/settings/new: Form to create a new item""" # url('admin_new_setting') - + @HasPermissionAllDecorator('hg.admin') def update(self, setting_id): """PUT /admin/settings/setting_id: Update an existing item""" @@ -101,44 +101,44 @@ class SettingsController(BaseController) initial = HgModel.repo_scan(g.paths[0][0], g.paths[0][1], g.baseui) repo2db_mapper(initial, rm_obsolete) invalidate_cache('cached_repo_list') - h.flash(_('Repositories successfully rescanned'), category='success') - + h.flash(_('Repositories successfully rescanned'), category='success') + if setting_id == 'whoosh': repo_location = get_hg_ui_settings()['paths_root_path'] full_index = request.POST.get('full_index', False) task = run_task(tasks.whoosh_index, repo_location, full_index) - + h.flash(_('Whoosh reindex task scheduled'), category='success') if setting_id == 'global': - + application_form = ApplicationSettingsForm()() try: form_result = application_form.to_python(dict(request.POST)) - + try: hgsettings1 = self.sa.query(RhodeCodeSettings)\ .filter(RhodeCodeSettings.app_settings_name == 'title').one() - hgsettings1.app_settings_value = form_result['rhodecode_title'] - + hgsettings1.app_settings_value = form_result['rhodecode_title'] + hgsettings2 = self.sa.query(RhodeCodeSettings)\ .filter(RhodeCodeSettings.app_settings_name == 'realm').one() - hgsettings2.app_settings_value = form_result['rhodecode_realm'] - - + hgsettings2.app_settings_value = form_result['rhodecode_realm'] + + self.sa.add(hgsettings1) self.sa.add(hgsettings2) self.sa.commit() set_rhodecode_config(config) h.flash(_('Updated application settings'), category='success') - + except: log.error(traceback.format_exc()) h.flash(_('error occurred during updating application settings'), category='error') - + self.sa.rollback() - + except formencode.Invalid, errors: return htmlfill.render( @@ -146,52 +146,52 @@ class SettingsController(BaseController) defaults=errors.value, errors=errors.error_dict or {}, prefix_error=False, - encoding="UTF-8") - + encoding="UTF-8") + if setting_id == 'mercurial': application_form = ApplicationUiSettingsForm()() try: form_result = application_form.to_python(dict(request.POST)) - + try: - + hgsettings1 = self.sa.query(RhodeCodeUi)\ .filter(RhodeCodeUi.ui_key == 'push_ssl').one() hgsettings1.ui_value = form_result['web_push_ssl'] - + hgsettings2 = self.sa.query(RhodeCodeUi)\ .filter(RhodeCodeUi.ui_key == '/').one() - hgsettings2.ui_value = form_result['paths_root_path'] - - + hgsettings2.ui_value = form_result['paths_root_path'] + + #HOOKS hgsettings3 = self.sa.query(RhodeCodeUi)\ .filter(RhodeCodeUi.ui_key == 'changegroup.update').one() - hgsettings3.ui_active = bool(form_result['hooks_changegroup_update']) - + hgsettings3.ui_active = bool(form_result['hooks_changegroup_update']) + hgsettings4 = self.sa.query(RhodeCodeUi)\ .filter(RhodeCodeUi.ui_key == 'changegroup.repo_size').one() - hgsettings4.ui_active = bool(form_result['hooks_changegroup_repo_size']) - - - - + hgsettings4.ui_active = bool(form_result['hooks_changegroup_repo_size']) + + + + self.sa.add(hgsettings1) self.sa.add(hgsettings2) self.sa.add(hgsettings3) self.sa.add(hgsettings4) self.sa.commit() - + h.flash(_('Updated mercurial settings'), category='success') - + except: log.error(traceback.format_exc()) h.flash(_('error occurred during updating application settings'), category='error') - + self.sa.rollback() - + except formencode.Invalid, errors: return htmlfill.render( @@ -199,12 +199,12 @@ class SettingsController(BaseController) defaults=errors.value, errors=errors.error_dict or {}, prefix_error=False, - encoding="UTF-8") - - - + encoding="UTF-8") + + + return redirect(url('admin_settings')) - + @HasPermissionAllDecorator('hg.admin') def delete(self, setting_id): """DELETE /admin/settings/setting_id: Delete an existing item""" @@ -214,13 +214,13 @@ class SettingsController(BaseController) # h.form(url('admin_setting', setting_id=ID), # method='delete') # url('admin_setting', setting_id=ID) - + @HasPermissionAllDecorator('hg.admin') def show(self, setting_id, format='html'): """GET /admin/settings/setting_id: Show a specific item""" # url('admin_setting', setting_id=ID) - - @HasPermissionAllDecorator('hg.admin') + + @HasPermissionAllDecorator('hg.admin') def edit(self, setting_id, format='html'): """GET /admin/settings/setting_id/edit: Form to edit an existing item""" # url('admin_edit_setting', setting_id=ID) @@ -231,24 +231,24 @@ class SettingsController(BaseController) GET /_admin/my_account Displays info about my account """ # url('admin_settings_my_account') - c.user = self.sa.query(User).get(c.rhodecode_user.user_id) + c.user = UserModel(self.sa).get(c.rhodecode_user.user_id, cache=False) c.user_repos = [] for repo in c.cached_repo_list.values(): if repo.dbrepo.user.username == c.user.username: c.user_repos.append(repo) - + if c.user.username == 'default': - h.flash(_("You can't edit this user since it's" + h.flash(_("You can't edit this user since it's" " crucial for entire application"), category='warning') return redirect(url('users')) - + defaults = c.user.__dict__ return htmlfill.render( render('admin/users/user_edit_my_account.html'), defaults=defaults, encoding="UTF-8", force_defaults=False - ) + ) def my_account_update(self): """PUT /_admin/my_account_update: Update an existing item""" @@ -268,13 +268,13 @@ class SettingsController(BaseController) user_model.update_my_account(uid, form_result) h.flash(_('Your account was updated succesfully'), category='success') - + except formencode.Invalid, errors: - c.user = self.sa.query(User).get(c.rhodecode_user.user_id) + c.user = user_model.get(c.rhodecode_user.user_id, cache=False) c.user_repos = [] for repo in c.cached_repo_list.values(): if repo.dbrepo.user.username == c.user.username: - c.user_repos.append(repo) + c.user_repos.append(repo) return htmlfill.render( render('admin/users/user_edit_my_account.html'), defaults=errors.value, @@ -285,9 +285,9 @@ class SettingsController(BaseController) log.error(traceback.format_exc()) h.flash(_('error occured during update of user %s') \ % form_result.get('username'), category='error') - + return redirect(url('my_account')) - + @HasPermissionAnyDecorator('hg.admin', 'hg.create.repository') def create_repository(self): """GET /_admin/create_repository: Form to create a new item""" @@ -295,4 +295,4 @@ class SettingsController(BaseController) c.new_repo = h.repo_name_slug(new_repo) return render('admin/repos/repo_add_create_repository.html') - + diff --git a/rhodecode/controllers/admin/users.py b/rhodecode/controllers/admin/users.py --- a/rhodecode/controllers/admin/users.py +++ b/rhodecode/controllers/admin/users.py @@ -33,7 +33,7 @@ from rhodecode.lib.auth import LoginRequ from rhodecode.lib.base import BaseController, render from rhodecode.model.db import User, UserLog from rhodecode.model.forms import UserForm -from rhodecode.model.user_model import UserModel, DefaultUserException +from rhodecode.model.user import UserModel, DefaultUserException import formencode import logging import traceback @@ -100,7 +100,7 @@ class UsersController(BaseController): # method='put') # url('user', id=ID) user_model = UserModel() - c.user = user_model.get_user(id) + c.user = user_model.get(id) _form = UserForm(edit=True, old_data={'user_id':id, 'email':c.user.email})() diff --git a/rhodecode/controllers/branches.py b/rhodecode/controllers/branches.py --- a/rhodecode/controllers/branches.py +++ b/rhodecode/controllers/branches.py @@ -26,7 +26,7 @@ from pylons import tmpl_context as c from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator from rhodecode.lib.base import BaseController, render from rhodecode.lib.utils import OrderedDict -from rhodecode.model.hg_model import HgModel +from rhodecode.model.hg import HgModel import logging log = logging.getLogger(__name__) diff --git a/rhodecode/controllers/changelog.py b/rhodecode/controllers/changelog.py --- a/rhodecode/controllers/changelog.py +++ b/rhodecode/controllers/changelog.py @@ -32,7 +32,7 @@ from mercurial.graphmod import colored, from pylons import request, session, tmpl_context as c from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator from rhodecode.lib.base import BaseController, render -from rhodecode.model.hg_model import HgModel +from rhodecode.model.hg import HgModel from webhelpers.paginate import Page import logging log = logging.getLogger(__name__) diff --git a/rhodecode/controllers/changeset.py b/rhodecode/controllers/changeset.py --- a/rhodecode/controllers/changeset.py +++ b/rhodecode/controllers/changeset.py @@ -27,7 +27,7 @@ from pylons.i18n.translation import _ from pylons.controllers.util import redirect from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator from rhodecode.lib.base import BaseController, render -from rhodecode.model.hg_model import HgModel +from rhodecode.model.hg import HgModel from vcs.exceptions import RepositoryError, ChangesetError from vcs.nodes import FileNode from vcs.utils import diffs as differ diff --git a/rhodecode/controllers/feed.py b/rhodecode/controllers/feed.py --- a/rhodecode/controllers/feed.py +++ b/rhodecode/controllers/feed.py @@ -24,7 +24,7 @@ feed controller for pylons """ from pylons import tmpl_context as c, url, response from rhodecode.lib.base import BaseController, render -from rhodecode.model.hg_model import HgModel +from rhodecode.model.hg import HgModel from webhelpers.feedgenerator import Atom1Feed, Rss201rev2Feed import logging log = logging.getLogger(__name__) diff --git a/rhodecode/controllers/files.py b/rhodecode/controllers/files.py --- a/rhodecode/controllers/files.py +++ b/rhodecode/controllers/files.py @@ -29,7 +29,7 @@ from pylons.controllers.util import redi from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator from rhodecode.lib.base import BaseController, render from rhodecode.lib.utils import EmptyChangeset -from rhodecode.model.hg_model import HgModel +from rhodecode.model.hg import HgModel from vcs.exceptions import RepositoryError, ChangesetError from vcs.nodes import FileNode from vcs.utils import diffs as differ diff --git a/rhodecode/controllers/hg.py b/rhodecode/controllers/hg.py --- a/rhodecode/controllers/hg.py +++ b/rhodecode/controllers/hg.py @@ -26,7 +26,7 @@ from operator import itemgetter from pylons import tmpl_context as c, request from rhodecode.lib.auth import LoginRequired from rhodecode.lib.base import BaseController, render -from rhodecode.model.hg_model import HgModel +from rhodecode.model.hg import HgModel import logging log = logging.getLogger(__name__) diff --git a/rhodecode/controllers/login.py b/rhodecode/controllers/login.py --- a/rhodecode/controllers/login.py +++ b/rhodecode/controllers/login.py @@ -31,7 +31,7 @@ from rhodecode.lib.base import BaseContr import rhodecode.lib.helpers as h from pylons.i18n.translation import _ from rhodecode.model.forms import LoginForm, RegisterForm, PasswordResetForm -from rhodecode.model.user_model import UserModel +from rhodecode.model.user import UserModel import formencode import logging @@ -55,7 +55,7 @@ class LoginController(BaseController): try: c.form_result = login_form.to_python(dict(request.POST)) username = c.form_result['username'] - user = UserModel().get_user_by_name(username) + user = UserModel().get_by_username(username) auth_user = AuthUser() auth_user.username = user.username auth_user.is_authenticated = True @@ -89,7 +89,7 @@ class LoginController(BaseController): def register(self): user_model = UserModel() c.auto_active = False - for perm in user_model.get_default().user_perms: + for perm in user_model.get_by_username('default', cache=False).user_perms: if perm.permission.permission_name == 'hg.register.auto_activate': c.auto_active = True break diff --git a/rhodecode/controllers/settings.py b/rhodecode/controllers/settings.py --- a/rhodecode/controllers/settings.py +++ b/rhodecode/controllers/settings.py @@ -30,7 +30,7 @@ from rhodecode.lib.auth import LoginRequ from rhodecode.lib.base import BaseController, render from rhodecode.lib.utils import invalidate_cache, action_logger from rhodecode.model.forms import RepoSettingsForm, RepoForkForm -from rhodecode.model.repo_model import RepoModel +from rhodecode.model.repo import RepoModel import formencode import logging import rhodecode.lib.helpers as h diff --git a/rhodecode/controllers/shortlog.py b/rhodecode/controllers/shortlog.py --- a/rhodecode/controllers/shortlog.py +++ b/rhodecode/controllers/shortlog.py @@ -25,7 +25,7 @@ shortlog controller for pylons from pylons import tmpl_context as c, request from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator from rhodecode.lib.base import BaseController, render -from rhodecode.model.hg_model import HgModel +from rhodecode.model.hg import HgModel from webhelpers.paginate import Page import logging log = logging.getLogger(__name__) diff --git a/rhodecode/controllers/summary.py b/rhodecode/controllers/summary.py --- a/rhodecode/controllers/summary.py +++ b/rhodecode/controllers/summary.py @@ -26,7 +26,7 @@ from pylons import tmpl_context as c, re from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator from rhodecode.lib.base import BaseController, render from rhodecode.lib.utils import OrderedDict -from rhodecode.model.hg_model import HgModel +from rhodecode.model.hg import HgModel from rhodecode.model.db import Statistics from webhelpers.paginate import Page from rhodecode.lib.celerylib import run_task diff --git a/rhodecode/controllers/tags.py b/rhodecode/controllers/tags.py --- a/rhodecode/controllers/tags.py +++ b/rhodecode/controllers/tags.py @@ -26,7 +26,7 @@ from pylons import tmpl_context as c from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator from rhodecode.lib.base import BaseController, render from rhodecode.lib.utils import OrderedDict -from rhodecode.model.hg_model import HgModel +from rhodecode.model.hg import HgModel import logging log = logging.getLogger(__name__) diff --git a/rhodecode/lib/auth.py b/rhodecode/lib/auth.py --- a/rhodecode/lib/auth.py +++ b/rhodecode/lib/auth.py @@ -22,16 +22,13 @@ Created on April 4, 2010 @author: marcink """ -from beaker.cache import cache_region from pylons import config, session, url, request from pylons.controllers.util import abort, redirect from rhodecode.lib.utils import get_repo_slug from rhodecode.model import meta from rhodecode.model.caching_query import FromCache from rhodecode.model.db import User, RepoToPerm, Repository, Permission, \ - UserToPerm -from sqlalchemy.exc import OperationalError -from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound + UserToPerm import bcrypt from decorator import decorator import logging @@ -74,22 +71,10 @@ def get_crypt_password(password): def check_password(password, hashed): return bcrypt.hashpw(password, hashed) == hashed -@cache_region('super_short_term', 'cached_user') -def get_user_cached(username): - sa = meta.Session - try: - user = sa.query(User).filter(User.username == username).one() - finally: - meta.Session.remove() - return user - def authfunc(environ, username, password): - try: - user = get_user_cached(username) - except (NoResultFound, MultipleResultsFound, OperationalError), e: - log.error(e) - user = None - + from rhodecode.model.user import UserModel + user = UserModel().get_by_username(username, cache=False) + if user: if user.active: if user.username == username and check_password(password, user.password): @@ -125,8 +110,10 @@ def set_available_permissions(config): """ log.info('getting information about all available permissions') try: - sa = meta.Session + sa = meta.Session() all_perms = sa.query(Permission).all() + except: + pass finally: meta.Session.remove() @@ -141,10 +128,16 @@ def fill_data(user): in database :param user: """ - sa = meta.Session - dbuser = sa.query(User).options(FromCache('sql_cache_short', - 'getuser_%s' % user.user_id))\ + sa = meta.Session() + try: + dbuser = sa.query(User)\ + .options(FromCache('sql_cache_short', 'getuser_%s' % user.user_id))\ .get(user.user_id) + except: + pass + finally: + meta.Session.remove() + if dbuser: user.username = dbuser.username user.is_admin = dbuser.admin @@ -153,7 +146,8 @@ def fill_data(user): user.email = dbuser.email else: user.is_authenticated = False - meta.Session.remove() + + return user def fill_perms(user): @@ -162,7 +156,7 @@ def fill_perms(user): :param user: """ - sa = meta.Session + sa = meta.Session() user.permissions['repositories'] = {} user.permissions['global'] = set() @@ -170,7 +164,7 @@ def fill_perms(user): # fetch default permissions #=========================================================================== default_user = sa.query(User)\ - .options(FromCache('sql_cache_short','getuser_%s' % 'default'))\ + .options(FromCache('sql_cache_short', 'getuser_%s' % 'default'))\ .filter(User.username == 'default').scalar() default_perms = sa.query(RepoToPerm, Repository, Permission)\ diff --git a/rhodecode/lib/base.py b/rhodecode/lib/base.py --- a/rhodecode/lib/base.py +++ b/rhodecode/lib/base.py @@ -9,7 +9,7 @@ from rhodecode import __version__ from rhodecode.lib import auth from rhodecode.lib.utils import get_repo_slug from rhodecode.model import meta -from rhodecode.model.hg_model import _get_repos_cached, \ +from rhodecode.model.hg import _get_repos_cached, \ _get_repos_switcher_cached class BaseController(WSGIController): @@ -31,7 +31,7 @@ class BaseController(WSGIController): c.repository_tags = {} c.repository_branches = {} - self.sa = meta.Session + self.sa = meta.Session() def __call__(self, environ, start_response): """Invoke the Controller""" diff --git a/rhodecode/lib/celerylib/tasks.py b/rhodecode/lib/celerylib/tasks.py --- a/rhodecode/lib/celerylib/tasks.py +++ b/rhodecode/lib/celerylib/tasks.py @@ -38,28 +38,28 @@ def get_session(): else: #If we don't use celery reuse our current application Session from rhodecode.model.meta import Session - sa = Session - + sa = Session() + return sa def get_hg_settings(): from rhodecode.model.db import RhodeCodeSettings sa = get_session() ret = sa.query(RhodeCodeSettings).all() - + if not ret: raise Exception('Could not get application settings !') settings = {} for each in ret: - settings['rhodecode_' + each.app_settings_name] = each.app_settings_value - + settings['rhodecode_' + each.app_settings_name] = each.app_settings_value + return settings def get_hg_ui_settings(): from rhodecode.model.db import RhodeCodeUi sa = get_session() ret = sa.query(RhodeCodeUi).all() - + if not ret: raise Exception('Could not get application ui settings !') settings = {} @@ -68,16 +68,16 @@ def get_hg_ui_settings(): v = each.ui_value if k == '/': k = 'root_path' - + if k.find('.') != -1: k = k.replace('.', '_') - + if each.ui_section == 'hooks': v = each.ui_active - - settings[each.ui_section + '_' + k] = v - - return settings + + settings[each.ui_section + '_' + k] = v + + return settings @task @locked_task @@ -92,7 +92,7 @@ def get_commits_stats(repo_name, ts_min_ from rhodecode.model.db import Statistics, Repository log = get_commits_stats.get_logger() author_key_cleaner = lambda k: person(k).replace('"', "") #for js data compatibilty - + commits_by_day_author_aggregate = {} commits_by_day_aggregate = {} repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '') @@ -103,9 +103,9 @@ def get_commits_stats(repo_name, ts_min_ last_rev = 0 last_cs = None timegetter = itemgetter('time') - + sa = get_session() - + dbrepo = sa.query(Repository)\ .filter(Repository.repo_name == repo_name).scalar() cur_stats = sa.query(Statistics)\ @@ -114,18 +114,18 @@ def get_commits_stats(repo_name, ts_min_ last_rev = cur_stats.stat_on_revision if not repo.revisions: return True - + if last_rev == repo.revisions[-1] and len(repo.revisions) > 1: #pass silently without any work if we're not on first revision or current #state of parsing revision(from db marker) is the last revision return True - + if cur_stats: commits_by_day_aggregate = OrderedDict( json.loads( cur_stats.commit_activity_combined)) commits_by_day_author_aggregate = json.loads(cur_stats.commit_activity) - + log.debug('starting parsing %s', parse_limit) for cnt, rev in enumerate(repo.revisions[last_rev:]): last_cs = cs = repo.get_changeset(rev) @@ -141,20 +141,20 @@ def get_commits_stats(repo_name, ts_min_ time_pos = l.index(k) except ValueError: time_pos = False - + if time_pos >= 0 and time_pos is not False: - + datadict = commits_by_day_author_aggregate\ [author_key_cleaner(cs.author)]['data'][time_pos] - + datadict["commits"] += 1 datadict["added"] += len(cs.added) datadict["changed"] += len(cs.changed) datadict["removed"] += len(cs.removed) - + else: if k >= ts_min_y and k <= ts_max_y or skip_date_limit: - + datadict = {"time":k, "commits":1, "added":len(cs.added), @@ -163,7 +163,7 @@ def get_commits_stats(repo_name, ts_min_ } commits_by_day_author_aggregate\ [author_key_cleaner(cs.author)]['data'].append(datadict) - + else: if k >= ts_min_y and k <= ts_max_y or skip_date_limit: commits_by_day_author_aggregate[author_key_cleaner(cs.author)] = { @@ -175,14 +175,14 @@ def get_commits_stats(repo_name, ts_min_ "removed":len(cs.removed), }], "schema":["commits"], - } - + } + #gather all data by day if commits_by_day_aggregate.has_key(k): commits_by_day_aggregate[k] += 1 else: commits_by_day_aggregate[k] = 1 - + if cnt >= parse_limit: #don't fetch to much data since we can freeze application break @@ -191,7 +191,7 @@ def get_commits_stats(repo_name, ts_min_ for k, v in commits_by_day_aggregate.items(): overview_data.append([k, v]) overview_data = sorted(overview_data, key=itemgetter(0)) - + if not commits_by_day_author_aggregate: commits_by_day_author_aggregate[author_key_cleaner(repo.contact)] = { "label":author_key_cleaner(repo.contact), @@ -206,23 +206,23 @@ def get_commits_stats(repo_name, ts_min_ log.debug('last revison %s', last_rev) leftovers = len(repo.revisions[last_rev:]) log.debug('revisions to parse %s', leftovers) - - if last_rev == 0 or leftovers < parse_limit: + + if last_rev == 0 or leftovers < parse_limit: stats.languages = json.dumps(__get_codes_stats(repo_name)) - + stats.repository = dbrepo stats.stat_on_revision = last_cs.revision - + try: sa.add(stats) - sa.commit() + sa.commit() except: log.error(traceback.format_exc()) sa.rollback() return False if len(repo.revisions) > 1: run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y) - + return True @task @@ -230,7 +230,7 @@ def reset_user_password(user_email): log = reset_user_password.get_logger() from rhodecode.lib import auth from rhodecode.model.db import User - + try: try: sa = get_session() @@ -244,26 +244,26 @@ def reset_user_password(user_email): log.info('change password for %s', user_email) if new_passwd is None: raise Exception('unable to generate new password') - + except: log.error(traceback.format_exc()) sa.rollback() - + run_task(send_email, user_email, "Your new rhodecode password", 'Your new rhodecode password:%s' % (new_passwd)) log.info('send new password mail to %s', user_email) - - + + except: log.error('Failed to update user password') log.error(traceback.format_exc()) return True -@task +@task def send_email(recipients, subject, body): log = send_email.get_logger() - email_config = dict(config.items('DEFAULT')) + email_config = dict(config.items('DEFAULT')) mail_from = email_config.get('app_email_from') user = email_config.get('smtp_username') passwd = email_config.get('smtp_password') @@ -271,11 +271,11 @@ def send_email(recipients, subject, body mail_port = email_config.get('smtp_port') tls = email_config.get('smtp_use_tls') ssl = False - + try: m = SmtpMailer(mail_from, user, passwd, mail_server, mail_port, ssl, tls) - m.send(recipients, subject, body) + m.send(recipients, subject, body) except: log.error('Mail sending failed') log.error(traceback.format_exc()) @@ -285,19 +285,19 @@ def send_email(recipients, subject, body @task def create_repo_fork(form_data, cur_user): import os - from rhodecode.model.repo_model import RepoModel + from rhodecode.model.repo import RepoModel sa = get_session() rm = RepoModel(sa) - + rm.create(form_data, cur_user, just_db=True, fork=True) - + repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '') repo_path = os.path.join(repos_path, form_data['repo_name']) repo_fork_path = os.path.join(repos_path, form_data['fork_name']) - + MercurialRepository(str(repo_fork_path), True, clone_url=str(repo_path)) - + def __get_codes_stats(repo_name): LANGUAGES_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c', 'cfg', 'cfm', 'cpp', 'cs', 'diff', 'do', 'el', 'erl', @@ -309,7 +309,7 @@ def __get_codes_stats(repo_name): repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '') repo = MercurialRepository(repos_path + repo_name) tip = repo.get_changeset() - + code_stats = {} for topnode, dirs, files in tip.walk('/'): for f in files: @@ -319,10 +319,10 @@ def __get_codes_stats(repo_name): code_stats[k] += 1 else: code_stats[k] = 1 - + return code_stats or {} - + diff --git a/rhodecode/lib/db_manage.py b/rhodecode/lib/db_manage.py --- a/rhodecode/lib/db_manage.py +++ b/rhodecode/lib/db_manage.py @@ -46,11 +46,11 @@ class DbManage(object): self.tests = tests self.root = root dburi = 'sqlite:////%s' % jn(self.root, self.dbname) - engine = create_engine(dburi, echo=log_sql) + engine = create_engine(dburi, echo=log_sql) init_model(engine) - self.sa = meta.Session + self.sa = meta.Session() self.db_exists = False - + def check_for_db(self, override): db_path = jn(self.root, self.dbname) log.info('checking for existing db in %s', db_path) @@ -77,34 +77,34 @@ class DbManage(object): checkfirst = not override meta.Base.metadata.create_all(checkfirst=checkfirst) log.info('Created tables for %s', self.dbname) - + def admin_prompt(self, second=False): if not self.tests: import getpass - - + + def get_password(): password = getpass.getpass('Specify admin password (min 6 chars):') confirm = getpass.getpass('Confirm password:') - + if password != confirm: log.error('passwords mismatch') return False if len(password) < 6: log.error('password is to short use at least 6 characters') return False - + return password - + username = raw_input('Specify admin username:') - + password = get_password() if not password: #second try password = get_password() if not password: sys.exit() - + email = raw_input('Specify admin email:') self.create_user(username, password, email, True) else: @@ -112,68 +112,68 @@ class DbManage(object): self.create_user('test_admin', 'test12', 'test_admin@mail.com', True) self.create_user('test_regular', 'test12', 'test_regular@mail.com', False) self.create_user('test_regular2', 'test12', 'test_regular2@mail.com', False) - - - + + + def config_prompt(self, test_repo_path=''): log.info('Setting up repositories config') - + if not self.tests and not test_repo_path: path = raw_input('Specify valid full path to your repositories' ' you can change this later in application settings:') else: path = test_repo_path - + if not os.path.isdir(path): log.error('You entered wrong path: %s', path) sys.exit() - + hooks1 = RhodeCodeUi() hooks1.ui_section = 'hooks' hooks1.ui_key = 'changegroup.update' hooks1.ui_value = 'hg update >&2' hooks1.ui_active = False - + hooks2 = RhodeCodeUi() hooks2.ui_section = 'hooks' hooks2.ui_key = 'changegroup.repo_size' - hooks2.ui_value = 'python:rhodecode.lib.hooks.repo_size' - + hooks2.ui_value = 'python:rhodecode.lib.hooks.repo_size' + web1 = RhodeCodeUi() web1.ui_section = 'web' web1.ui_key = 'push_ssl' web1.ui_value = 'false' - + web2 = RhodeCodeUi() web2.ui_section = 'web' web2.ui_key = 'allow_archive' web2.ui_value = 'gz zip bz2' - + web3 = RhodeCodeUi() web3.ui_section = 'web' web3.ui_key = 'allow_push' web3.ui_value = '*' - + web4 = RhodeCodeUi() web4.ui_section = 'web' web4.ui_key = 'baseurl' - web4.ui_value = '/' - + web4.ui_value = '/' + paths = RhodeCodeUi() paths.ui_section = 'paths' paths.ui_key = '/' paths.ui_value = os.path.join(path, '*') - - + + hgsettings1 = RhodeCodeSettings() - + hgsettings1.app_settings_name = 'realm' hgsettings1.app_settings_value = 'RhodeCode authentication' - + hgsettings2 = RhodeCodeSettings() hgsettings2.app_settings_name = 'title' - hgsettings2.app_settings_value = 'RhodeCode' - + hgsettings2.app_settings_value = 'RhodeCode' + try: self.sa.add(hooks1) self.sa.add(hooks2) @@ -187,9 +187,9 @@ class DbManage(object): self.sa.commit() except: self.sa.rollback() - raise + raise log.info('created ui config') - + def create_user(self, username, password, email='', admin=False): log.info('creating administrator user %s', username) new_user = User() @@ -200,7 +200,7 @@ class DbManage(object): new_user.email = email new_user.admin = admin new_user.active = True - + try: self.sa.add(new_user) self.sa.commit() @@ -225,7 +225,7 @@ class DbManage(object): except: self.sa.rollback() raise - + def create_permissions(self): #module.(access|create|change|delete)_[name] #module.(read|write|owner) @@ -240,7 +240,7 @@ class DbManage(object): ('hg.register.manual_activate', 'Register new user with rhodecode without manual activation'), ('hg.register.auto_activate', 'Register new user with rhodecode without auto activation'), ] - + for p in perms: new_perm = Permission() new_perm.permission_name = p[0] @@ -254,28 +254,28 @@ class DbManage(object): def populate_default_permissions(self): log.info('creating default user permissions') - + default_user = self.sa.query(User)\ .filter(User.username == 'default').scalar() - + reg_perm = UserToPerm() reg_perm.user = default_user reg_perm.permission = self.sa.query(Permission)\ .filter(Permission.permission_name == 'hg.register.manual_activate')\ - .scalar() - + .scalar() + create_repo_perm = UserToPerm() create_repo_perm.user = default_user create_repo_perm.permission = self.sa.query(Permission)\ .filter(Permission.permission_name == 'hg.create.repository')\ - .scalar() - + .scalar() + default_repo_perm = UserToPerm() default_repo_perm.user = default_user default_repo_perm.permission = self.sa.query(Permission)\ .filter(Permission.permission_name == 'repository.read')\ - .scalar() - + .scalar() + try: self.sa.add(reg_perm) self.sa.add(create_repo_perm) @@ -283,5 +283,5 @@ class DbManage(object): self.sa.commit() except: self.sa.rollback() - raise - + raise + diff --git a/rhodecode/lib/hooks.py b/rhodecode/lib/hooks.py --- a/rhodecode/lib/hooks.py +++ b/rhodecode/lib/hooks.py @@ -59,7 +59,7 @@ def user_action_mapper(ui, repo, hooktyp """ try: - sa = meta.Session + sa = meta.Session() username = kwargs['url'].split(':')[-1] user_log = sa.query(UserLog)\ .filter(UserLog.user == sa.query(User)\ diff --git a/rhodecode/lib/indexers/__init__.py b/rhodecode/lib/indexers/__init__.py --- a/rhodecode/lib/indexers/__init__.py +++ b/rhodecode/lib/indexers/__init__.py @@ -1,6 +1,6 @@ from os.path import dirname as dn, join as jn from rhodecode.config.environment import load_environment -from rhodecode.model.hg_model import HgModel +from rhodecode.model.hg import HgModel from shutil import rmtree from webhelpers.html.builder import escape from vcs.utils.lazy import LazyProperty diff --git a/rhodecode/lib/indexers/daemon.py b/rhodecode/lib/indexers/daemon.py --- a/rhodecode/lib/indexers/daemon.py +++ b/rhodecode/lib/indexers/daemon.py @@ -33,7 +33,7 @@ project_path = dn(dn(dn(dn(os.path.realp sys.path.append(project_path) from rhodecode.lib.pidlock import LockHeld, DaemonLock -from rhodecode.model.hg_model import HgModel +from rhodecode.model.hg import HgModel from rhodecode.lib.helpers import safe_unicode from whoosh.index import create_in, open_dir from shutil import rmtree diff --git a/rhodecode/lib/middleware/simplehg.py b/rhodecode/lib/middleware/simplehg.py --- a/rhodecode/lib/middleware/simplehg.py +++ b/rhodecode/lib/middleware/simplehg.py @@ -30,10 +30,10 @@ from mercurial.hgweb import hgweb from mercurial.hgweb.request import wsgiapplication from paste.auth.basic import AuthBasicAuthenticator from paste.httpheaders import REMOTE_USER, AUTH_TYPE -from rhodecode.lib.auth import authfunc, HasPermissionAnyMiddleware, \ - get_user_cached +from rhodecode.lib.auth import authfunc, HasPermissionAnyMiddleware from rhodecode.lib.utils import is_mercurial, make_ui, invalidate_cache, \ check_repo_fast, ui_sections +from rhodecode.model.user import UserModel from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError from rhodecode.lib.utils import action_logger import logging @@ -163,7 +163,7 @@ class SimpleHg(object): return environ.get('REMOTE_USER') def __get_user(self, username): - return get_user_cached(username) + return UserModel().get_by_username(username, cache=True) def __get_action(self, environ): """ diff --git a/rhodecode/lib/utils.py b/rhodecode/lib/utils.py --- a/rhodecode/lib/utils.py +++ b/rhodecode/lib/utils.py @@ -22,10 +22,12 @@ Created on April 18, 2010 Utilities for RhodeCode @author: marcink """ -from beaker.cache import cache_region +from rhodecode.model.caching_query import FromCache from mercurial import ui, config, hg from mercurial.error import RepoError from rhodecode.model import meta +from rhodecode.model.user import UserModel +from rhodecode.model.repo import RepoModel from rhodecode.model.db import Repository, User, RhodeCodeUi, RhodeCodeSettings, UserLog from vcs.backends.base import BaseChangeset from vcs.utils.lazy import LazyProperty @@ -67,13 +69,13 @@ def action_logger(user, action, repo, ip """ if not sa: - sa = meta.Session + sa = meta.Session() try: if hasattr(user, 'user_id'): user_id = user.user_id elif isinstance(user, basestring): - user_id = sa.query(User).filter(User.username == user).one() + user_id = UserModel(sa).get_by_username(user, cache=False).user_id else: raise Exception('You have to provide user object or username') @@ -82,8 +84,7 @@ def action_logger(user, action, repo, ip user_log.user_id = user_id user_log.action = action user_log.repository_name = repo_name - user_log.repository = sa.query(Repository)\ - .filter(Repository.repo_name == repo_name).one() + user_log.repository = RepoModel(sa).get(repo_name, cache=False) user_log.action_date = datetime.datetime.now() user_log.user_ip = ipaddr sa.add(user_log) @@ -135,11 +136,14 @@ def ask_ok(prompt, retries=4, complaint= if retries < 0: raise IOError print complaint -@cache_region('super_short_term', 'cached_hg_ui') def get_hg_ui_cached(): try: sa = meta.Session - ret = sa.query(RhodeCodeUi).all() + ret = sa.query(RhodeCodeUi)\ + .options(FromCache("sql_cache_short", "get_hg_ui_settings"))\ + .all() + except: + pass finally: meta.Session.remove() return ret @@ -147,8 +151,12 @@ def get_hg_ui_cached(): def get_hg_settings(): try: - sa = meta.Session - ret = sa.query(RhodeCodeSettings).all() + sa = meta.Session() + ret = sa.query(RhodeCodeSettings)\ + .options(FromCache("sql_cache_short", "get_hg_settings"))\ + .all() + except: + pass finally: meta.Session.remove() @@ -162,8 +170,10 @@ def get_hg_settings(): def get_hg_ui_settings(): try: - sa = meta.Session + sa = meta.Session() ret = sa.query(RhodeCodeUi).all() + except: + pass finally: meta.Session.remove() @@ -255,11 +265,11 @@ def invalidate_cache(name, *args): args = tuple(tmp) if name == 'cached_repo_list': - from rhodecode.model.hg_model import _get_repos_cached + from rhodecode.model.hg import _get_repos_cached region_invalidate(_get_repos_cached, None, *args) if name == 'full_changelog': - from rhodecode.model.hg_model import _full_changelog_cached + from rhodecode.model.hg import _full_changelog_cached region_invalidate(_full_changelog_cached, None, *args) class EmptyChangeset(BaseChangeset): @@ -296,15 +306,14 @@ def repo2db_mapper(initial_repo_list, re """ maps all found repositories into db """ - from rhodecode.model.repo_model import RepoModel - sa = meta.Session + sa = meta.Session() user = sa.query(User).filter(User.admin == True).first() rm = RepoModel() for name, repo in initial_repo_list.items(): - if not sa.query(Repository).filter(Repository.repo_name == name).scalar(): + if not RepoModel(sa).get(name, cache=False): log.info('repository %s not found creating default', name) form_data = { @@ -429,7 +438,7 @@ class OrderedDict(dict, DictMixin): #=============================================================================== -# TEST FUNCTIONS +# TEST FUNCTIONS AND CREATORS #=============================================================================== def create_test_index(repo_location, full_index): """Makes default test index diff --git a/rhodecode/model/forms.py b/rhodecode/model/forms.py --- a/rhodecode/model/forms.py +++ b/rhodecode/model/forms.py @@ -26,22 +26,21 @@ from pylons import session from pylons.i18n.translation import _ from rhodecode.lib.auth import check_password, get_crypt_password from rhodecode.model import meta -from rhodecode.model.user_model import UserModel -from rhodecode.model.db import User, Repository -from sqlalchemy.exc import OperationalError -from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound +from rhodecode.model.user import UserModel +from rhodecode.model.repo import RepoModel +from rhodecode.model.db import User from webhelpers.pylonslib.secure_form import authentication_token import formencode import logging import os import rhodecode.lib.helpers as h + log = logging.getLogger(__name__) - #this is needed to translate the messages using _() in validators class State_obj(object): _ = staticmethod(_) - + #=============================================================================== # VALIDATORS #=============================================================================== @@ -53,53 +52,51 @@ class ValidAuthToken(formencode.validato if value != authentication_token(): raise formencode.Invalid(self.message('invalid_token', state, search_number=value), value, state) - -def ValidUsername(edit, old_data): + +def ValidUsername(edit, old_data): class _ValidUsername(formencode.validators.FancyValidator): - + def validate_python(self, value, state): if value in ['default', 'new_user']: raise formencode.Invalid(_('Invalid username'), value, state) - #check if user is uniq - sa = meta.Session + #check if user is unique old_un = None if edit: - old_un = sa.query(User).get(old_data.get('user_id')).username - - if old_un != value or not edit: - if sa.query(User).filter(User.username == value).scalar(): + old_un = UserModel().get(old_data.get('user_id')).username + + if old_un != value or not edit: + if UserModel().get_by_username(value, cache=False): raise formencode.Invalid(_('This username already exists') , value, state) - meta.Session.remove() - - return _ValidUsername - + + return _ValidUsername + class ValidPassword(formencode.validators.FancyValidator): - + def to_python(self, value, state): if value: return get_crypt_password(value) - + class ValidAuth(formencode.validators.FancyValidator): messages = { 'invalid_password':_('invalid password'), 'invalid_login':_('invalid user name'), 'disabled_account':_('Your acccount is disabled') - + } #error mapping e_dict = {'username':messages['invalid_login'], 'password':messages['invalid_password']} e_dict_disable = {'username':messages['disabled_account']} - + def validate_python(self, value, state): password = value['password'] username = value['username'] - user = UserModel().get_user_by_name(username) + user = UserModel().get_by_username(username) if user is None: raise formencode.Invalid(self.message('invalid_password', state=State_obj), value, state, - error_dict=self.e_dict) + error_dict=self.e_dict) if user: if user.active: if user.username == username and check_password(password, @@ -116,12 +113,13 @@ class ValidAuth(formencode.validators.Fa state=State_obj), value, state, error_dict=self.e_dict_disable) - + class ValidRepoUser(formencode.validators.FancyValidator): - + def to_python(self, value, state): + sa = meta.Session() try: - self.user_db = meta.Session.query(User)\ + self.user_db = sa.query(User)\ .filter(User.active == True)\ .filter(User.username == value).one() except Exception: @@ -129,31 +127,29 @@ class ValidRepoUser(formencode.validator value, state) finally: meta.Session.remove() - + return self.user_db.user_id -def ValidRepoName(edit, old_data): +def ValidRepoName(edit, old_data): class _ValidRepoName(formencode.validators.FancyValidator): - + def to_python(self, value, state): slug = h.repo_name_slug(value) if slug in ['_admin']: raise formencode.Invalid(_('This repository name is disallowed'), value, state) - if old_data.get('repo_name') != value or not edit: - sa = meta.Session - if sa.query(Repository).filter(Repository.repo_name == slug).scalar(): + if old_data.get('repo_name') != value or not edit: + if RepoModel().get(slug, cache=False): raise formencode.Invalid(_('This repository already exists') , value, state) - meta.Session.remove() - return slug - - + return slug + + return _ValidRepoName class ValidPerms(formencode.validators.FancyValidator): messages = {'perm_new_user_name':_('This username is not valid')} - + def to_python(self, value, state): perms_update = [] perms_new = [] @@ -167,7 +163,7 @@ class ValidPerms(formencode.validators.F if (new_user, new_perm) not in perms_new: perms_new.append((new_user, new_perm)) else: - usr = k[5:] + usr = k[5:] if usr == 'default': if value['private']: #set none for default when updating to private repo @@ -184,36 +180,36 @@ class ValidPerms(formencode.validators.F except Exception: msg = self.message('perm_new_user_name', state=State_obj) - raise formencode.Invalid(msg, value, state, error_dict={'perm_new_user_name':msg}) + raise formencode.Invalid(msg, value, state, error_dict={'perm_new_user_name':msg}) return value - + class ValidSettings(formencode.validators.FancyValidator): - + def to_python(self, value, state): #settings form can't edit user if value.has_key('user'): del['value']['user'] - + return value - + class ValidPath(formencode.validators.FancyValidator): def to_python(self, value, state): isdir = os.path.isdir(value.replace('*', '')) if (value.endswith('/*') or value.endswith('/**')) and isdir: return value elif not isdir: - msg = _('This is not a valid path') + msg = _('This is not a valid path') else: msg = _('You need to specify * or ** at the end of path (ie. /tmp/*)') - + raise formencode.Invalid(msg, value, state, - error_dict={'paths_root_path':msg}) + error_dict={'paths_root_path':msg}) def UniqSystemEmail(old_data): class _UniqSystemEmail(formencode.validators.FancyValidator): def to_python(self, value, state): if old_data.get('email') != value: - sa = meta.Session + sa = meta.Session() try: user = sa.query(User).filter(User.email == value).scalar() if user: @@ -221,11 +217,11 @@ def UniqSystemEmail(old_data): value, state) finally: meta.Session.remove() - + return value - + return _UniqSystemEmail - + class ValidSystemEmail(formencode.validators.FancyValidator): def to_python(self, value, state): sa = meta.Session @@ -236,8 +232,8 @@ class ValidSystemEmail(formencode.valida value, state) finally: meta.Session.remove() - - return value + + return value #=============================================================================== # FORMS @@ -266,7 +262,7 @@ class LoginForm(formencode.Schema): #chained validators have access to all data chained_validators = [ValidAuth] - + def UserForm(edit=False, old_data={}): class _UserForm(formencode.Schema): allow_extra_fields = True @@ -281,7 +277,7 @@ def UserForm(edit=False, old_data={}): name = UnicodeString(strip=True, min=1, not_empty=True) lastname = UnicodeString(strip=True, min=1, not_empty=True) email = All(Email(not_empty=True), UniqSystemEmail(old_data)) - + return _UserForm RegisterForm = UserForm @@ -290,7 +286,7 @@ def PasswordResetForm(): class _PasswordResetForm(formencode.Schema): allow_extra_fields = True filter_extra_fields = True - email = All(ValidSystemEmail(), Email(not_empty=True)) + email = All(ValidSystemEmail(), Email(not_empty=True)) return _PasswordResetForm def RepoForm(edit=False, old_data={}): @@ -300,10 +296,10 @@ def RepoForm(edit=False, old_data={}): repo_name = All(UnicodeString(strip=True, min=1, not_empty=True), ValidRepoName(edit, old_data)) description = UnicodeString(strip=True, min=1, not_empty=True) private = StringBoolean(if_missing=False) - + if edit: user = All(Int(not_empty=True), ValidRepoUser) - + chained_validators = [ValidPerms] return _RepoForm @@ -314,7 +310,7 @@ def RepoForkForm(edit=False, old_data={} fork_name = All(UnicodeString(strip=True, min=1, not_empty=True), ValidRepoName(edit, old_data)) description = UnicodeString(strip=True, min=1, not_empty=True) private = StringBoolean(if_missing=False) - + return _RepoForkForm def RepoSettingsForm(edit=False, old_data={}): @@ -324,7 +320,7 @@ def RepoSettingsForm(edit=False, old_dat repo_name = All(UnicodeString(strip=True, min=1, not_empty=True), ValidRepoName(edit, old_data)) description = UnicodeString(strip=True, min=1, not_empty=True) private = StringBoolean(if_missing=False) - + chained_validators = [ValidPerms, ValidSettings] return _RepoForm @@ -335,9 +331,9 @@ def ApplicationSettingsForm(): filter_extra_fields = False rhodecode_title = UnicodeString(strip=True, min=1, not_empty=True) rhodecode_realm = UnicodeString(strip=True, min=1, not_empty=True) - + return _ApplicationSettingsForm - + def ApplicationUiSettingsForm(): class _ApplicationUiSettingsForm(formencode.Schema): allow_extra_fields = True @@ -346,7 +342,7 @@ def ApplicationUiSettingsForm(): paths_root_path = All(ValidPath(), UnicodeString(strip=True, min=1, not_empty=True)) hooks_changegroup_update = OneOf(['True', 'False'], if_missing=False) hooks_changegroup_repo_size = OneOf(['True', 'False'], if_missing=False) - + return _ApplicationUiSettingsForm def DefaultPermissionsForm(perms_choices, register_choices, create_choices): @@ -357,5 +353,5 @@ def DefaultPermissionsForm(perms_choices default_perm = OneOf(perms_choices) default_register = OneOf(register_choices) default_create = OneOf(create_choices) - + return _DefaultPermissionsForm diff --git a/rhodecode/model/hg_model.py b/rhodecode/model/hg.py rename from rhodecode/model/hg_model.py rename to rhodecode/model/hg.py --- a/rhodecode/model/hg_model.py +++ b/rhodecode/model/hg.py @@ -25,7 +25,6 @@ Model for RhodeCode from beaker.cache import cache_region from mercurial import ui from mercurial.hgweb.hgwebdir_mod import findrepos -from pylons.i18n.translation import _ from rhodecode.lib import helpers as h from rhodecode.lib.utils import invalidate_cache from rhodecode.lib.auth import HasRepoPermissionAny @@ -65,7 +64,7 @@ def _get_repos_switcher_cached(cached_re if HasRepoPermissionAny('repository.write', 'repository.read', 'repository.admin')(repo.name, 'main page check'): repos_lst.append((repo.name, repo.dbrepo.private,)) - + return sorted(repos_lst, key=lambda k:k[0].lower()) @cache_region('long_term', 'full_changelog') @@ -79,7 +78,7 @@ class HgModel(object): def __init__(self): pass - + @staticmethod def repo_scan(repos_prefix, repos_path, baseui, initial=False): """ @@ -99,17 +98,17 @@ class HgModel(object): if repos_path[0] != '/': repos_path[0] = '/' if not os.path.isdir(os.path.join(*repos_path)): - raise RepositoryError('Not a valid repository in %s' % path) + raise RepositoryError('Not a valid repository in %s' % path) if not repos_path.endswith('*'): raise VCSError('You need to specify * or ** at the end of path ' 'for recursive scanning') - + check_repo_dir(repos_path) log.info('scanning for repositories in %s', repos_path) repos = findrepos([(repos_prefix, repos_path)]) if not isinstance(baseui, ui.ui): baseui = ui.ui() - + repos_list = {} for name, path in repos: try: @@ -118,10 +117,10 @@ class HgModel(object): raise RepositoryError('Duplicate repository name %s found in' ' %s' % (name, path)) else: - + repos_list[name] = MercurialRepository(path, baseui=baseui) repos_list[name].name = name - + dbrepo = None if not initial: #for initial scann on application first run we don't @@ -130,7 +129,7 @@ class HgModel(object): .options(joinedload(Repository.fork))\ .filter(Repository.repo_name == name)\ .scalar() - + if dbrepo: log.info('Adding db instance to cached list') repos_list[name].dbrepo = dbrepo @@ -144,16 +143,16 @@ class HgModel(object): continue meta.Session.remove() return repos_list - + def get_repos(self): for name, repo in _get_repos_cached().items(): if repo._get_hidden(): #skip hidden web repository continue - + last_change = repo.last_change tip = h.get_changeset_safe(repo, 'tip') - + tmp_d = {} tmp_d['name'] = repo.name tmp_d['name_sort'] = tmp_d['name'].lower() @@ -162,7 +161,7 @@ class HgModel(object): tmp_d['last_change'] = last_change tmp_d['last_change_sort'] = last_change[1] - last_change[0] tmp_d['tip'] = tip.short_id - tmp_d['tip_sort'] = tip.revision + tmp_d['tip_sort'] = tip.revision tmp_d['rev'] = tip.revision tmp_d['contact'] = repo.contact tmp_d['contact_sort'] = tmp_d['contact'] @@ -181,6 +180,6 @@ class HgModel(object): invalidate_cache('cached_repo_list') repo = _get_repos_cached()[repo_name] return repo - - - + + + diff --git a/rhodecode/model/permission_model.py b/rhodecode/model/permission_model.py --- a/rhodecode/model/permission_model.py +++ b/rhodecode/model/permission_model.py @@ -2,7 +2,7 @@ # encoding: utf-8 # Model for permissions # Copyright (C) 2009-2010 Marcin Kuzminski - + # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; version 2 @@ -23,8 +23,8 @@ Model for permissions @author: marcink """ -from pylons.i18n.translation import _ from rhodecode.model.db import User, Permission, UserToPerm, RepoToPerm +from rhodecode.model.caching_query import FromCache from rhodecode.model.meta import Session import logging import traceback @@ -33,40 +33,47 @@ log = logging.getLogger(__name__) class PermissionModel(object): - def __init__(self): - self.sa = Session() - - def get_default(self): - return self.sa.query(User).filter(User.username == 'default').scalar() - - def get_permission(self, id): - return self.sa.query(Permission).get(id) - - def get_permission_by_name(self, name): - return self.sa.query(Permission)\ - .filter(Permission.permission_name == name).scalar() - - + def __init__(self, sa=None): + if not sa: + self.sa = Session() + else: + self.sa = sa + + def get_permission(self, permission_id, cache=False): + perm = self.sa.query(Permission) + if cache: + perm = perm.options(FromCache("sql_cache_short", + "get_permission_%s" % permission_id)) + return perm.get(permission_id) + + def get_permission_by_name(self, name, cache=False): + perm = self.sa.query(Permission)\ + .filter(Permission.permission_name == name) + if cache: + perm = perm.options(FromCache("sql_cache_short", + "get_permission_%s" % name)) + return perm.scalar() + def update(self, form_result): perm_user = self.sa.query(User)\ .filter(User.username == form_result['perm_user_name']).scalar() u2p = self.sa.query(UserToPerm).filter(UserToPerm.user == perm_user).all() if len(u2p) != 3: - raise Exception('There is more than 3 defined \ - permissions for defualt user. This should not happen please verify\ - your database') - + raise Exception('There is more than 3 defined' + ' permissions for default user. This should not happen please verify' + ' your database') + try: #stage 1 change defaults for p in u2p: if p.permission.permission_name.startswith('repository.'): p.permission = self.get_permission_by_name(form_result['default_perm']) self.sa.add(p) - + if p.permission.permission_name.startswith('hg.register.'): p.permission = self.get_permission_by_name(form_result['default_register']) self.sa.add(p) - + if p.permission.permission_name.startswith('hg.create.'): p.permission = self.get_permission_by_name(form_result['default_create']) self.sa.add(p) @@ -75,14 +82,14 @@ class PermissionModel(object): for r2p in self.sa.query(RepoToPerm).filter(RepoToPerm.user == perm_user).all(): r2p.permission = self.get_permission_by_name(form_result['default_perm']) self.sa.add(r2p) - + self.sa.commit() except: log.error(traceback.format_exc()) self.sa.rollback() - raise - - - - - + raise + + + + + diff --git a/rhodecode/model/repo_model.py b/rhodecode/model/repo.py rename from rhodecode/model/repo_model.py rename to rhodecode/model/repo.py --- a/rhodecode/model/repo_model.py +++ b/rhodecode/model/repo.py @@ -19,15 +19,15 @@ """ Created on Jun 5, 2010 model for handling repositories actions -@author: marcink +:author: marcink """ + from datetime import datetime from pylons import app_globals as g -from rhodecode.lib.utils import check_repo from rhodecode.model.db import Repository, RepoToPerm, User, Permission from rhodecode.model.meta import Session -from rhodecode.model.user_model import UserModel -from rhodecode.lib.celerylib.tasks import create_repo_fork, run_task +from rhodecode.model.user import UserModel +from rhodecode.model.caching_query import FromCache import logging import os import shutil @@ -35,82 +35,85 @@ import traceback log = logging.getLogger(__name__) class RepoModel(object): - + def __init__(self, sa=None): if not sa: self.sa = Session() else: self.sa = sa - - def get(self, id): - return self.sa.query(Repository)\ - .filter(Repository.repo_name == id).scalar() - + + def get(self, repo_id, cache=False): + repo = self.sa.query(Repository)\ + .filter(Repository.repo_name == repo_id) + + if cache: + repo = repo.options(FromCache("sql_cache_short", + "get_repo_%s" % repo)) + return repo.scalar() + def get_users_js(self): - + users = self.sa.query(User).filter(User.active == True).all() u_tmpl = '''{id:%s, fname:"%s", lname:"%s", nname:"%s"},''' users_array = '[%s];' % '\n'.join([u_tmpl % (u.user_id, u.name, - u.lastname, u.username) + u.lastname, u.username) for u in users]) - return users_array - - + return users_array + + def update(self, repo_name, form_data): try: #update permissions for username, perm in form_data['perms_updates']: r2p = self.sa.query(RepoToPerm)\ - .filter(RepoToPerm.user == self.sa.query(User)\ - .filter(User.username == username).one())\ + .filter(RepoToPerm.user == UserModel(self.sa).get_by_username(username, cache=False))\ .filter(RepoToPerm.repository == self.get(repo_name))\ .one() - + r2p.permission_id = self.sa.query(Permission).filter( - Permission.permission_name == + Permission.permission_name == perm).one().permission_id self.sa.add(r2p) - + #set new permissions for username, perm in form_data['perms_new']: r2p = RepoToPerm() r2p.repository = self.get(repo_name) - r2p.user = self.sa.query(User)\ - .filter(User.username == username).one() - + r2p.user = UserModel(self.sa).get_by_username(username, cache=False) + r2p.permission_id = self.sa.query(Permission).filter( Permission.permission_name == perm)\ .one().permission_id self.sa.add(r2p) - + #update current repo - cur_repo = self.get(repo_name) - + cur_repo = self.get(repo_name, cache=False) + for k, v in form_data.items(): if k == 'user': cur_repo.user_id = v else: setattr(cur_repo, k, v) - + self.sa.add(cur_repo) - + if repo_name != form_data['repo_name']: #rename our data - self.__rename_repo(repo_name, form_data['repo_name']) - + self.__rename_repo(repo_name, form_data['repo_name']) + self.sa.commit() except: log.error(traceback.format_exc()) self.sa.rollback() - raise - + raise + def create(self, form_data, cur_user, just_db=False, fork=False): try: if fork: repo_name = str(form_data['fork_name']) org_name = str(form_data['repo_name']) - + else: org_name = repo_name = str(form_data['repo_name']) new_repo = Repository() @@ -118,33 +121,32 @@ class RepoModel(object): if k == 'repo_name': v = repo_name setattr(new_repo, k, v) - + if fork: parent_repo = self.sa.query(Repository)\ .filter(Repository.repo_name == org_name).scalar() new_repo.fork = parent_repo - + new_repo.user_id = cur_user.user_id self.sa.add(new_repo) - + #create default permission repo_to_perm = RepoToPerm() default = 'repository.read' - for p in UserModel(self.sa).get_default().user_perms: + for p in UserModel(self.sa).get_by_username('default', cache=False).user_perms: if p.permission.permission_name.startswith('repository.'): default = p.permission.permission_name break - + default_perm = 'repository.none' if form_data['private'] else default - + repo_to_perm.permission_id = self.sa.query(Permission)\ .filter(Permission.permission_name == default_perm)\ .one().permission_id - + repo_to_perm.repository_id = new_repo.repo_id - repo_to_perm.user_id = self.sa.query(User)\ - .filter(User.username == 'default').one().user_id - + repo_to_perm.user_id = UserModel(self.sa).get_by_username('default', cache=False).user_id + self.sa.add(repo_to_perm) self.sa.commit() if not just_db: @@ -152,11 +154,12 @@ class RepoModel(object): except: log.error(traceback.format_exc()) self.sa.rollback() - raise - + raise + def create_fork(self, form_data, cur_user): - run_task(create_repo_fork, form_data, cur_user) - + from rhodecode.lib.celerylib import tasks, run_task + run_task(tasks.create_repo_fork, form_data, cur_user) + def delete(self, repo): try: self.sa.delete(repo) @@ -166,7 +169,7 @@ class RepoModel(object): log.error(traceback.format_exc()) self.sa.rollback() raise - + def delete_perm_user(self, form_data, repo_name): try: self.sa.query(RepoToPerm)\ @@ -177,8 +180,9 @@ class RepoModel(object): log.error(traceback.format_exc()) self.sa.rollback() raise - - def __create_repo(self, repo_name): + + def __create_repo(self, repo_name): + from rhodecode.lib.utils import check_repo repo_path = os.path.join(g.base_path, repo_name) if check_repo(repo_name, g.base_path): log.info('creating repo %s in %s', repo_name, repo_path) @@ -187,14 +191,14 @@ class RepoModel(object): def __rename_repo(self, old, new): log.info('renaming repo from %s to %s', old, new) - + old_path = os.path.join(g.base_path, old) new_path = os.path.join(g.base_path, new) if os.path.isdir(new_path): raise Exception('Was trying to rename to already existing dir %s', - new_path) + new_path) shutil.move(old_path, new_path) - + def __delete_repo(self, name): rm_path = os.path.join(g.base_path, name) log.info("Removing %s", rm_path) diff --git a/rhodecode/model/user_model.py b/rhodecode/model/user.py rename from rhodecode/model/user_model.py rename to rhodecode/model/user.py --- a/rhodecode/model/user_model.py +++ b/rhodecode/model/user.py @@ -17,19 +17,19 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, # MA 02110-1301, USA. - """ Created on April 9, 2010 Model for users -@author: marcink +:author: marcink """ -from rhodecode.lib import auth + from pylons.i18n.translation import _ -from rhodecode.lib.celerylib import tasks, run_task +from rhodecode.model.caching_query import FromCache from rhodecode.model.db import User from rhodecode.model.meta import Session +import logging import traceback -import logging + log = logging.getLogger(__name__) class DefaultUserException(Exception):pass @@ -41,69 +41,76 @@ class UserModel(object): self.sa = Session() else: self.sa = sa - - def get_default(self): - return self.sa.query(User).filter(User.username == 'default').scalar() - - def get_user(self, id): - return self.sa.query(User).get(id) - - def get_user_by_name(self, name): - return self.sa.query(User).filter(User.username == name).scalar() - + + def get(self, user_id, cache=False): + user = self.sa.query(User) + if cache: + user = user.options(FromCache("sql_cache_short", + "get_user_%s" % user_id)) + return user.get(user_id) + + + def get_by_username(self, username, cache=False): + user = self.sa.query(User)\ + .filter(User.username == username) + if cache: + user = user.options(FromCache("sql_cache_short", + "get_user_%s" % username)) + return user.scalar() + def create(self, form_data): try: new_user = User() for k, v in form_data.items(): setattr(new_user, k, v) - + self.sa.add(new_user) self.sa.commit() except: log.error(traceback.format_exc()) self.sa.rollback() - raise - + raise + def create_registration(self, form_data): try: new_user = User() for k, v in form_data.items(): if k != 'admin': setattr(new_user, k, v) - + self.sa.add(new_user) self.sa.commit() except: log.error(traceback.format_exc()) self.sa.rollback() - raise - - def update(self, uid, form_data): + raise + + def update(self, user_id, form_data): try: - new_user = self.sa.query(User).get(uid) + new_user = self.get(user_id, cache=False) if new_user.username == 'default': raise DefaultUserException( - _("You can't Edit this user since it's" + _("You can't Edit this user since it's" " crucial for entire application")) for k, v in form_data.items(): if k == 'new_password' and v != '': new_user.password = v else: setattr(new_user, k, v) - + self.sa.add(new_user) self.sa.commit() except: log.error(traceback.format_exc()) self.sa.rollback() - raise - - def update_my_account(self, uid, form_data): + raise + + def update_my_account(self, user_id, form_data): try: - new_user = self.sa.query(User).get(uid) + new_user = self.get(user_id, cache=False) if new_user.username == 'default': raise DefaultUserException( - _("You can't Edit this user since it's" + _("You can't Edit this user since it's" " crucial for entire application")) for k, v in form_data.items(): if k == 'new_password' and v != '': @@ -111,28 +118,28 @@ class UserModel(object): else: if k not in ['admin', 'active']: setattr(new_user, k, v) - + self.sa.add(new_user) self.sa.commit() except: log.error(traceback.format_exc()) self.sa.rollback() - raise - - def delete(self, id): + raise + + def delete(self, user_id): try: - - user = self.sa.query(User).get(id) + user = self.get(user_id, cache=False) if user.username == 'default': raise DefaultUserException( - _("You can't remove this user since it's" + _("You can't remove this user since it's" " crucial for entire application")) self.sa.delete(user) - self.sa.commit() + self.sa.commit() except: log.error(traceback.format_exc()) self.sa.rollback() - raise + raise def reset_password(self, data): + from rhodecode.lib.celerylib import tasks, run_task run_task(tasks.reset_user_password, data['email']) diff --git a/rhodecode/public/css/style.css b/rhodecode/public/css/style.css --- a/rhodecode/public/css/style.css +++ b/rhodecode/public/css/style.css @@ -2301,6 +2301,8 @@ padding-left:5px; #changeset_content .container .left .date,.ac .match { font-weight:700; +padding-top: 5px; +padding-bottom:5px; } div#legend_container table td,div#legend_choices table td { diff --git a/test.ini b/test.ini --- a/test.ini +++ b/test.ini @@ -49,19 +49,32 @@ cache_dir = %(here)s/data #################################### beaker.cache.data_dir=/%(here)s/data/cache/data beaker.cache.lock_dir=/%(here)s/data/cache/lock -beaker.cache.regions=super_short_term,short_term,long_term +beaker.cache.regions=super_short_term,short_term,long_term,sql_cache_short,sql_cache_med,sql_cache_long + +beaker.cache.super_short_term.type=memory +beaker.cache.super_short_term.expire=10 + +beaker.cache.short_term.type=memory +beaker.cache.short_term.expire=60 + beaker.cache.long_term.type=memory beaker.cache.long_term.expire=36000 -beaker.cache.short_term.type=memory -beaker.cache.short_term.expire=60 -beaker.cache.super_short_term.type=memory -beaker.cache.super_short_term.expire=10 + + +beaker.cache.sql_cache_short.type=memory +beaker.cache.sql_cache_short.expire=5 + +beaker.cache.sql_cache_med.type=memory +beaker.cache.sql_cache_med.expire=360 + +beaker.cache.sql_cache_long.type=file +beaker.cache.sql_cache_long.expire=3600 #################################### ### BEAKER SESSION #### #################################### ## Type of storage used for the session, current types are -## "dbm", "file", "memcached", "database", and "memory". +## dbm, file, memcached, database, and memory. ## The storage uses the Container API ##that is also used by the cache system. beaker.session.type = file