diff --git a/rhodecode/config/environment.py b/rhodecode/config/environment.py --- a/rhodecode/config/environment.py +++ b/rhodecode/config/environment.py @@ -6,7 +6,7 @@ from rhodecode.config.routing import mak from rhodecode.lib.auth import set_available_permissions, set_base_path from rhodecode.lib.utils import repo2db_mapper, make_ui, set_rhodecode_config from rhodecode.model import init_model -from rhodecode.model.hg import _get_repos_cached_initial +from rhodecode.model.hg import HgModel from sqlalchemy import engine_from_config import logging import os @@ -69,7 +69,8 @@ def load_environment(global_conf, app_co #init baseui config['pylons.app_globals'].baseui = make_ui('db') - repo2db_mapper(_get_repos_cached_initial(config['pylons.app_globals'], initial)) + g = config['pylons.app_globals'] + repo2db_mapper(HgModel().repo_scan(g.paths[0][1], g.baseui, initial)) set_available_permissions(config) set_base_path(config) set_rhodecode_config(config) diff --git a/rhodecode/controllers/admin/repos.py b/rhodecode/controllers/admin/repos.py --- a/rhodecode/controllers/admin/repos.py +++ b/rhodecode/controllers/admin/repos.py @@ -74,7 +74,6 @@ class ReposController(BaseController): try: form_result = _form.to_python(dict(request.POST)) repo_model.create(form_result, c.rhodecode_user) - invalidate_cache('cached_repo_list') h.flash(_('created repository %s') % form_result['repo_name'], category='success') @@ -133,7 +132,7 @@ class ReposController(BaseController): try: form_result = _form.to_python(dict(request.POST)) repo_model.update(repo_name, form_result) - invalidate_cache('cached_repo_list') + invalidate_cache('get_repo_cached_%s' % repo_name) h.flash(_('Repository %s updated succesfully' % repo_name), category='success') changed_name = form_result['repo_name'] @@ -182,7 +181,7 @@ class ReposController(BaseController): action_logger(self.rhodecode_user, 'admin_deleted_repo', repo_name, '', self.sa) repo_model.delete(repo) - invalidate_cache('cached_repo_list') + invalidate_cache('get_repo_cached_%s' % repo_name) h.flash(_('deleted repository %s') % repo_name, category='success') except Exception, e: diff --git a/rhodecode/controllers/admin/settings.py b/rhodecode/controllers/admin/settings.py --- a/rhodecode/controllers/admin/settings.py +++ b/rhodecode/controllers/admin/settings.py @@ -33,12 +33,13 @@ from rhodecode.lib.auth import LoginRequ from rhodecode.lib.base import BaseController, render from rhodecode.lib.utils import repo2db_mapper, invalidate_cache, \ set_rhodecode_config, get_hg_settings, get_hg_ui_settings -from rhodecode.model.db import RhodeCodeSettings, RhodeCodeUi +from rhodecode.model.db import RhodeCodeSettings, RhodeCodeUi, Repository from rhodecode.model.forms import UserForm, ApplicationSettingsForm, \ ApplicationUiSettingsForm from rhodecode.model.hg import HgModel from rhodecode.model.user import UserModel from rhodecode.lib.celerylib import tasks, run_task +from sqlalchemy import func import formencode import logging import traceback @@ -98,9 +99,12 @@ class SettingsController(BaseController) rm_obsolete = request.POST.get('destroy', False) log.debug('Rescanning directories with destroy=%s', rm_obsolete) - initial = HgModel.repo_scan(g.paths[0][0], g.paths[0][1], g.baseui) + initial = HgModel().repo_scan(g.paths[0][1], g.baseui) + for repo_name in initial.keys(): + invalidate_cache('get_repo_cached_%s' % repo_name) + repo2db_mapper(initial, rm_obsolete) - invalidate_cache('cached_repo_list') + h.flash(_('Repositories successfully rescanned'), category='success') if setting_id == 'whoosh': @@ -238,12 +242,14 @@ class SettingsController(BaseController) """ GET /_admin/my_account Displays info about my account """ + # url('admin_settings_my_account') c.user = UserModel(self.sa).get(c.rhodecode_user.user_id, cache=False) - c.user_repos = [] - for repo in c.cached_repo_list.values(): - if repo.dbrepo.user.username == c.user.username: - c.user_repos.append(repo) + all_repos = self.sa.query(Repository)\ + .filter(Repository.user_id == c.user.user_id)\ + .order_by(func.lower(Repository.repo_name))\ + .all() + c.user_repos = HgModel().get_repos(all_repos) if c.user.username == 'default': h.flash(_("You can't edit this user since it's" diff --git a/rhodecode/controllers/settings.py b/rhodecode/controllers/settings.py --- a/rhodecode/controllers/settings.py +++ b/rhodecode/controllers/settings.py @@ -78,7 +78,7 @@ class SettingsController(BaseController) try: form_result = _form.to_python(dict(request.POST)) repo_model.update(repo_name, form_result) - invalidate_cache('cached_repo_list') + invalidate_cache('get_repo_cached_%s' % repo_name) h.flash(_('Repository %s updated successfully' % repo_name), category='success') changed_name = form_result['repo_name'] @@ -96,7 +96,7 @@ class SettingsController(BaseController) encoding="UTF-8") except Exception: log.error(traceback.format_exc()) - h.flash(_('error occured during update of repository %s') \ + h.flash(_('error occurred during update of repository %s') \ % repo_name, category='error') return redirect(url('repo_settings_home', repo_name=changed_name)) @@ -126,7 +126,7 @@ class SettingsController(BaseController) action_logger(self.rhodecode_user, 'user_deleted_repo', repo_name, '', self.sa) repo_model.delete(repo) - invalidate_cache('cached_repo_list') + invalidate_cache('get_repo_cached_%s' % repo_name) h.flash(_('deleted repository %s') % repo_name, category='success') except Exception: h.flash(_('An error occurred during deletion of %s') % repo_name, diff --git a/rhodecode/lib/base.py b/rhodecode/lib/base.py --- a/rhodecode/lib/base.py +++ b/rhodecode/lib/base.py @@ -9,20 +9,20 @@ from rhodecode import __version__ from rhodecode.lib import auth from rhodecode.lib.utils import get_repo_slug from rhodecode.model import meta -from rhodecode.model.hg import _get_repos_cached, \ - _get_repos_switcher_cached +from rhodecode.model.hg import HgModel from vcs import BACKENDS + class BaseController(WSGIController): def __before__(self): c.rhodecode_version = __version__ c.rhodecode_name = config['rhodecode_title'] c.repo_name = get_repo_slug(request) - c.cached_repo_list = _get_repos_cached() - c.repo_switcher_list = _get_repos_switcher_cached(c.cached_repo_list) + c.cached_repo_list = HgModel().get_repos() c.backends = BACKENDS.keys() + if c.repo_name: - cached_repo = c.cached_repo_list.get(c.repo_name) + cached_repo = HgModel().get(c.repo_name) if cached_repo: c.repository_tags = cached_repo.tags diff --git a/rhodecode/lib/middleware/simplegit.py b/rhodecode/lib/middleware/simplegit.py --- a/rhodecode/lib/middleware/simplegit.py +++ b/rhodecode/lib/middleware/simplegit.py @@ -83,15 +83,15 @@ class SimpleGit(object): self.repository = None self.username = None self.action = None - + def __call__(self, environ, start_response): if not is_git(environ): return self.application(environ, start_response) - + proxy_key = 'HTTP_X_REAL_IP' def_key = 'REMOTE_ADDR' self.ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0')) - + #=================================================================== # AUTHENTICATE THIS GIT REQUEST #=================================================================== @@ -104,7 +104,7 @@ class SimpleGit(object): REMOTE_USER.update(environ, result) else: return result.wsgi_application(environ, start_response) - + #======================================================================= # GET REPOSITORY #======================================================================= @@ -206,5 +206,4 @@ class SimpleGit(object): """we know that some change was made to repositories and we should invalidate the cache to see the changes right away but only for push requests""" - invalidate_cache('cached_repo_list') - invalidate_cache('full_changelog', repo_name) + invalidate_cache('get_repo_cached_%s' % repo_name) diff --git a/rhodecode/lib/middleware/simplehg.py b/rhodecode/lib/middleware/simplehg.py --- a/rhodecode/lib/middleware/simplehg.py +++ b/rhodecode/lib/middleware/simplehg.py @@ -52,20 +52,20 @@ class SimpleHg(object): self.repository = None self.username = None self.action = None - + def __call__(self, environ, start_response): if not is_mercurial(environ): return self.application(environ, start_response) - + proxy_key = 'HTTP_X_REAL_IP' def_key = 'REMOTE_ADDR' self.ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0')) - + #=================================================================== # AUTHENTICATE THIS MERCURIAL REQUEST #=================================================================== username = REMOTE_USER(environ) - + if not username: self.authenticate.realm = self.config['rhodecode_realm'] result = self.authenticate(environ) @@ -74,7 +74,7 @@ class SimpleHg(object): REMOTE_USER.update(environ, result) else: return result.wsgi_application(environ, start_response) - + #======================================================================= # GET REPOSITORY #======================================================================= @@ -114,7 +114,7 @@ class SimpleHg(object): 'repository.admin')\ (user, repo_name): return HTTPForbidden()(environ, start_response) - + self.extras = {'ip':self.ipaddr, 'username':self.username, 'action':self.action, @@ -200,8 +200,7 @@ class SimpleHg(object): """we know that some change was made to repositories and we should invalidate the cache to see the changes right away but only for push requests""" - invalidate_cache('cached_repo_list') - invalidate_cache('full_changelog', repo_name) + invalidate_cache('get_repo_cached_%s' % repo_name) def __load_web_settings(self, hgserve, extras={}): @@ -209,12 +208,12 @@ class SimpleHg(object): hgserve.repo.ui = self.baseui hgrc = os.path.join(self.repo_path, '.hg', 'hgrc') - + #inject some additional parameters that will be available in ui #for hooks for k, v in extras.items(): hgserve.repo.ui.setconfig('rhodecode_extras', k, v) - + repoui = make_ui('file', hgrc, False) if repoui: @@ -222,7 +221,7 @@ class SimpleHg(object): for section in ui_sections: for k, v in repoui.configitems(section): hgserve.repo.ui.setconfig(section, k, v) - + return hgserve diff --git a/rhodecode/lib/utils.py b/rhodecode/lib/utils.py --- a/rhodecode/lib/utils.py +++ b/rhodecode/lib/utils.py @@ -275,25 +275,11 @@ def set_rhodecode_config(config): config[k] = v def invalidate_cache(name, *args): - """Invalidates given name cache""" - - from beaker.cache import region_invalidate - log.info('INVALIDATING CACHE FOR %s', name) - - """propagate our arguments to make sure invalidation works. First - argument has to be the name of cached func name give to cache decorator - without that the invalidation would not work""" - tmp = [name] - tmp.extend(args) - args = tuple(tmp) - - if name == 'cached_repo_list': - from rhodecode.model.hg import _get_repos_cached - region_invalidate(_get_repos_cached, None, *args) - - if name == 'full_changelog': - from rhodecode.model.hg import _full_changelog_cached - region_invalidate(_full_changelog_cached, None, *args) + """ + Puts cache invalidation task into db for + further global cache invalidation + """ + pass class EmptyChangeset(BaseChangeset): """ @@ -352,7 +338,6 @@ def repo2db_mapper(initial_repo_list, re } rm.create(form_data, user, just_db=True) - if remove_obsolete: #remove from database those repositories that are not in the filesystem for repo in sa.query(Repository).all(): @@ -360,10 +345,6 @@ def repo2db_mapper(initial_repo_list, re sa.delete(repo) sa.commit() - - meta.Session.remove() - - class OrderedDict(dict, DictMixin): def __init__(self, *args, **kwds): diff --git a/rhodecode/model/hg.py b/rhodecode/model/hg.py --- a/rhodecode/model/hg.py +++ b/rhodecode/model/hg.py @@ -22,57 +22,25 @@ Created on April 9, 2010 Model for RhodeCode @author: marcink """ -from beaker.cache import cache_region +from beaker.cache import cache_region, region_invalidate from mercurial import ui from rhodecode.lib import helpers as h -from rhodecode.lib.utils import invalidate_cache from rhodecode.lib.auth import HasRepoPermissionAny +from rhodecode.lib.utils import get_repos from rhodecode.model import meta -from rhodecode.model.db import Repository, User +from rhodecode.model.caching_query import FromCache +from rhodecode.model.db import Repository, User, RhodeCodeUi from sqlalchemy.orm import joinedload +from vcs import get_repo as vcs_get_repo, get_backend +from vcs.backends.hg import MercurialRepository from vcs.exceptions import RepositoryError, VCSError +from vcs.utils.lazy import LazyProperty import logging -import sys +import os import time log = logging.getLogger(__name__) -try: - from vcs.backends.hg import MercurialRepository - from vcs.backends.git import GitRepository -except ImportError: - sys.stderr.write('You have to import vcs module') - raise Exception('Unable to import vcs') - -def _get_repos_cached_initial(app_globals, initial): - """return cached dict with repos - """ - g = app_globals - return HgModel().repo_scan(g.paths[0][1], g.baseui, initial) - -@cache_region('long_term', 'cached_repo_list') -def _get_repos_cached(): - """return cached dict with repos - """ - log.info('getting all repositories list') - from pylons import app_globals as g - return HgModel().repo_scan(g.paths[0][1], g.baseui) - -@cache_region('super_short_term', 'cached_repos_switcher_list') -def _get_repos_switcher_cached(cached_repo_list): - repos_lst = [] - for repo in [x for x in cached_repo_list.values()]: - if HasRepoPermissionAny('repository.write', 'repository.read', - 'repository.admin')(repo.name, 'main page check'): - repos_lst.append((repo.name, repo.dbrepo.private,)) - - return sorted(repos_lst, key=lambda k:k[0].lower()) - -@cache_region('long_term', 'full_changelog') -def _full_changelog_cached(repo_name): - log.info('getting full changelog for %s', repo_name) - return list(reversed(list(HgModel().get_repo(repo_name)))) - class HgModel(object): """ Mercurial Model @@ -84,6 +52,16 @@ class HgModel(object): else: self.sa = sa + + @LazyProperty + def repos_path(self): + """ + Get's the repositories root path from database + """ + q = self.sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one() + + return q.ui_value + def repo_scan(self, repos_path, baseui, initial=False): """ Listing of repositories in given path. This path should not be a @@ -91,93 +69,100 @@ class HgModel(object): :param repos_path: path to directory containing repositories :param baseui - :param initial: initial scann + :param initial: initial scan """ log.info('scanning for repositories in %s', repos_path) if not isinstance(baseui, ui.ui): baseui = ui.ui() - - from rhodecode.lib.utils import get_repos - repos = get_repos(repos_path) - - repos_list = {} - for name, path in repos: + for name, path in get_repos(repos_path): try: - #name = name.split('/')[-1] if repos_list.has_key(name): - raise RepositoryError('Duplicate repository name %s found in' - ' %s' % (name, path)) + raise RepositoryError('Duplicate repository name %s ' + 'found in %s' % (name, path)) else: + + klass = get_backend(path[0]) + if path[0] == 'hg': - repos_list[name] = MercurialRepository(path[1], baseui=baseui) - repos_list[name].name = name + repos_list[name] = klass(path[1], baseui=baseui) if path[0] == 'git': - repos_list[name] = GitRepository(path[1]) - repos_list[name].name = name - - dbrepo = None - if not initial: - #for initial scann on application first run we don't - #have db repos yet. - dbrepo = self.sa.query(Repository)\ - .options(joinedload(Repository.fork))\ - .filter(Repository.repo_name == name)\ - .scalar() - - if dbrepo: - log.info('Adding db instance to cached list') - repos_list[name].dbrepo = dbrepo - repos_list[name].description = dbrepo.description - if dbrepo.user: - repos_list[name].contact = dbrepo.user.full_contact - else: - repos_list[name].contact = self.sa.query(User)\ - .filter(User.admin == True).first().full_contact + repos_list[name] = klass(path[1]) except OSError: continue return repos_list - def get_repos(self): - for name, repo in _get_repos_cached().items(): + def get_repos(self, all_repos=None): + """ + Get all repos from db and for each such repo make backend and + fetch dependent data from db + """ + if not all_repos: + all_repos = self.sa.query(Repository).all() - if isinstance(repo, MercurialRepository) and repo._get_hidden(): - #skip hidden web repository - continue + for r in all_repos: + + repo = self.get(r.repo_name) - last_change = repo.last_change - tip = h.get_changeset_safe(repo, 'tip') + if repo is not None: + last_change = repo.last_change + tip = h.get_changeset_safe(repo, 'tip') - tmp_d = {} - tmp_d['name'] = repo.name - tmp_d['name_sort'] = tmp_d['name'].lower() - tmp_d['description'] = repo.description - tmp_d['description_sort'] = tmp_d['description'] - tmp_d['last_change'] = last_change - tmp_d['last_change_sort'] = time.mktime(last_change.timetuple()) - tmp_d['tip'] = tip.raw_id - tmp_d['tip_sort'] = tip.revision - tmp_d['rev'] = tip.revision - tmp_d['contact'] = repo.contact - tmp_d['contact_sort'] = tmp_d['contact'] - tmp_d['repo_archives'] = list(repo._get_archives()) - tmp_d['last_msg'] = tip.message - tmp_d['repo'] = repo - yield tmp_d + tmp_d = {} + tmp_d['name'] = repo.name + tmp_d['name_sort'] = tmp_d['name'].lower() + tmp_d['description'] = repo.dbrepo.description + tmp_d['description_sort'] = tmp_d['description'] + tmp_d['last_change'] = last_change + tmp_d['last_change_sort'] = time.mktime(last_change.timetuple()) + tmp_d['tip'] = tip.raw_id + tmp_d['tip_sort'] = tip.revision + tmp_d['rev'] = tip.revision + tmp_d['contact'] = repo.dbrepo.user.full_contact + tmp_d['contact_sort'] = tmp_d['contact'] + tmp_d['repo_archives'] = list(repo._get_archives()) + tmp_d['last_msg'] = tip.message + tmp_d['repo'] = repo + yield tmp_d def get_repo(self, repo_name): - try: - repo = _get_repos_cached()[repo_name] - return repo - except KeyError: - #i we're here and we got key errors let's try to invalidate the - #cahce and try again - invalidate_cache('cached_repo_list') - repo = _get_repos_cached()[repo_name] + return self.get(repo_name) + + def get(self, repo_name): + """ + Get's repository from given name, creates BackendInstance and + propagates it's data from database with all additional information + :param repo_name: + """ + if not HasRepoPermissionAny('repository.read', 'repository.write', + 'repository.admin')(repo_name, 'get repo check'): + return + + @cache_region('long_term', 'get_repo_cached_%s' % repo_name) + def _get_repo(repo_name): + + repo = vcs_get_repo(os.path.join(self.repos_path, repo_name), + alias=None, create=False) + + #skip hidden web repository + if isinstance(repo, MercurialRepository) and repo._get_hidden(): + return + + dbrepo = self.sa.query(Repository)\ + .options(joinedload(Repository.fork))\ + .options(joinedload(Repository.user))\ + .filter(Repository.repo_name == repo_name)\ + .scalar() + repo.dbrepo = dbrepo return repo + invalidate = False + if invalidate: + log.info('INVALIDATING CACHE FOR %s', repo_name) + region_invalidate(_get_repo, None, repo_name) + return _get_repo(repo_name) diff --git a/rhodecode/templates/admin/users/user_edit_my_account.html b/rhodecode/templates/admin/users/user_edit_my_account.html --- a/rhodecode/templates/admin/users/user_edit_my_account.html +++ b/rhodecode/templates/admin/users/user_edit_my_account.html @@ -100,32 +100,32 @@ %for repo in c.user_repos: - %if repo.dbrepo.repo_type =='hg': + %if repo['repo'].dbrepo.repo_type =='hg': ${_('Mercurial repository')} - %elif repo.dbrepo.repo_type =='git': + %elif repo['repo'].dbrepo.repo_type =='git': ${_('Git repository')} %else: %endif - %if repo.dbrepo.private: + %if repo['repo'].dbrepo.private: ${_('private')} %else: ${_('public')} %endif - ${h.link_to(repo.name, h.url('summary_home',repo_name=repo.name),class_="repo_name")} - %if repo.dbrepo.fork: - + ${h.link_to(repo['repo'].name, h.url('summary_home',repo_name=repo['repo'].name),class_="repo_name")} + %if repo['repo'].dbrepo.fork: + ${_('public')} %endif - ${("r%s:%s") % (h.get_changeset_safe(repo,'tip').revision,h.short_id(h.get_changeset_safe(repo,'tip').raw_id))} - ${_('private')} + ${("r%s:%s") % (h.get_changeset_safe(repo['repo'],'tip').revision,h.short_id(h.get_changeset_safe(repo['repo'],'tip').raw_id))} + ${_('private')} - ${h.form(url('repo_settings_delete', repo_name=repo.name),method='delete')} - ${h.submit('remove_%s' % repo.name,'',class_="delete_icon action_button",onclick="return confirm('Confirm to delete this repository');")} + ${h.form(url('repo_settings_delete', repo_name=repo['repo'].name),method='delete')} + ${h.submit('remove_%s' % repo['repo'].name,'',class_="delete_icon action_button",onclick="return confirm('Confirm to delete this repository');")} ${h.end_form()} diff --git a/rhodecode/templates/base/base.html b/rhodecode/templates/base/base.html --- a/rhodecode/templates/base/base.html +++ b/rhodecode/templates/base/base.html @@ -98,11 +98,12 @@ ↓