diff --git a/rhodecode/controllers/admin/repos.py b/rhodecode/controllers/admin/repos.py --- a/rhodecode/controllers/admin/repos.py +++ b/rhodecode/controllers/admin/repos.py @@ -89,10 +89,8 @@ class ReposController(BaseController): """ self.__load_defaults() - repo, dbrepo = ScmModel().get(repo_name, retval='repo') - - repo_model = RepoModel() - c.repo_info = repo_model.get_by_repo_name(repo_name) + c.repo_info = db_repo = Repository.by_repo_name(repo_name) + repo = scm_repo = db_repo.scm_instance if c.repo_info is None: h.flash(_('%s repository is not mapped to db perhaps' @@ -153,10 +151,9 @@ class ReposController(BaseController): """GET /repos: All items in the collection""" # url('repos') - all_repos = [r.repo_name for r in Repository.query().all()] - - cached_repo_list = ScmModel().get_repos(all_repos) - c.repos_list = sorted(cached_repo_list, key=itemgetter('name_sort')) + c.repos_list = ScmModel().get_repos(Repository.query() + .order_by(Repository.repo_name) + .all(), sort_key='name_sort') return render('admin/repos/repos.html') @HasPermissionAnyDecorator('hg.admin', 'hg.create.repository') diff --git a/rhodecode/controllers/admin/repos_groups.py b/rhodecode/controllers/admin/repos_groups.py --- a/rhodecode/controllers/admin/repos_groups.py +++ b/rhodecode/controllers/admin/repos_groups.py @@ -181,12 +181,14 @@ class ReposGroupsController(BaseControll """GET /repos_groups/id: Show a specific item""" # url('repos_group', id=ID) - c.group = Group.get(id) + gr = c.group = Group.get(id) + if c.group: c.group_repos = c.group.repositories.all() else: return redirect(url('repos_group')) + sortables = ['name', 'description', 'last_change', 'tip', 'owner'] current_sort = request.GET.get('sort', 'name') current_sort_slug = current_sort.replace('-', '') @@ -201,18 +203,12 @@ class ReposGroupsController(BaseControll sort_key = current_sort_slug + '_sort' #overwrite our cached list with current filter - gr_filter = [r.repo_name for r in c.group_repos] + gr_filter = c.group_repos c.cached_repo_list = self.scm_model.get_repos(all_repos=gr_filter) - if c.sort_by.startswith('-'): - c.repos_list = sorted(c.cached_repo_list, key=itemgetter(sort_key), - reverse=True) - else: - c.repos_list = sorted(c.cached_repo_list, key=itemgetter(sort_key), - reverse=False) + c.repos_list = c.cached_repo_list - c.repo_cnt = len(c.repos_list) - + c.repo_cnt = 0 c.groups = self.sa.query(Group).order_by(Group.group_name)\ .filter(Group.group_parent_id == id).all() diff --git a/rhodecode/controllers/admin/settings.py b/rhodecode/controllers/admin/settings.py --- a/rhodecode/controllers/admin/settings.py +++ b/rhodecode/controllers/admin/settings.py @@ -258,9 +258,10 @@ class SettingsController(BaseController) # url('admin_settings_my_account') c.user = UserModel().get(self.rhodecode_user.user_id, cache=False) - all_repos = [r.repo_name for r in self.sa.query(Repository)\ + all_repos = self.sa.query(Repository)\ .filter(Repository.user_id == c.user.user_id)\ - .order_by(func.lower(Repository.repo_name)).all()] + .order_by(func.lower(Repository.repo_name)).all() + c.user_repos = ScmModel().get_repos(all_repos) if c.user.username == 'default': diff --git a/rhodecode/controllers/home.py b/rhodecode/controllers/home.py --- a/rhodecode/controllers/home.py +++ b/rhodecode/controllers/home.py @@ -31,7 +31,7 @@ from paste.httpexceptions import HTTPBad from rhodecode.lib.auth import LoginRequired from rhodecode.lib.base import BaseController, render -from rhodecode.model.db import Group +from rhodecode.model.db import Group, Repository log = logging.getLogger(__name__) @@ -56,16 +56,11 @@ class HomeController(BaseController): sort_key = current_sort_slug + '_sort' - if c.sort_by.startswith('-'): - c.repos_list = sorted(c.cached_repo_list, key=itemgetter(sort_key), - reverse=True) - else: - c.repos_list = sorted(c.cached_repo_list, key=itemgetter(sort_key), - reverse=False) + + c.repos_list = self.scm_model.get_repos(sort_key=sort_key) c.repo_cnt = len(c.repos_list) - c.groups = Group.query().filter(Group.group_parent_id == None).all() @@ -73,8 +68,9 @@ class HomeController(BaseController): def repo_switcher(self): if request.is_xhr: - c.repos_list = sorted(c.cached_repo_list, - key=itemgetter('name_sort'), reverse=False) + all_repos = Repository.query().order_by(Repository.repo_name).all() + c.repos_list = self.scm_model.get_repos(all_repos, + sort_key='name_sort') return render('/repo_switcher_list.html') else: return HTTPBadRequest() diff --git a/rhodecode/controllers/settings.py b/rhodecode/controllers/settings.py --- a/rhodecode/controllers/settings.py +++ b/rhodecode/controllers/settings.py @@ -155,6 +155,7 @@ class SettingsController(BaseRepoControl invalidate_cache('get_repo_cached_%s' % repo_name) h.flash(_('deleted repository %s') % repo_name, category='success') except Exception: + log.error(traceback.format_exc()) h.flash(_('An error occurred during deletion of %s') % repo_name, category='error') @@ -205,4 +206,9 @@ class SettingsController(BaseRepoControl errors=errors.error_dict or {}, prefix_error=False, encoding="UTF-8") + except Exception: + log.error(traceback.format_exc()) + h.flash(_('An error occurred during repository forking %s') % + repo_name, category='error') + return redirect(url('home')) diff --git a/rhodecode/lib/base.py b/rhodecode/lib/base.py --- a/rhodecode/lib/base.py +++ b/rhodecode/lib/base.py @@ -12,6 +12,7 @@ from rhodecode.lib.utils import get_repo from rhodecode.model import meta from rhodecode.model.scm import ScmModel from rhodecode import BACKENDS +from rhodecode.model.db import Repository class BaseController(WSGIController): @@ -26,7 +27,7 @@ class BaseController(WSGIController): self.sa = meta.Session() self.scm_model = ScmModel(self.sa) - c.cached_repo_list = self.scm_model.get_repos() + #c.unread_journal = scm_model.get_unread_journal() def __call__(self, environ, start_response): @@ -62,8 +63,7 @@ class BaseRepoController(BaseController) super(BaseRepoController, self).__before__() if c.repo_name: - c.rhodecode_repo, dbrepo = self.scm_model.get(c.repo_name, - retval='repo') + c.rhodecode_repo = Repository.by_repo_name(c.repo_name).scm_instance if c.rhodecode_repo is not None: c.repository_followers = \ diff --git a/rhodecode/lib/celerylib/tasks.py b/rhodecode/lib/celerylib/tasks.py --- a/rhodecode/lib/celerylib/tasks.py +++ b/rhodecode/lib/celerylib/tasks.py @@ -102,7 +102,6 @@ def get_commits_stats(repo_name, ts_min_ lockkey = __get_lockkey('get_commits_stats', repo_name, ts_min_y, ts_max_y) lockkey_path = dn(dn(dn(dn(os.path.abspath(__file__))))) - print jn(lockkey_path, lockkey) log.info('running task with lockkey %s', lockkey) try: lock = l = DaemonLock(jn(lockkey_path, lockkey)) diff --git a/rhodecode/lib/helpers.py b/rhodecode/lib/helpers.py --- a/rhodecode/lib/helpers.py +++ b/rhodecode/lib/helpers.py @@ -372,8 +372,7 @@ def action_parser(user_log, feed=False): repo_name = user_log.repository.repo_name from rhodecode.model.scm import ScmModel - repo, dbrepo = ScmModel().get(repo_name, retval='repo', - invalidation_list=[]) + repo = user_log.repository.scm_instance message = lambda rev: get_changeset_safe(repo, rev).message cs_links = [] diff --git a/rhodecode/lib/utils.py b/rhodecode/lib/utils.py --- a/rhodecode/lib/utils.py +++ b/rhodecode/lib/utils.py @@ -472,7 +472,7 @@ def create_test_index(repo_location, ful shutil.rmtree(index_location) try: - l = DaemonLock(file=jn(dn(dn(index_location)), 'make_index.lock')) + l = DaemonLock(file=jn(dn(index_location), 'make_index.lock')) WhooshIndexingDaemon(index_location=index_location, repo_location=repo_location)\ .run(full_index=full_index) diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py --- a/rhodecode/model/db.py +++ b/rhodecode/model/db.py @@ -26,13 +26,23 @@ import os import logging import datetime +import traceback from datetime import date from sqlalchemy import * from sqlalchemy.exc import DatabaseError -from sqlalchemy.orm import relationship, backref +from sqlalchemy.orm import relationship, backref, joinedload from sqlalchemy.orm.interfaces import MapperExtension +from beaker.cache import cache_region, region_invalidate + + +from vcs import get_backend +from vcs.utils.helpers import get_scm +from vcs.exceptions import RepositoryError, VCSError +from vcs.utils.lazy import LazyProperty +from vcs.nodes import FileNode + from rhodecode.lib import str2bool from rhodecode.model.meta import Base, Session from rhodecode.model.caching_query import FromCache @@ -150,6 +160,7 @@ class User(Base): return self.admin def __repr__(self): + return 'ahmmm' return "<%s('id:%s:%s')>" % (self.__class__.__name__, self.user_id, self.username) @@ -266,8 +277,13 @@ class Repository(Base): @classmethod def by_repo_name(cls, repo_name): - return Session.query(cls).filter(cls.repo_name == repo_name).one() + q = Session.query(cls).filter(cls.repo_name == repo_name) + q = q.options(joinedload(Repository.fork))\ + .options(joinedload(Repository.user))\ + .options(joinedload(Repository.group))\ + + return q.one() @classmethod def get_repo_forks(cls, repo_id): @@ -298,6 +314,127 @@ class Repository(Base): def groups_and_repo(self): return self.groups_with_parents, self.just_name + @LazyProperty + def repo_path(self): + """ + Returns base full path for that repository means where it actually + exists on a filesystem + """ + + q = Session.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one() + return q.ui_value + + @property + def repo_full_path(self): + p = [self.repo_path] + # we need to split the name by / since this is how we store the + # names in the database, but that eventually needs to be converted + # into a valid system path + p += self.repo_name.split('/') + return os.path.join(*p) + + @property + def _ui(self): + """ + Creates an db based ui object for this repository + """ + from mercurial import ui + from mercurial import config + baseui = ui.ui() + + #clean the baseui object + baseui._ocfg = config.config() + baseui._ucfg = config.config() + baseui._tcfg = config.config() + + + ret = Session.query(RhodeCodeUi)\ + .options(FromCache("sql_cache_short", + "repository_repo_ui")).all() + + hg_ui = ret + for ui_ in hg_ui: + if ui_.ui_active: + log.debug('settings ui from db[%s]%s:%s', ui_.ui_section, + ui_.ui_key, ui_.ui_value) + baseui.setconfig(ui_.ui_section, ui_.ui_key, ui_.ui_value) + + return baseui + + #========================================================================== + # SCM CACHE INSTANCE + #========================================================================== + + @property + def invalidate(self): + """ + Returns Invalidation object if this repo should be invalidated + None otherwise. `cache_active = False` means that this cache + state is not valid and needs to be invalidated + """ + return Session.query(CacheInvalidation)\ + .filter(CacheInvalidation.cache_key == self.repo_name)\ + .filter(CacheInvalidation.cache_active == False)\ + .scalar() + + @property + def set_invalidate(self): + """ + set a cache for invalidation for this instance + """ + inv = Session.query(CacheInvalidation)\ + .filter(CacheInvalidation.cache_key == self.repo_name)\ + .scalar() + + if inv is None: + inv = CacheInvalidation(self.repo_name) + inv.cache_active = True + Session.add(inv) + Session.commit() + + @property + def scm_instance(self): + return self.__get_instance(self.repo_name) + + @property + def scm_instance_cached(self): + @cache_region('long_term') + def _c(repo_name): + return self.__get_instance(repo_name) + + inv = self.invalidate + if inv: + region_invalidate(_c, None, self.repo_name) + #update our cache + inv.cache_key.cache_active = True + Session.add(inv) + Session.commit() + + return _c(self.repo_name) + + def __get_instance(self, repo_name): + try: + alias = get_scm(self.repo_full_path)[0] + log.debug('Creating instance of %s repository', alias) + backend = get_backend(alias) + except VCSError: + log.error(traceback.format_exc()) + log.error('Perhaps this repository is in db and not in ' + 'filesystem run rescan repositories with ' + '"destroy old data " option from admin panel') + return + + if alias == 'hg': + repo = backend(self.repo_full_path, create=False, + baseui=self._ui) + #skip hidden web repository + if repo._get_hidden(): + return + else: + repo = backend(self.repo_full_path, create=False) + + return repo + class Group(Base): __tablename__ = 'groups' diff --git a/rhodecode/model/forms.py b/rhodecode/model/forms.py --- a/rhodecode/model/forms.py +++ b/rhodecode/model/forms.py @@ -280,6 +280,13 @@ def ValidRepoName(edit, old_data): return _ValidRepoName +def ValidForkName(): + class _ValidForkName(formencode.validators.FancyValidator): + def to_python(self, value, state): + return value + return _ValidForkName + + def SlugifyName(): class _SlugifyName(formencode.validators.FancyValidator): @@ -326,6 +333,7 @@ def ValidForkType(old_data): if old_data['repo_type'] != value: raise formencode.Invalid(_('Fork have to be the same ' 'type as original'), value, state) + return value return _ValidForkType @@ -583,6 +591,9 @@ def RepoForkForm(edit=False, old_data={} description = UnicodeString(strip=True, min=1, not_empty=True) private = StringBoolean(if_missing=False) repo_type = All(ValidForkType(old_data), OneOf(supported_backends)) + + chained_validators = [ValidForkName()] + return _RepoForkForm def RepoSettingsForm(edit=False, old_data={}): diff --git a/rhodecode/model/repo.py b/rhodecode/model/repo.py --- a/rhodecode/model/repo.py +++ b/rhodecode/model/repo.py @@ -70,28 +70,6 @@ class RepoModel(BaseModel): "get_repo_%s" % repo_name)) return repo.scalar() - def get_full(self, repo_name, cache=False, invalidate=False): - repo = self.sa.query(Repository)\ - .options(joinedload(Repository.fork))\ - .options(joinedload(Repository.user))\ - .options(joinedload(Repository.group))\ - .filter(Repository.repo_name == repo_name)\ - - if cache: - repo = repo.options(FromCache("sql_cache_long", - "get_repo_full_%s" % repo_name)) - if invalidate and cache: - repo.invalidate() - - ret = repo.scalar() - - #make transient for sake of errors - make_transient(ret) - for k in ['fork', 'user', 'group']: - attr = getattr(ret, k, False) - if attr: - make_transient(attr) - return ret def get_users_js(self): @@ -193,12 +171,13 @@ class RepoModel(BaseModel): raise def create(self, form_data, cur_user, just_db=False, fork=False): + try: if fork: #force str since hg doesn't go with unicode repo_name = str(form_data['fork_name']) org_name = str(form_data['repo_name']) - org_full_name = str(form_data['repo_name_full']) + org_full_name = org_name#str(form_data['fork_name_full']) else: org_name = repo_name = str(form_data['repo_name']) @@ -208,7 +187,10 @@ class RepoModel(BaseModel): new_repo.enable_statistics = False for k, v in form_data.items(): if k == 'repo_name': - v = repo_name_full + if fork: + v = repo_name + else: + v = repo_name_full if k == 'repo_group': k = 'group_id' @@ -216,7 +198,7 @@ class RepoModel(BaseModel): if fork: parent_repo = self.sa.query(Repository)\ - .filter(Repository.repo_name == org_full_name).scalar() + .filter(Repository.repo_name == org_full_name).one() new_repo.fork = parent_repo new_repo.user_id = cur_user.user_id diff --git a/rhodecode/model/scm.py b/rhodecode/model/scm.py --- a/rhodecode/model/scm.py +++ b/rhodecode/model/scm.py @@ -70,6 +70,75 @@ class RepoTemp(object): def __repr__(self): return "<%s('id:%s')>" % (self.__class__.__name__, self.repo_id) +class CachedRepoList(object): + + def __init__(self, db_repo_list, invalidation_list, repos_path, + order_by=None): + self.db_repo_list = db_repo_list + self.invalidation_list = invalidation_list + self.repos_path = repos_path + self.order_by = order_by + self.reversed = (order_by or '').startswith('-') + + def __len__(self): + return len(self.db_repo_list) + + def __repr__(self): + return '<%s (%s)>' % (self.__class__.__name__, self.__len__()) + + def __iter__(self): + for db_repo in self.db_repo_list: + dbr = db_repo + + # invalidate the repo cache if needed before getting the + # scm instance + + scm_invalidate = False + if self.invalidation_list is not None: + scm_invalidate = dbr.repo_name in self.invalidation_list + + if scm_invalidate: + log.info('invalidating cache for repository %s', + dbr.repo_name) + db_repo.set_invalidate + + scmr = db_repo.scm_instance_cached + + #check permission at this level + if not HasRepoPermissionAny('repository.read', + 'repository.write', + 'repository.admin')(dbr.repo_name, + 'get repo check'): + continue + + + + + + last_change = scmr.last_change + tip = h.get_changeset_safe(scmr, 'tip') + + tmp_d = {} + tmp_d['name'] = dbr.repo_name + tmp_d['name_sort'] = tmp_d['name'].lower() + tmp_d['description'] = dbr.description + tmp_d['description_sort'] = tmp_d['description'] + tmp_d['last_change'] = last_change + tmp_d['last_change_sort'] = time.mktime(last_change \ + .timetuple()) + tmp_d['tip'] = tip.raw_id + tmp_d['tip_sort'] = tip.revision + tmp_d['rev'] = tip.revision + tmp_d['contact'] = dbr.user.full_contact + tmp_d['contact_sort'] = tmp_d['contact'] + tmp_d['owner_sort'] = tmp_d['contact'] + tmp_d['repo_archives'] = list(scmr._get_archives()) + tmp_d['last_msg'] = tip.message + tmp_d['repo'] = scmr + tmp_d['dbrepo'] = dbr.get_dict() + tmp_d['dbrepo_fork'] = dbr.fork.get_dict() if dbr.fork \ + else {} + yield tmp_d class ScmModel(BaseModel): """Generic Scm Model @@ -118,7 +187,7 @@ class ScmModel(BaseModel): return repos_list - def get_repos(self, all_repos=None): + def get_repos(self, all_repos=None, sort_key=None): """ Get all repos from db and for each repo create it's backend instance and fill that backed with information from database @@ -127,120 +196,21 @@ class ScmModel(BaseModel): give specific repositories list, good for filtering """ if all_repos is None: - repos = self.sa.query(Repository)\ + all_repos = self.sa.query(Repository)\ .filter(Repository.group_id == None)\ .order_by(Repository.repo_name).all() - all_repos = [r.repo_name for r in repos] #get the repositories that should be invalidated invalidation_list = [str(x.cache_key) for x in \ self.sa.query(CacheInvalidation.cache_key)\ .filter(CacheInvalidation.cache_active == False)\ .all()] - for r_name in all_repos: - r_dbr = self.get(r_name, invalidation_list) - if r_dbr is not None: - repo, dbrepo = r_dbr - - if repo is None or dbrepo is None: - log.error('Repository "%s" looks somehow corrupted ' - 'fs-repo:%s,db-repo:%s both values should be ' - 'present', r_name, repo, dbrepo) - continue - last_change = repo.last_change - tip = h.get_changeset_safe(repo, 'tip') - - tmp_d = {} - tmp_d['name'] = dbrepo.repo_name - tmp_d['name_sort'] = tmp_d['name'].lower() - tmp_d['description'] = dbrepo.description - tmp_d['description_sort'] = tmp_d['description'] - tmp_d['last_change'] = last_change - tmp_d['last_change_sort'] = time.mktime(last_change \ - .timetuple()) - tmp_d['tip'] = tip.raw_id - tmp_d['tip_sort'] = tip.revision - tmp_d['rev'] = tip.revision - tmp_d['contact'] = dbrepo.user.full_contact - tmp_d['contact_sort'] = tmp_d['contact'] - tmp_d['owner_sort'] = tmp_d['contact'] - tmp_d['repo_archives'] = list(repo._get_archives()) - tmp_d['last_msg'] = tip.message - tmp_d['repo'] = repo - tmp_d['dbrepo'] = dbrepo.get_dict() - tmp_d['dbrepo_fork'] = dbrepo.fork.get_dict() if dbrepo.fork \ - else {} - yield tmp_d - - def get(self, repo_name, invalidation_list=None, retval='all'): - """Returns a tuple of Repository,DbRepository, - Get's repository from given name, creates BackendInstance and - propagates it's data from database with all additional information - - :param repo_name: - :param invalidation_list: if a invalidation list is given the get - method should not manually check if this repository needs - invalidation and just invalidate the repositories in list - :param retval: string specifing what to return one of 'repo','dbrepo', - 'all'if repo or dbrepo is given it'll just lazy load chosen type - and return None as the second - """ - if not HasRepoPermissionAny('repository.read', 'repository.write', - 'repository.admin')(repo_name, 'get repo check'): - return - #====================================================================== - # CACHE FUNCTION - #====================================================================== - @cache_region('long_term') - def _get_repo(repo_name): - - repo_path = os.path.join(self.repos_path, repo_name) - - try: - alias = get_scm(repo_path)[0] - log.debug('Creating instance of %s repository', alias) - backend = get_backend(alias) - except VCSError: - log.error(traceback.format_exc()) - log.error('Perhaps this repository is in db and not in ' - 'filesystem run rescan repositories with ' - '"destroy old data " option from admin panel') - return + repo_iter = CachedRepoList(all_repos, invalidation_list, + repos_path=self.repos_path, + order_by=sort_key) - if alias == 'hg': - repo = backend(repo_path, create=False, baseui=make_ui('db')) - #skip hidden web repository - if repo._get_hidden(): - return - else: - repo = backend(repo_path, create=False) - - return repo - - pre_invalidate = True - dbinvalidate = False - - if invalidation_list is not None: - pre_invalidate = repo_name in invalidation_list - - if pre_invalidate: - #this returns object to invalidate - invalidate = self._should_invalidate(repo_name) - if invalidate: - log.info('invalidating cache for repository %s', repo_name) - region_invalidate(_get_repo, None, repo_name) - self._mark_invalidated(invalidate) - dbinvalidate = True - - r, dbr = None, None - if retval == 'repo' or 'all': - r = _get_repo(repo_name) - if retval == 'dbrepo' or 'all': - dbr = RepoModel().get_full(repo_name, cache=True, - invalidate=dbinvalidate) - - return r, dbr + return repo_iter def mark_for_invalidation(self, repo_name): """Puts cache invalidation task into db for diff --git a/rhodecode/templates/admin/users/user_edit.html b/rhodecode/templates/admin/users/user_edit.html --- a/rhodecode/templates/admin/users/user_edit.html +++ b/rhodecode/templates/admin/users/user_edit.html @@ -65,7 +65,7 @@