# HG changeset patch # User Marcin Kuzminski # Date 2011-06-07 15:58:51 # Node ID 9c0f5d5587895561148b46c68b989d19eeddb0ff # Parent cd865113423eeee6273577c0ef9a38f8fec5f5ba fixes #200, rewrote the whole caching mechanism to get rid of such problems. Now cached instances are attached to db repository instance, and then fetched from cache. Also made all current test work. diff --git a/rhodecode/controllers/admin/repos.py b/rhodecode/controllers/admin/repos.py --- a/rhodecode/controllers/admin/repos.py +++ b/rhodecode/controllers/admin/repos.py @@ -89,10 +89,8 @@ class ReposController(BaseController): """ self.__load_defaults() - repo, dbrepo = ScmModel().get(repo_name, retval='repo') - - repo_model = RepoModel() - c.repo_info = repo_model.get_by_repo_name(repo_name) + c.repo_info = db_repo = Repository.by_repo_name(repo_name) + repo = scm_repo = db_repo.scm_instance if c.repo_info is None: h.flash(_('%s repository is not mapped to db perhaps' @@ -153,10 +151,9 @@ class ReposController(BaseController): """GET /repos: All items in the collection""" # url('repos') - all_repos = [r.repo_name for r in Repository.query().all()] - - cached_repo_list = ScmModel().get_repos(all_repos) - c.repos_list = sorted(cached_repo_list, key=itemgetter('name_sort')) + c.repos_list = ScmModel().get_repos(Repository.query() + .order_by(Repository.repo_name) + .all(), sort_key='name_sort') return render('admin/repos/repos.html') @HasPermissionAnyDecorator('hg.admin', 'hg.create.repository') diff --git a/rhodecode/controllers/admin/repos_groups.py b/rhodecode/controllers/admin/repos_groups.py --- a/rhodecode/controllers/admin/repos_groups.py +++ b/rhodecode/controllers/admin/repos_groups.py @@ -181,12 +181,14 @@ class ReposGroupsController(BaseControll """GET /repos_groups/id: Show a specific item""" # url('repos_group', id=ID) - c.group = Group.get(id) + gr = c.group = Group.get(id) + if c.group: c.group_repos = c.group.repositories.all() else: return redirect(url('repos_group')) + sortables = ['name', 'description', 'last_change', 'tip', 'owner'] current_sort = request.GET.get('sort', 'name') current_sort_slug = current_sort.replace('-', '') @@ -201,18 +203,12 @@ class ReposGroupsController(BaseControll sort_key = current_sort_slug + '_sort' #overwrite our cached list with current filter - gr_filter = [r.repo_name for r in c.group_repos] + gr_filter = c.group_repos c.cached_repo_list = self.scm_model.get_repos(all_repos=gr_filter) - if c.sort_by.startswith('-'): - c.repos_list = sorted(c.cached_repo_list, key=itemgetter(sort_key), - reverse=True) - else: - c.repos_list = sorted(c.cached_repo_list, key=itemgetter(sort_key), - reverse=False) + c.repos_list = c.cached_repo_list - c.repo_cnt = len(c.repos_list) - + c.repo_cnt = 0 c.groups = self.sa.query(Group).order_by(Group.group_name)\ .filter(Group.group_parent_id == id).all() diff --git a/rhodecode/controllers/admin/settings.py b/rhodecode/controllers/admin/settings.py --- a/rhodecode/controllers/admin/settings.py +++ b/rhodecode/controllers/admin/settings.py @@ -258,9 +258,10 @@ class SettingsController(BaseController) # url('admin_settings_my_account') c.user = UserModel().get(self.rhodecode_user.user_id, cache=False) - all_repos = [r.repo_name for r in self.sa.query(Repository)\ + all_repos = self.sa.query(Repository)\ .filter(Repository.user_id == c.user.user_id)\ - .order_by(func.lower(Repository.repo_name)).all()] + .order_by(func.lower(Repository.repo_name)).all() + c.user_repos = ScmModel().get_repos(all_repos) if c.user.username == 'default': diff --git a/rhodecode/controllers/home.py b/rhodecode/controllers/home.py --- a/rhodecode/controllers/home.py +++ b/rhodecode/controllers/home.py @@ -31,7 +31,7 @@ from paste.httpexceptions import HTTPBad from rhodecode.lib.auth import LoginRequired from rhodecode.lib.base import BaseController, render -from rhodecode.model.db import Group +from rhodecode.model.db import Group, Repository log = logging.getLogger(__name__) @@ -56,16 +56,11 @@ class HomeController(BaseController): sort_key = current_sort_slug + '_sort' - if c.sort_by.startswith('-'): - c.repos_list = sorted(c.cached_repo_list, key=itemgetter(sort_key), - reverse=True) - else: - c.repos_list = sorted(c.cached_repo_list, key=itemgetter(sort_key), - reverse=False) + + c.repos_list = self.scm_model.get_repos(sort_key=sort_key) c.repo_cnt = len(c.repos_list) - c.groups = Group.query().filter(Group.group_parent_id == None).all() @@ -73,8 +68,9 @@ class HomeController(BaseController): def repo_switcher(self): if request.is_xhr: - c.repos_list = sorted(c.cached_repo_list, - key=itemgetter('name_sort'), reverse=False) + all_repos = Repository.query().order_by(Repository.repo_name).all() + c.repos_list = self.scm_model.get_repos(all_repos, + sort_key='name_sort') return render('/repo_switcher_list.html') else: return HTTPBadRequest() diff --git a/rhodecode/controllers/settings.py b/rhodecode/controllers/settings.py --- a/rhodecode/controllers/settings.py +++ b/rhodecode/controllers/settings.py @@ -155,6 +155,7 @@ class SettingsController(BaseRepoControl invalidate_cache('get_repo_cached_%s' % repo_name) h.flash(_('deleted repository %s') % repo_name, category='success') except Exception: + log.error(traceback.format_exc()) h.flash(_('An error occurred during deletion of %s') % repo_name, category='error') @@ -205,4 +206,9 @@ class SettingsController(BaseRepoControl errors=errors.error_dict or {}, prefix_error=False, encoding="UTF-8") + except Exception: + log.error(traceback.format_exc()) + h.flash(_('An error occurred during repository forking %s') % + repo_name, category='error') + return redirect(url('home')) diff --git a/rhodecode/lib/base.py b/rhodecode/lib/base.py --- a/rhodecode/lib/base.py +++ b/rhodecode/lib/base.py @@ -12,6 +12,7 @@ from rhodecode.lib.utils import get_repo from rhodecode.model import meta from rhodecode.model.scm import ScmModel from rhodecode import BACKENDS +from rhodecode.model.db import Repository class BaseController(WSGIController): @@ -26,7 +27,7 @@ class BaseController(WSGIController): self.sa = meta.Session() self.scm_model = ScmModel(self.sa) - c.cached_repo_list = self.scm_model.get_repos() + #c.unread_journal = scm_model.get_unread_journal() def __call__(self, environ, start_response): @@ -62,8 +63,7 @@ class BaseRepoController(BaseController) super(BaseRepoController, self).__before__() if c.repo_name: - c.rhodecode_repo, dbrepo = self.scm_model.get(c.repo_name, - retval='repo') + c.rhodecode_repo = Repository.by_repo_name(c.repo_name).scm_instance if c.rhodecode_repo is not None: c.repository_followers = \ diff --git a/rhodecode/lib/celerylib/tasks.py b/rhodecode/lib/celerylib/tasks.py --- a/rhodecode/lib/celerylib/tasks.py +++ b/rhodecode/lib/celerylib/tasks.py @@ -102,7 +102,6 @@ def get_commits_stats(repo_name, ts_min_ lockkey = __get_lockkey('get_commits_stats', repo_name, ts_min_y, ts_max_y) lockkey_path = dn(dn(dn(dn(os.path.abspath(__file__))))) - print jn(lockkey_path, lockkey) log.info('running task with lockkey %s', lockkey) try: lock = l = DaemonLock(jn(lockkey_path, lockkey)) diff --git a/rhodecode/lib/helpers.py b/rhodecode/lib/helpers.py --- a/rhodecode/lib/helpers.py +++ b/rhodecode/lib/helpers.py @@ -372,8 +372,7 @@ def action_parser(user_log, feed=False): repo_name = user_log.repository.repo_name from rhodecode.model.scm import ScmModel - repo, dbrepo = ScmModel().get(repo_name, retval='repo', - invalidation_list=[]) + repo = user_log.repository.scm_instance message = lambda rev: get_changeset_safe(repo, rev).message cs_links = [] diff --git a/rhodecode/lib/utils.py b/rhodecode/lib/utils.py --- a/rhodecode/lib/utils.py +++ b/rhodecode/lib/utils.py @@ -472,7 +472,7 @@ def create_test_index(repo_location, ful shutil.rmtree(index_location) try: - l = DaemonLock(file=jn(dn(dn(index_location)), 'make_index.lock')) + l = DaemonLock(file=jn(dn(index_location), 'make_index.lock')) WhooshIndexingDaemon(index_location=index_location, repo_location=repo_location)\ .run(full_index=full_index) diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py --- a/rhodecode/model/db.py +++ b/rhodecode/model/db.py @@ -26,13 +26,23 @@ import os import logging import datetime +import traceback from datetime import date from sqlalchemy import * from sqlalchemy.exc import DatabaseError -from sqlalchemy.orm import relationship, backref +from sqlalchemy.orm import relationship, backref, joinedload from sqlalchemy.orm.interfaces import MapperExtension +from beaker.cache import cache_region, region_invalidate + + +from vcs import get_backend +from vcs.utils.helpers import get_scm +from vcs.exceptions import RepositoryError, VCSError +from vcs.utils.lazy import LazyProperty +from vcs.nodes import FileNode + from rhodecode.lib import str2bool from rhodecode.model.meta import Base, Session from rhodecode.model.caching_query import FromCache @@ -150,6 +160,7 @@ class User(Base): return self.admin def __repr__(self): + return 'ahmmm' return "<%s('id:%s:%s')>" % (self.__class__.__name__, self.user_id, self.username) @@ -266,8 +277,13 @@ class Repository(Base): @classmethod def by_repo_name(cls, repo_name): - return Session.query(cls).filter(cls.repo_name == repo_name).one() + q = Session.query(cls).filter(cls.repo_name == repo_name) + q = q.options(joinedload(Repository.fork))\ + .options(joinedload(Repository.user))\ + .options(joinedload(Repository.group))\ + + return q.one() @classmethod def get_repo_forks(cls, repo_id): @@ -298,6 +314,127 @@ class Repository(Base): def groups_and_repo(self): return self.groups_with_parents, self.just_name + @LazyProperty + def repo_path(self): + """ + Returns base full path for that repository means where it actually + exists on a filesystem + """ + + q = Session.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one() + return q.ui_value + + @property + def repo_full_path(self): + p = [self.repo_path] + # we need to split the name by / since this is how we store the + # names in the database, but that eventually needs to be converted + # into a valid system path + p += self.repo_name.split('/') + return os.path.join(*p) + + @property + def _ui(self): + """ + Creates an db based ui object for this repository + """ + from mercurial import ui + from mercurial import config + baseui = ui.ui() + + #clean the baseui object + baseui._ocfg = config.config() + baseui._ucfg = config.config() + baseui._tcfg = config.config() + + + ret = Session.query(RhodeCodeUi)\ + .options(FromCache("sql_cache_short", + "repository_repo_ui")).all() + + hg_ui = ret + for ui_ in hg_ui: + if ui_.ui_active: + log.debug('settings ui from db[%s]%s:%s', ui_.ui_section, + ui_.ui_key, ui_.ui_value) + baseui.setconfig(ui_.ui_section, ui_.ui_key, ui_.ui_value) + + return baseui + + #========================================================================== + # SCM CACHE INSTANCE + #========================================================================== + + @property + def invalidate(self): + """ + Returns Invalidation object if this repo should be invalidated + None otherwise. `cache_active = False` means that this cache + state is not valid and needs to be invalidated + """ + return Session.query(CacheInvalidation)\ + .filter(CacheInvalidation.cache_key == self.repo_name)\ + .filter(CacheInvalidation.cache_active == False)\ + .scalar() + + @property + def set_invalidate(self): + """ + set a cache for invalidation for this instance + """ + inv = Session.query(CacheInvalidation)\ + .filter(CacheInvalidation.cache_key == self.repo_name)\ + .scalar() + + if inv is None: + inv = CacheInvalidation(self.repo_name) + inv.cache_active = True + Session.add(inv) + Session.commit() + + @property + def scm_instance(self): + return self.__get_instance(self.repo_name) + + @property + def scm_instance_cached(self): + @cache_region('long_term') + def _c(repo_name): + return self.__get_instance(repo_name) + + inv = self.invalidate + if inv: + region_invalidate(_c, None, self.repo_name) + #update our cache + inv.cache_key.cache_active = True + Session.add(inv) + Session.commit() + + return _c(self.repo_name) + + def __get_instance(self, repo_name): + try: + alias = get_scm(self.repo_full_path)[0] + log.debug('Creating instance of %s repository', alias) + backend = get_backend(alias) + except VCSError: + log.error(traceback.format_exc()) + log.error('Perhaps this repository is in db and not in ' + 'filesystem run rescan repositories with ' + '"destroy old data " option from admin panel') + return + + if alias == 'hg': + repo = backend(self.repo_full_path, create=False, + baseui=self._ui) + #skip hidden web repository + if repo._get_hidden(): + return + else: + repo = backend(self.repo_full_path, create=False) + + return repo + class Group(Base): __tablename__ = 'groups' diff --git a/rhodecode/model/forms.py b/rhodecode/model/forms.py --- a/rhodecode/model/forms.py +++ b/rhodecode/model/forms.py @@ -280,6 +280,13 @@ def ValidRepoName(edit, old_data): return _ValidRepoName +def ValidForkName(): + class _ValidForkName(formencode.validators.FancyValidator): + def to_python(self, value, state): + return value + return _ValidForkName + + def SlugifyName(): class _SlugifyName(formencode.validators.FancyValidator): @@ -326,6 +333,7 @@ def ValidForkType(old_data): if old_data['repo_type'] != value: raise formencode.Invalid(_('Fork have to be the same ' 'type as original'), value, state) + return value return _ValidForkType @@ -583,6 +591,9 @@ def RepoForkForm(edit=False, old_data={} description = UnicodeString(strip=True, min=1, not_empty=True) private = StringBoolean(if_missing=False) repo_type = All(ValidForkType(old_data), OneOf(supported_backends)) + + chained_validators = [ValidForkName()] + return _RepoForkForm def RepoSettingsForm(edit=False, old_data={}): diff --git a/rhodecode/model/repo.py b/rhodecode/model/repo.py --- a/rhodecode/model/repo.py +++ b/rhodecode/model/repo.py @@ -70,28 +70,6 @@ class RepoModel(BaseModel): "get_repo_%s" % repo_name)) return repo.scalar() - def get_full(self, repo_name, cache=False, invalidate=False): - repo = self.sa.query(Repository)\ - .options(joinedload(Repository.fork))\ - .options(joinedload(Repository.user))\ - .options(joinedload(Repository.group))\ - .filter(Repository.repo_name == repo_name)\ - - if cache: - repo = repo.options(FromCache("sql_cache_long", - "get_repo_full_%s" % repo_name)) - if invalidate and cache: - repo.invalidate() - - ret = repo.scalar() - - #make transient for sake of errors - make_transient(ret) - for k in ['fork', 'user', 'group']: - attr = getattr(ret, k, False) - if attr: - make_transient(attr) - return ret def get_users_js(self): @@ -193,12 +171,13 @@ class RepoModel(BaseModel): raise def create(self, form_data, cur_user, just_db=False, fork=False): + try: if fork: #force str since hg doesn't go with unicode repo_name = str(form_data['fork_name']) org_name = str(form_data['repo_name']) - org_full_name = str(form_data['repo_name_full']) + org_full_name = org_name#str(form_data['fork_name_full']) else: org_name = repo_name = str(form_data['repo_name']) @@ -208,7 +187,10 @@ class RepoModel(BaseModel): new_repo.enable_statistics = False for k, v in form_data.items(): if k == 'repo_name': - v = repo_name_full + if fork: + v = repo_name + else: + v = repo_name_full if k == 'repo_group': k = 'group_id' @@ -216,7 +198,7 @@ class RepoModel(BaseModel): if fork: parent_repo = self.sa.query(Repository)\ - .filter(Repository.repo_name == org_full_name).scalar() + .filter(Repository.repo_name == org_full_name).one() new_repo.fork = parent_repo new_repo.user_id = cur_user.user_id diff --git a/rhodecode/model/scm.py b/rhodecode/model/scm.py --- a/rhodecode/model/scm.py +++ b/rhodecode/model/scm.py @@ -70,6 +70,75 @@ class RepoTemp(object): def __repr__(self): return "<%s('id:%s')>" % (self.__class__.__name__, self.repo_id) +class CachedRepoList(object): + + def __init__(self, db_repo_list, invalidation_list, repos_path, + order_by=None): + self.db_repo_list = db_repo_list + self.invalidation_list = invalidation_list + self.repos_path = repos_path + self.order_by = order_by + self.reversed = (order_by or '').startswith('-') + + def __len__(self): + return len(self.db_repo_list) + + def __repr__(self): + return '<%s (%s)>' % (self.__class__.__name__, self.__len__()) + + def __iter__(self): + for db_repo in self.db_repo_list: + dbr = db_repo + + # invalidate the repo cache if needed before getting the + # scm instance + + scm_invalidate = False + if self.invalidation_list is not None: + scm_invalidate = dbr.repo_name in self.invalidation_list + + if scm_invalidate: + log.info('invalidating cache for repository %s', + dbr.repo_name) + db_repo.set_invalidate + + scmr = db_repo.scm_instance_cached + + #check permission at this level + if not HasRepoPermissionAny('repository.read', + 'repository.write', + 'repository.admin')(dbr.repo_name, + 'get repo check'): + continue + + + + + + last_change = scmr.last_change + tip = h.get_changeset_safe(scmr, 'tip') + + tmp_d = {} + tmp_d['name'] = dbr.repo_name + tmp_d['name_sort'] = tmp_d['name'].lower() + tmp_d['description'] = dbr.description + tmp_d['description_sort'] = tmp_d['description'] + tmp_d['last_change'] = last_change + tmp_d['last_change_sort'] = time.mktime(last_change \ + .timetuple()) + tmp_d['tip'] = tip.raw_id + tmp_d['tip_sort'] = tip.revision + tmp_d['rev'] = tip.revision + tmp_d['contact'] = dbr.user.full_contact + tmp_d['contact_sort'] = tmp_d['contact'] + tmp_d['owner_sort'] = tmp_d['contact'] + tmp_d['repo_archives'] = list(scmr._get_archives()) + tmp_d['last_msg'] = tip.message + tmp_d['repo'] = scmr + tmp_d['dbrepo'] = dbr.get_dict() + tmp_d['dbrepo_fork'] = dbr.fork.get_dict() if dbr.fork \ + else {} + yield tmp_d class ScmModel(BaseModel): """Generic Scm Model @@ -118,7 +187,7 @@ class ScmModel(BaseModel): return repos_list - def get_repos(self, all_repos=None): + def get_repos(self, all_repos=None, sort_key=None): """ Get all repos from db and for each repo create it's backend instance and fill that backed with information from database @@ -127,120 +196,21 @@ class ScmModel(BaseModel): give specific repositories list, good for filtering """ if all_repos is None: - repos = self.sa.query(Repository)\ + all_repos = self.sa.query(Repository)\ .filter(Repository.group_id == None)\ .order_by(Repository.repo_name).all() - all_repos = [r.repo_name for r in repos] #get the repositories that should be invalidated invalidation_list = [str(x.cache_key) for x in \ self.sa.query(CacheInvalidation.cache_key)\ .filter(CacheInvalidation.cache_active == False)\ .all()] - for r_name in all_repos: - r_dbr = self.get(r_name, invalidation_list) - if r_dbr is not None: - repo, dbrepo = r_dbr - - if repo is None or dbrepo is None: - log.error('Repository "%s" looks somehow corrupted ' - 'fs-repo:%s,db-repo:%s both values should be ' - 'present', r_name, repo, dbrepo) - continue - last_change = repo.last_change - tip = h.get_changeset_safe(repo, 'tip') - - tmp_d = {} - tmp_d['name'] = dbrepo.repo_name - tmp_d['name_sort'] = tmp_d['name'].lower() - tmp_d['description'] = dbrepo.description - tmp_d['description_sort'] = tmp_d['description'] - tmp_d['last_change'] = last_change - tmp_d['last_change_sort'] = time.mktime(last_change \ - .timetuple()) - tmp_d['tip'] = tip.raw_id - tmp_d['tip_sort'] = tip.revision - tmp_d['rev'] = tip.revision - tmp_d['contact'] = dbrepo.user.full_contact - tmp_d['contact_sort'] = tmp_d['contact'] - tmp_d['owner_sort'] = tmp_d['contact'] - tmp_d['repo_archives'] = list(repo._get_archives()) - tmp_d['last_msg'] = tip.message - tmp_d['repo'] = repo - tmp_d['dbrepo'] = dbrepo.get_dict() - tmp_d['dbrepo_fork'] = dbrepo.fork.get_dict() if dbrepo.fork \ - else {} - yield tmp_d - - def get(self, repo_name, invalidation_list=None, retval='all'): - """Returns a tuple of Repository,DbRepository, - Get's repository from given name, creates BackendInstance and - propagates it's data from database with all additional information - - :param repo_name: - :param invalidation_list: if a invalidation list is given the get - method should not manually check if this repository needs - invalidation and just invalidate the repositories in list - :param retval: string specifing what to return one of 'repo','dbrepo', - 'all'if repo or dbrepo is given it'll just lazy load chosen type - and return None as the second - """ - if not HasRepoPermissionAny('repository.read', 'repository.write', - 'repository.admin')(repo_name, 'get repo check'): - return - #====================================================================== - # CACHE FUNCTION - #====================================================================== - @cache_region('long_term') - def _get_repo(repo_name): - - repo_path = os.path.join(self.repos_path, repo_name) - - try: - alias = get_scm(repo_path)[0] - log.debug('Creating instance of %s repository', alias) - backend = get_backend(alias) - except VCSError: - log.error(traceback.format_exc()) - log.error('Perhaps this repository is in db and not in ' - 'filesystem run rescan repositories with ' - '"destroy old data " option from admin panel') - return + repo_iter = CachedRepoList(all_repos, invalidation_list, + repos_path=self.repos_path, + order_by=sort_key) - if alias == 'hg': - repo = backend(repo_path, create=False, baseui=make_ui('db')) - #skip hidden web repository - if repo._get_hidden(): - return - else: - repo = backend(repo_path, create=False) - - return repo - - pre_invalidate = True - dbinvalidate = False - - if invalidation_list is not None: - pre_invalidate = repo_name in invalidation_list - - if pre_invalidate: - #this returns object to invalidate - invalidate = self._should_invalidate(repo_name) - if invalidate: - log.info('invalidating cache for repository %s', repo_name) - region_invalidate(_get_repo, None, repo_name) - self._mark_invalidated(invalidate) - dbinvalidate = True - - r, dbr = None, None - if retval == 'repo' or 'all': - r = _get_repo(repo_name) - if retval == 'dbrepo' or 'all': - dbr = RepoModel().get_full(repo_name, cache=True, - invalidate=dbinvalidate) - - return r, dbr + return repo_iter def mark_for_invalidation(self, repo_name): """Puts cache invalidation task into db for diff --git a/rhodecode/templates/admin/users/user_edit.html b/rhodecode/templates/admin/users/user_edit.html --- a/rhodecode/templates/admin/users/user_edit.html +++ b/rhodecode/templates/admin/users/user_edit.html @@ -65,7 +65,7 @@
- ${h.password('new_password',class_='medium')} + ${h.password('new_password',class_='medium',autocomplete="off")}
diff --git a/rhodecode/templates/admin/users/user_edit_my_account.html b/rhodecode/templates/admin/users/user_edit_my_account.html --- a/rhodecode/templates/admin/users/user_edit_my_account.html +++ b/rhodecode/templates/admin/users/user_edit_my_account.html @@ -54,7 +54,7 @@
- ${h.password('new_password',class_="medium")} + ${h.password('new_password',class_="medium",autocomplete="off")}
diff --git a/rhodecode/tests/__init__.py b/rhodecode/tests/__init__.py --- a/rhodecode/tests/__init__.py +++ b/rhodecode/tests/__init__.py @@ -7,6 +7,9 @@ command. This module initializes the application via ``websetup`` (`paster setup-app`) and provides the base testing objects. """ +import os +from os.path import join as jn + from unittest import TestCase from paste.deploy import loadapp @@ -14,7 +17,7 @@ from paste.script.appinstall import Setu from pylons import config, url from routes.util import URLGenerator from webtest import TestApp -import os + from rhodecode.model import meta import logging @@ -35,7 +38,7 @@ import pylons.test environ = {} #SOME GLOBALS FOR TESTS -TESTS_TMP_PATH = '/tmp' +TESTS_TMP_PATH = jn('/', 'tmp') HG_REPO = 'vcs_test_hg' GIT_REPO = 'vcs_test_git' @@ -64,8 +67,8 @@ class TestController(TestCase): 'password':password}) if 'invalid user name' in response.body: - assert False, 'could not login using %s %s' % (username, password) + self.fail('could not login using %s %s' % (username, password)) - assert response.status == '302 Found', 'Wrong response code from login got %s' % response.status - assert response.session['rhodecode_user'].username == username, 'wrong logged in user got %s expected %s' % (response.session['rhodecode_user'].username, username) + self.assertEqual(response.status, '302 Found') + self.assertEqual(response.session['rhodecode_user'].username, username) return response.follow() diff --git a/rhodecode/tests/functional/test_admin_repos.py b/rhodecode/tests/functional/test_admin_repos.py --- a/rhodecode/tests/functional/test_admin_repos.py +++ b/rhodecode/tests/functional/test_admin_repos.py @@ -6,6 +6,11 @@ from rhodecode.tests import * class TestAdminReposController(TestController): + + def __make_repo(self): + pass + + def test_index(self): self.log_user() response = self.app.get(url('repos')) @@ -21,31 +26,39 @@ class TestAdminReposController(TestContr private = False response = self.app.post(url('repos'), {'repo_name':repo_name, 'repo_type':'hg', + 'clone_uri':'', + 'repo_group':'', 'description':description, 'private':private}) - + self.assertTrue('flash' in response.session) #test if we have a message for that repository - assert '''created repository %s''' % (repo_name) in response.session['flash'][0], 'No flash message about new repo' + self.assertTrue('''created repository %s''' % (repo_name) in + response.session['flash'][0]) - #test if the fork was created in the database - new_repo = self.sa.query(Repository).filter(Repository.repo_name == repo_name).one() + #test if the repo was created in the database + new_repo = self.sa.query(Repository).filter(Repository.repo_name == + repo_name).one() - assert new_repo.repo_name == repo_name, 'wrong name of repo name in db' - assert new_repo.description == description, 'wrong description' + self.assertEqual(new_repo.repo_name, repo_name) + self.assertEqual(new_repo.description, description) #test if repository is visible in the list ? response = response.follow() - assert repo_name in response.body, 'missing new repo from the main repos list' + self.assertTrue(repo_name in response.body) #test if repository was created on filesystem try: vcs.get_repo(os.path.join(TESTS_TMP_PATH, repo_name)) except: - assert False , 'no repo in filesystem' + self.fail('no repo in filesystem') + + def test_create_hg_in_group(self): + #TODO: write test ! + pass def test_create_git(self): return @@ -55,6 +68,8 @@ class TestAdminReposController(TestContr private = False response = self.app.post(url('repos'), {'repo_name':repo_name, 'repo_type':'git', + 'clone_uri':'', + 'repo_group':'', 'description':description, 'private':private}) @@ -90,58 +105,74 @@ class TestAdminReposController(TestContr response = self.app.put(url('repo', repo_name=HG_REPO)) def test_update_browser_fakeout(self): - response = self.app.post(url('repo', repo_name=HG_REPO), params=dict(_method='put')) + response = self.app.post(url('repo', repo_name=HG_REPO), + params=dict(_method='put')) def test_delete(self): self.log_user() repo_name = 'vcs_test_new_to_delete' description = 'description for newly created repo' private = False + response = self.app.post(url('repos'), {'repo_name':repo_name, 'repo_type':'hg', - 'description':description, - 'private':private}) - + 'clone_uri':'', + 'repo_group':'', + 'description':description, + 'private':private}) + self.assertTrue('flash' in response.session) #test if we have a message for that repository - assert '''created repository %s''' % (repo_name) in response.session['flash'][0], 'No flash message about new repo' + self.assertTrue('''created repository %s''' % (repo_name) in + response.session['flash'][0]) #test if the repo was created in the database - new_repo = self.sa.query(Repository).filter(Repository.repo_name == repo_name).one() + new_repo = self.sa.query(Repository).filter(Repository.repo_name == + repo_name).one() - assert new_repo.repo_name == repo_name, 'wrong name of repo name in db' - assert new_repo.description == description, 'wrong description' + self.assertEqual(new_repo.repo_name, repo_name) + self.assertEqual(new_repo.description, description) #test if repository is visible in the list ? response = response.follow() - assert repo_name in response.body, 'missing new repo from the main repos list' + self.assertTrue(repo_name in response.body) response = self.app.delete(url('repo', repo_name=repo_name)) - assert '''deleted repository %s''' % (repo_name) in response.session['flash'][0], 'No flash message about delete repo' + self.assertTrue('''deleted repository %s''' % (repo_name) in + response.session['flash'][0]) response.follow() #check if repo was deleted from db - deleted_repo = self.sa.query(Repository).filter(Repository.repo_name == repo_name).scalar() + deleted_repo = self.sa.query(Repository).filter(Repository.repo_name + == repo_name).scalar() + + self.assertEqual(deleted_repo, None) - assert deleted_repo is None, 'Deleted repository was found in db' + + def test_delete_repo_with_group(self): + #TODO: + pass def test_delete_browser_fakeout(self): - response = self.app.post(url('repo', repo_name=HG_REPO), params=dict(_method='delete')) + response = self.app.post(url('repo', repo_name=HG_REPO), + params=dict(_method='delete')) def test_show(self): self.log_user() response = self.app.get(url('repo', repo_name=HG_REPO)) def test_show_as_xml(self): - response = self.app.get(url('formatted_repo', repo_name=HG_REPO, format='xml')) + response = self.app.get(url('formatted_repo', repo_name=HG_REPO, + format='xml')) def test_edit(self): response = self.app.get(url('edit_repo', repo_name=HG_REPO)) def test_edit_as_xml(self): - response = self.app.get(url('formatted_edit_repo', repo_name=HG_REPO, format='xml')) + response = self.app.get(url('formatted_edit_repo', repo_name=HG_REPO, + format='xml')) diff --git a/rhodecode/tests/functional/test_admin_settings.py b/rhodecode/tests/functional/test_admin_settings.py --- a/rhodecode/tests/functional/test_admin_settings.py +++ b/rhodecode/tests/functional/test_admin_settings.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + from rhodecode.lib.auth import get_crypt_password, check_password from rhodecode.model.db import User, RhodeCodeSettings from rhodecode.tests import * @@ -42,7 +44,8 @@ class TestAdminSettingsController(TestCo response = self.app.get(url('admin_edit_setting', setting_id=1)) def test_edit_as_xml(self): - response = self.app.get(url('formatted_admin_edit_setting', setting_id=1, format='xml')) + response = self.app.get(url('formatted_admin_edit_setting', + setting_id=1, format='xml')) def test_ga_code_active(self): @@ -58,11 +61,14 @@ class TestAdminSettingsController(TestCo rhodecode_ga_code=new_ga_code )) - assert 'Updated application settings' in response.session['flash'][0][1], 'no flash message about success of change' - assert RhodeCodeSettings.get_app_settings()['rhodecode_ga_code'] == new_ga_code, 'change not in database' + self.assertTrue('Updated application settings' in + response.session['flash'][0][1]) + self.assertEqual(RhodeCodeSettings + .get_app_settings()['rhodecode_ga_code'], new_ga_code) response = response.follow() - assert """_gaq.push(['_setAccount', '%s']);""" % new_ga_code in response.body + self.assertTrue("""_gaq.push(['_setAccount', '%s']);""" % new_ga_code + in response.body) def test_ga_code_inactive(self): self.log_user() @@ -77,11 +83,14 @@ class TestAdminSettingsController(TestCo rhodecode_ga_code=new_ga_code )) - assert 'Updated application settings' in response.session['flash'][0][1], 'no flash message about success of change' - assert RhodeCodeSettings.get_app_settings()['rhodecode_ga_code'] == new_ga_code, 'change not in database' + self.assertTrue('Updated application settings' in + response.session['flash'][0][1]) + self.assertEqual(RhodeCodeSettings + .get_app_settings()['rhodecode_ga_code'], new_ga_code) response = response.follow() - assert """_gaq.push(['_setAccount', '%s']);""" % new_ga_code not in response.body + self.assertTrue("""_gaq.push(['_setAccount', '%s']);""" % new_ga_code + not in response.body) def test_title_change(self): @@ -89,27 +98,33 @@ class TestAdminSettingsController(TestCo old_title = 'RhodeCode' new_title = old_title + '_changed' old_realm = 'RhodeCode authentication' - response = self.app.post(url('admin_setting', setting_id='global'), - params=dict( - _method='put', - rhodecode_title=new_title, - rhodecode_realm=old_realm, - rhodecode_ga_code='' - )) + + for new_title in ['Changed', 'Żółwik', old_title]: + response = self.app.post(url('admin_setting', setting_id='global'), + params=dict( + _method='put', + rhodecode_title=new_title, + rhodecode_realm=old_realm, + rhodecode_ga_code='' + )) - assert 'Updated application settings' in response.session['flash'][0][1], 'no flash message about success of change' - assert RhodeCodeSettings.get_app_settings()['rhodecode_title'] == new_title, 'change not in database' + self.assertTrue('Updated application settings' in + response.session['flash'][0][1]) + self.assertEqual(RhodeCodeSettings + .get_app_settings()['rhodecode_title'], + new_title.decode('utf-8')) - response = response.follow() - assert """

%s

""" % new_title in response.body + response = response.follow() + self.assertTrue("""

%s

""" % new_title + in response.body) def test_my_account(self): self.log_user() response = self.app.get(url('admin_settings_my_account')) - print response - assert 'value="test_admin' in response.body + + self.assertTrue('value="test_admin' in response.body) def test_my_account_update(self): self.log_user() @@ -120,14 +135,14 @@ class TestAdminSettingsController(TestCo new_password = 'test123' - response = self.app.post(url('admin_settings_my_account_update'), params=dict( - _method='put', - username='test_admin', - new_password=new_password, - password='', - name=new_name, - lastname=new_lastname, - email=new_email,)) + response = self.app.post(url('admin_settings_my_account_update'), + params=dict(_method='put', + username='test_admin', + new_password=new_password, + password='', + name=new_name, + lastname=new_lastname, + email=new_email,)) response.follow() assert 'Your account was updated successfully' in response.session['flash'][0][1], 'no flash message about success of change' diff --git a/rhodecode/tests/functional/test_login.py b/rhodecode/tests/functional/test_login.py --- a/rhodecode/tests/functional/test_login.py +++ b/rhodecode/tests/functional/test_login.py @@ -45,11 +45,11 @@ class TestLoginController(TestController def test_login_short_password(self): response = self.app.post(url(controller='login', action='index'), - {'username':'error', - 'password':'test'}) - assert response.status == '200 OK', 'Wrong response from login page' + {'username':'test_admin', + 'password':'as'}) + self.assertEqual(response.status, '200 OK') print response.body - assert 'Enter 6 characters or more' in response.body, 'No error password message in response' + self.assertTrue('Enter 3 characters or more' in response.body) def test_login_wrong_username_password(self): response = self.app.post(url(controller='login', action='index'), diff --git a/rhodecode/tests/functional/test_summary.py b/rhodecode/tests/functional/test_summary.py --- a/rhodecode/tests/functional/test_summary.py +++ b/rhodecode/tests/functional/test_summary.py @@ -6,22 +6,38 @@ class TestSummaryController(TestControll def test_index(self): self.log_user() - response = self.app.get(url(controller='summary', action='index', repo_name=HG_REPO)) + response = self.app.get(url(controller='summary', + action='index', repo_name=HG_REPO)) #repo type - assert """Mercurial repository""" in response.body - assert """public repository""" in response.body + self.assertTrue("""Mercurial """ + in response.body) + self.assertTrue("""public """ + in response.body) #codes stats + self._enable_stats() - self._enable_stats() invalidate_cache('get_repo_cached_%s' % HG_REPO) - response = self.app.get(url(controller='summary', action='index', repo_name=HG_REPO)) - assert """var data = {"Python": 42, "Rst": 11, "Bash": 2, "Makefile": 1, "Batch": 1, "Ini": 1, "Css": 1};""" in response.body, 'wrong info about % of codes stats' + response = self.app.get(url(controller='summary', action='index', + repo_name=HG_REPO)) + + self.assertTrue("""var data = {"py": {"count": 42, "desc": """ + """["Python"]}, "rst": {"count": 11, "desc": """ + """["Rst"]}, "sh": {"count": 2, "desc": ["Bash"]}, """ + """"makefile": {"count": 1, "desc": ["Makefile", """ + """"Makefile"]}, "cfg": {"count": 1, "desc": ["Ini"]},""" + """ "css": {"count": 1, "desc": ["Css"]}, "bat": """ + """{"count": 1, "desc": ["Batch"]}};""" + in response.body) # clone url... - assert """""" % HG_REPO in response.body + self.assertTrue("""""" % HG_REPO in response.body) def _enable_stats(self): diff --git a/test.ini b/test.ini --- a/test.ini +++ b/test.ini @@ -7,6 +7,7 @@ [DEFAULT] debug = true +pdebug = false ################################################################################ ## Uncomment and replace with the address which should receive ## ## any error reports after application crash ##