diff --git a/rhodecode/__init__.py b/rhodecode/__init__.py --- a/rhodecode/__init__.py +++ b/rhodecode/__init__.py @@ -45,7 +45,7 @@ PYRAMID_SETTINGS = {} EXTENSIONS = {} __version__ = ('.'.join((str(each) for each in VERSION[:3]))) -__dbversion__ = 95 # defines current db version for migrations +__dbversion__ = 97 # defines current db version for migrations __platform__ = platform.system() __license__ = 'AGPLv3, and Commercial License' __author__ = 'RhodeCode GmbH' diff --git a/rhodecode/apps/admin/__init__.py b/rhodecode/apps/admin/__init__.py --- a/rhodecode/apps/admin/__init__.py +++ b/rhodecode/apps/admin/__init__.py @@ -424,6 +424,10 @@ def admin_routes(config): pattern='/repo_groups') config.add_route( + name='repo_groups_data', + pattern='/repo_groups_data') + + config.add_route( name='repo_group_new', pattern='/repo_group/new') diff --git a/rhodecode/apps/admin/tests/test_admin_repository_groups.py b/rhodecode/apps/admin/tests/test_admin_repository_groups.py --- a/rhodecode/apps/admin/tests/test_admin_repository_groups.py +++ b/rhodecode/apps/admin/tests/test_admin_repository_groups.py @@ -23,11 +23,11 @@ import pytest from rhodecode.apps._base import ADMIN_PREFIX from rhodecode.lib import helpers as h -from rhodecode.model.db import Repository, UserRepoToPerm, User +from rhodecode.model.db import Repository, UserRepoToPerm, User, RepoGroup from rhodecode.model.meta import Session from rhodecode.model.repo_group import RepoGroupModel from rhodecode.tests import ( - assert_session_flash, TEST_USER_REGULAR_LOGIN, TESTS_TMP_PATH, TestController) + assert_session_flash, TEST_USER_REGULAR_LOGIN, TESTS_TMP_PATH) from rhodecode.tests.fixture import Fixture fixture = Fixture() @@ -38,6 +38,7 @@ def route_path(name, params=None, **kwar base_url = { 'repo_groups': ADMIN_PREFIX + '/repo_groups', + 'repo_groups_data': ADMIN_PREFIX + '/repo_groups_data', 'repo_group_new': ADMIN_PREFIX + '/repo_group/new', 'repo_group_create': ADMIN_PREFIX + '/repo_group/create', @@ -59,13 +60,30 @@ def _get_permission_for_user(user, repo) @pytest.mark.usefixtures("app") class TestAdminRepositoryGroups(object): + def test_show_repo_groups(self, autologin_user): - response = self.app.get(route_path('repo_groups')) - response.mustcontain('data: []') + self.app.get(route_path('repo_groups')) + + def test_show_repo_groups_data(self, autologin_user, xhr_header): + response = self.app.get(route_path( + 'repo_groups_data'), extra_environ=xhr_header) + + all_repo_groups = RepoGroup.query().count() + assert response.json['recordsTotal'] == all_repo_groups - def test_show_repo_groups_after_creating_group(self, autologin_user): + def test_show_repo_groups_data_filtered(self, autologin_user, xhr_header): + response = self.app.get(route_path( + 'repo_groups_data', params={'search[value]': 'empty_search'}), + extra_environ=xhr_header) + + all_repo_groups = RepoGroup.query().count() + assert response.json['recordsTotal'] == all_repo_groups + assert response.json['recordsFiltered'] == 0 + + def test_show_repo_groups_after_creating_group(self, autologin_user, xhr_header): fixture.create_repo_group('test_repo_group') - response = self.app.get(route_path('repo_groups')) + response = self.app.get(route_path( + 'repo_groups_data'), extra_environ=xhr_header) response.mustcontain('"name_raw": "test_repo_group"') fixture.destroy_repo_group('test_repo_group') diff --git a/rhodecode/apps/admin/views/repo_groups.py b/rhodecode/apps/admin/views/repo_groups.py --- a/rhodecode/apps/admin/views/repo_groups.py +++ b/rhodecode/apps/admin/views/repo_groups.py @@ -17,7 +17,7 @@ # This program is dual-licensed. If you wish to learn more about the # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ - +import datetime import logging import formencode import formencode.htmlfill @@ -30,16 +30,16 @@ from pyramid.response import Response from rhodecode import events from rhodecode.apps._base import BaseAppView, DataGridAppView -from rhodecode.lib.ext_json import json from rhodecode.lib.auth import ( LoginRequired, CSRFRequired, NotAnonymous, HasPermissionAny, HasRepoGroupPermissionAny) from rhodecode.lib import helpers as h, audit_logger -from rhodecode.lib.utils2 import safe_int, safe_unicode +from rhodecode.lib.utils2 import safe_int, safe_unicode, datetime_to_time from rhodecode.model.forms import RepoGroupForm from rhodecode.model.repo_group import RepoGroupModel from rhodecode.model.scm import RepoGroupList -from rhodecode.model.db import Session, RepoGroup +from rhodecode.model.db import ( + or_, count, func, in_filter_generator, Session, RepoGroup, User, Repository) log = logging.getLogger(__name__) @@ -88,22 +88,168 @@ class AdminRepoGroupsView(BaseAppView, D return False return False + # permission check in data loading of + # `repo_group_list_data` via RepoGroupList @LoginRequired() @NotAnonymous() - # perms check inside @view_config( route_name='repo_groups', request_method='GET', renderer='rhodecode:templates/admin/repo_groups/repo_groups.mako') def repo_group_list(self): c = self.load_default_context() + return self._get_template_context(c) - repo_group_list = RepoGroup.get_all_repo_groups() - repo_group_list_acl = RepoGroupList( - repo_group_list, perm_set=['group.admin']) - repo_group_data = RepoGroupModel().get_repo_groups_as_dict( - repo_group_list=repo_group_list_acl, admin=True) - c.data = json.dumps(repo_group_data) - return self._get_template_context(c) + # permission check inside + @LoginRequired() + @NotAnonymous() + @view_config( + route_name='repo_groups_data', request_method='GET', + renderer='json_ext', xhr=True) + def repo_group_list_data(self): + self.load_default_context() + column_map = { + 'name_raw': 'group_name_hash', + 'desc': 'group_description', + 'last_change_raw': 'updated_on', + 'top_level_repos': 'repos_total', + 'owner': 'user_username', + } + draw, start, limit = self._extract_chunk(self.request) + search_q, order_by, order_dir = self._extract_ordering( + self.request, column_map=column_map) + + _render = self.request.get_partial_renderer( + 'rhodecode:templates/data_table/_dt_elements.mako') + c = _render.get_call_context() + + def quick_menu(repo_group_name): + return _render('quick_repo_group_menu', repo_group_name) + + def repo_group_lnk(repo_group_name): + return _render('repo_group_name', repo_group_name) + + def last_change(last_change): + if isinstance(last_change, datetime.datetime) and not last_change.tzinfo: + delta = datetime.timedelta( + seconds=(datetime.datetime.now() - datetime.datetime.utcnow()).seconds) + last_change = last_change + delta + return _render("last_change", last_change) + + def desc(desc, personal): + return _render( + 'repo_group_desc', desc, personal, c.visual.stylify_metatags) + + def repo_group_actions(repo_group_id, repo_group_name, gr_count): + return _render( + 'repo_group_actions', repo_group_id, repo_group_name, gr_count) + + def user_profile(username): + return _render('user_profile', username) + + auth_repo_group_list = RepoGroupList( + RepoGroup.query().all(), perm_set=['group.admin']) + + allowed_ids = [-1] + for repo_group in auth_repo_group_list: + allowed_ids.append(repo_group.group_id) + + repo_groups_data_total_count = RepoGroup.query()\ + .filter(or_( + # generate multiple IN to fix limitation problems + *in_filter_generator(RepoGroup.group_id, allowed_ids) + )) \ + .count() + + repo_groups_data_total_inactive_count = RepoGroup.query()\ + .filter(RepoGroup.group_id.in_(allowed_ids))\ + .count() + + repo_count = count(Repository.repo_id) + base_q = Session.query( + RepoGroup.group_name, + RepoGroup.group_name_hash, + RepoGroup.group_description, + RepoGroup.group_id, + RepoGroup.personal, + RepoGroup.updated_on, + User, + repo_count.label('repos_count') + ) \ + .filter(or_( + # generate multiple IN to fix limitation problems + *in_filter_generator(RepoGroup.group_id, allowed_ids) + )) \ + .outerjoin(Repository) \ + .join(User, User.user_id == RepoGroup.user_id) \ + .group_by(RepoGroup, User) + + if search_q: + like_expression = u'%{}%'.format(safe_unicode(search_q)) + base_q = base_q.filter(or_( + RepoGroup.group_name.ilike(like_expression), + )) + + repo_groups_data_total_filtered_count = base_q.count() + # the inactive isn't really used, but we still make it same as other data grids + # which use inactive (users,user groups) + repo_groups_data_total_filtered_inactive_count = repo_groups_data_total_filtered_count + + sort_defined = False + if order_by == 'group_name': + sort_col = func.lower(RepoGroup.group_name) + sort_defined = True + elif order_by == 'repos_total': + sort_col = repo_count + sort_defined = True + elif order_by == 'user_username': + sort_col = User.username + else: + sort_col = getattr(RepoGroup, order_by, None) + + if sort_defined or sort_col: + if order_dir == 'asc': + sort_col = sort_col.asc() + else: + sort_col = sort_col.desc() + + base_q = base_q.order_by(sort_col) + base_q = base_q.offset(start).limit(limit) + + # authenticated access to user groups + auth_repo_group_list = base_q.all() + + repo_groups_data = [] + for repo_gr in auth_repo_group_list: + row = { + "menu": quick_menu(repo_gr.group_name), + "name": repo_group_lnk(repo_gr.group_name), + "name_raw": repo_gr.group_name, + "last_change": last_change(repo_gr.updated_on), + "last_change_raw": datetime_to_time(repo_gr.updated_on), + + "last_changeset": "", + "last_changeset_raw": "", + + "desc": desc(repo_gr.group_description, repo_gr.personal), + "owner": user_profile(repo_gr.User.username), + "top_level_repos": repo_gr.repos_count, + "action": repo_group_actions( + repo_gr.group_id, repo_gr.group_name, repo_gr.repos_count), + + } + + repo_groups_data.append(row) + + data = ({ + 'draw': draw, + 'data': repo_groups_data, + 'recordsTotal': repo_groups_data_total_count, + 'recordsTotalInactive': repo_groups_data_total_inactive_count, + 'recordsFiltered': repo_groups_data_total_filtered_count, + 'recordsFilteredInactive': repo_groups_data_total_filtered_inactive_count, + }) + + return data @LoginRequired() @NotAnonymous() diff --git a/rhodecode/lib/dbmigrate/versions/096_version_4_17_0.py b/rhodecode/lib/dbmigrate/versions/096_version_4_17_0.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/dbmigrate/versions/096_version_4_17_0.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- + +import logging + +from alembic.migration import MigrationContext +from alembic.operations import Operations +from sqlalchemy import String, Column +from sqlalchemy.sql import text + +from rhodecode.lib.dbmigrate.versions import _reset_base +from rhodecode.model import meta, init_model_encryption +from rhodecode.model.db import RepoGroup + + +log = logging.getLogger(__name__) + + +def upgrade(migrate_engine): + """ + Upgrade operations go here. + Don't create your own engine; bind migrate_engine to your metadata + """ + _reset_base(migrate_engine) + from rhodecode.lib.dbmigrate.schema import db_4_16_0_2 + + init_model_encryption(db_4_16_0_2) + + context = MigrationContext.configure(migrate_engine.connect()) + op = Operations(context) + + repo_group = db_4_16_0_2.RepoGroup.__table__ + + with op.batch_alter_table(repo_group.name) as batch_op: + batch_op.add_column( + Column("repo_group_name_hash", String(1024), nullable=True, unique=False)) + + _generate_repo_group_name_hashes(db_4_16_0_2, op, meta.Session) + + +def downgrade(migrate_engine): + pass + + +def _generate_repo_group_name_hashes(models, op, session): + repo_groups = models.RepoGroup.get_all() + for repo_group in repo_groups: + print(repo_group.group_name) + hash_ = RepoGroup.hash_repo_group_name(repo_group.group_name) + params = {'hash': hash_, 'id': repo_group.group_id} + query = text( + 'UPDATE groups SET repo_group_name_hash = :hash' + ' WHERE group_id = :id').bindparams(**params) + op.execute(query) + session().commit() diff --git a/rhodecode/lib/dbmigrate/versions/097_version_4_17_0.py b/rhodecode/lib/dbmigrate/versions/097_version_4_17_0.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/dbmigrate/versions/097_version_4_17_0.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +import logging + +from alembic.migration import MigrationContext +from alembic.operations import Operations + +from rhodecode.lib.dbmigrate.versions import _reset_base +from rhodecode.model import init_model_encryption + + +log = logging.getLogger(__name__) + + +def upgrade(migrate_engine): + """ + Upgrade operations go here. + Don't create your own engine; bind migrate_engine to your metadata + """ + _reset_base(migrate_engine) + from rhodecode.lib.dbmigrate.schema import db_4_16_0_2 + + init_model_encryption(db_4_16_0_2) + + context = MigrationContext.configure(migrate_engine.connect()) + op = Operations(context) + + repo_group = db_4_16_0_2.RepoGroup.__table__ + + with op.batch_alter_table(repo_group.name) as batch_op: + batch_op.alter_column("repo_group_name_hash", nullable=False) + + +def downgrade(migrate_engine): + pass + + +def _generate_repo_group_name_hashes(models, op, session): + pass diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py --- a/rhodecode/model/db.py +++ b/rhodecode/model/db.py @@ -25,6 +25,7 @@ Database Models for RhodeCode Enterprise import re import os import time +import string import hashlib import logging import datetime @@ -50,6 +51,7 @@ from sqlalchemy.dialects.mysql import LO from zope.cachedescriptors.property import Lazy as LazyProperty from pyramid import compat from pyramid.threadlocal import get_current_request +from webhelpers.text import collapse, remove_formatting from rhodecode.translation import _ from rhodecode.lib.vcs import get_vcs_instance @@ -2469,7 +2471,8 @@ class RepoGroup(Base, BaseModel): CHOICES_SEPARATOR = '/' # used to generate select2 choices for nested groups group_id = Column("group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) - group_name = Column("group_name", String(255), nullable=False, unique=True, default=None) + _group_name = Column("group_name", String(255), nullable=False, unique=True, default=None) + group_name_hash = Column("repo_group_name_hash", String(1024), nullable=False, unique=False) group_parent_id = Column("group_parent_id", Integer(), ForeignKey('groups.group_id'), nullable=True, unique=None, default=None) group_description = Column("group_description", String(10000), nullable=True, unique=None, default=None) enable_locking = Column("enable_locking", Boolean(), nullable=False, unique=None, default=False) @@ -2492,6 +2495,15 @@ class RepoGroup(Base, BaseModel): return u"<%s('id:%s:%s')>" % ( self.__class__.__name__, self.group_id, self.group_name) + @hybrid_property + def group_name(self): + return self._group_name + + @group_name.setter + def group_name(self, value): + self._group_name = value + self.group_name_hash = self.hash_repo_group_name(value) + @validates('group_parent_id') def validate_group_parent_id(self, key, val): """ @@ -2508,6 +2520,18 @@ class RepoGroup(Base, BaseModel): return h.escape(self.group_description) @classmethod + def hash_repo_group_name(cls, repo_group_name): + val = remove_formatting(repo_group_name) + val = safe_str(val).lower() + chars = [] + for c in val: + if c not in string.ascii_letters: + c = str(ord(c)) + chars.append(c) + + return ''.join(chars) + + @classmethod def _generate_choice(cls, repo_group): from webhelpers.html import literal as _literal _name = lambda k: _literal(cls.CHOICES_SEPARATOR.join(k)) @@ -2770,6 +2794,13 @@ class RepoGroup(Base, BaseModel): } return data + def get_dict(self): + # Since we transformed `group_name` to a hybrid property, we need to + # keep compatibility with the code which uses `group_name` field. + result = super(RepoGroup, self).get_dict() + result['group_name'] = result.pop('_group_name', None) + return result + class Permission(Base, BaseModel): __tablename__ = 'permissions' diff --git a/rhodecode/templates/admin/repo_groups/repo_groups.mako b/rhodecode/templates/admin/repo_groups/repo_groups.mako --- a/rhodecode/templates/admin/repo_groups/repo_groups.mako +++ b/rhodecode/templates/admin/repo_groups/repo_groups.mako @@ -10,7 +10,7 @@ <%def name="breadcrumbs_links()"> - ${h.link_to(_('Admin'),h.route_path('admin_home'))} » 0 ${_('repository groups')} + ${h.link_to(_('Admin'),h.route_path('admin_home'))} » 0 <%def name="menu_bar_nav()"> @@ -36,15 +36,32 @@ - // refilter table if page load via back button - $("#q_filter").trigger('keyup'); -}); - diff --git a/rhodecode/templates/admin/repos/repos.mako b/rhodecode/templates/admin/repos/repos.mako --- a/rhodecode/templates/admin/repos/repos.mako +++ b/rhodecode/templates/admin/repos/repos.mako @@ -66,7 +66,7 @@ { data: {"_": "state", "sort": "state"}, title: "${_('State')}", className: "td-tags td-state" }, { data: {"_": "action", - "sort": "action"}, title: "${_('Action')}", className: "td-action" } + "sort": "action"}, title: "${_('Action')}", className: "td-action", orderable: false } ], language: { paginate: DEFAULT_GRID_PAGINATION, diff --git a/rhodecode/tests/functional/test_delegated_admin.py b/rhodecode/tests/functional/test_delegated_admin.py --- a/rhodecode/tests/functional/test_delegated_admin.py +++ b/rhodecode/tests/functional/test_delegated_admin.py @@ -34,6 +34,8 @@ def route_path(name, params=None, **kwar ADMIN_PREFIX + '/repos', 'repo_groups': ADMIN_PREFIX + '/repo_groups', + 'repo_groups_data': + ADMIN_PREFIX + '/repo_groups_data', 'user_groups': ADMIN_PREFIX + '/user_groups', 'user_groups_data': @@ -67,8 +69,9 @@ class TestAdminDelegatedUser(TestControl response = self.app.get(route_path('repos'), status=200) response.mustcontain('data: []') - response = self.app.get(route_path('repo_groups'), status=200) - response.mustcontain('data: []') + response = self.app.get(route_path('repo_groups_data'), + status=200, extra_environ=xhr_header) + assert response.json['data'] == [] response = self.app.get(route_path('user_groups_data'), status=200, extra_environ=xhr_header) @@ -102,7 +105,8 @@ class TestAdminDelegatedUser(TestControl response = self.app.get(route_path('repos'), status=200) response.mustcontain('"name_raw": "{}"'.format(repo_name)) - response = self.app.get(route_path('repo_groups'), status=200) + response = self.app.get(route_path('repo_groups_data'), + extra_environ=xhr_header, status=200) response.mustcontain('"name_raw": "{}"'.format(repo_group_name)) response = self.app.get(route_path('user_groups_data'), @@ -144,7 +148,8 @@ class TestAdminDelegatedUser(TestControl response = self.app.get(route_path('repos'), status=200) response.mustcontain('"name_raw": "{}"'.format(repo_name)) - response = self.app.get(route_path('repo_groups'), status=200) + response = self.app.get(route_path('repo_groups_data'), + extra_environ=xhr_header, status=200) response.mustcontain('"name_raw": "{}"'.format(repo_group_name)) response = self.app.get(route_path('user_groups_data'),