repository-groups: use lazy loaded admin dashboard

rhodecode/lib/dbmigrate/versions/096_version_4_17_0.py

@@ -0,0 +1,54 b''
+# -*- coding: utf-8 -*-
+
+import logging
+
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from sqlalchemy import String, Column
+from sqlalchemy.sql import text
+
+from rhodecode.lib.dbmigrate.versions import _reset_base
+from rhodecode.model import meta, init_model_encryption
+from rhodecode.model.db import RepoGroup
+
+
+log = logging.getLogger(__name__)
+
+
+def upgrade(migrate_engine):
+    """
+    Upgrade operations go here.
+    Don't create your own engine; bind migrate_engine to your metadata
+    """
+    _reset_base(migrate_engine)
+    from rhodecode.lib.dbmigrate.schema import db_4_16_0_2
+
+    init_model_encryption(db_4_16_0_2)
+
+    context = MigrationContext.configure(migrate_engine.connect())
+    op = Operations(context)
+
+    repo_group = db_4_16_0_2.RepoGroup.__table__
+    
+    with op.batch_alter_table(repo_group.name) as batch_op:
+        batch_op.add_column(
+            Column("repo_group_name_hash", String(1024), nullable=True, unique=False))
+
+    _generate_repo_group_name_hashes(db_4_16_0_2, op, meta.Session)
+
+
+def downgrade(migrate_engine):
+    pass
+
+
+def _generate_repo_group_name_hashes(models, op, session):
+    repo_groups = models.RepoGroup.get_all()
+    for repo_group in repo_groups:
+        print(repo_group.group_name)
+        hash_ = RepoGroup.hash_repo_group_name(repo_group.group_name)
+        params = {'hash': hash_, 'id': repo_group.group_id}
+        query = text(
+            'UPDATE groups SET repo_group_name_hash = :hash'
+            ' WHERE group_id = :id').bindparams(**params)
+        op.execute(query)
+    session().commit()

rhodecode/lib/dbmigrate/versions/097_version_4_17_0.py

@@ -0,0 +1,39 b''
+# -*- coding: utf-8 -*-
+
+import logging
+
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+
+from rhodecode.lib.dbmigrate.versions import _reset_base
+from rhodecode.model import init_model_encryption
+
+
+log = logging.getLogger(__name__)
+
+
+def upgrade(migrate_engine):
+    """
+    Upgrade operations go here.
+    Don't create your own engine; bind migrate_engine to your metadata
+    """
+    _reset_base(migrate_engine)
+    from rhodecode.lib.dbmigrate.schema import db_4_16_0_2
+
+    init_model_encryption(db_4_16_0_2)
+
+    context = MigrationContext.configure(migrate_engine.connect())
+    op = Operations(context)
+
+    repo_group = db_4_16_0_2.RepoGroup.__table__
+    
+    with op.batch_alter_table(repo_group.name) as batch_op:
+        batch_op.alter_column("repo_group_name_hash", nullable=False)
+
+
+def downgrade(migrate_engine):
+    pass
+
+
+def _generate_repo_group_name_hashes(models, op, session):
+    pass

rhodecode/__init__.py

@@ -45,7 +45,7 b' PYRAMID_SETTINGS = {}'
 EXTENSIONS = {}
 
 __version__ = ('.'.join((str(each) for each in VERSION[:3])))
-__dbversion__ = 95  # defines current db version for migrations
+__dbversion__ = 97  # defines current db version for migrations
 __platform__ = platform.system()
 __license__ = 'AGPLv3, and Commercial License'
 __author__ = 'RhodeCode GmbH'

rhodecode/apps/admin/__init__.py

@@ -424,6 +424,10 b' def admin_routes(config):'
         pattern='/repo_groups')
 
     config.add_route(
+        name='repo_groups_data',
+        pattern='/repo_groups_data')
+
+    config.add_route(
         name='repo_group_new',
         pattern='/repo_group/new')
 

rhodecode/apps/admin/tests/test_admin_repository_groups.py

@@ -23,11 +23,11 b' import pytest'
 
 from rhodecode.apps._base import ADMIN_PREFIX
 from rhodecode.lib import helpers as h
-from rhodecode.model.db import Repository, UserRepoToPerm, User
+from rhodecode.model.db import Repository, UserRepoToPerm, User, RepoGroup
 from rhodecode.model.meta import Session
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.tests import (
-    assert_session_flash, TEST_USER_REGULAR_LOGIN, TESTS_TMP_PATH, TestController)
+    assert_session_flash, TEST_USER_REGULAR_LOGIN, TESTS_TMP_PATH)
 from rhodecode.tests.fixture import Fixture
 
 fixture = Fixture()
@@ -38,6 +38,7 b' def route_path(name, params=None, **kwar'
 
     base_url = {
         'repo_groups': ADMIN_PREFIX + '/repo_groups',
+        'repo_groups_data': ADMIN_PREFIX + '/repo_groups_data',
         'repo_group_new': ADMIN_PREFIX + '/repo_group/new',
         'repo_group_create': ADMIN_PREFIX + '/repo_group/create',
 
@@ -59,13 +60,30 b' def _get_permission_for_user(user, repo)'
 
 @pytest.mark.usefixtures("app")
 class TestAdminRepositoryGroups(object):
+
     def test_show_repo_groups(self, autologin_user):
-        response = self.app.get(route_path('repo_groups'))
-        response.mustcontain('data: []')
+        self.app.get(route_path('repo_groups'))
+
+    def test_show_repo_groups_data(self, autologin_user, xhr_header):
+        response = self.app.get(route_path(
+            'repo_groups_data'), extra_environ=xhr_header)
+
+        all_repo_groups = RepoGroup.query().count()
+        assert response.json['recordsTotal'] == all_repo_groups
 
-    def test_show_repo_groups_after_creating_group(self, autologin_user):
+    def test_show_repo_groups_data_filtered(self, autologin_user, xhr_header):
+        response = self.app.get(route_path(
+            'repo_groups_data', params={'search[value]': 'empty_search'}),
+            extra_environ=xhr_header)
+
+        all_repo_groups = RepoGroup.query().count()
+        assert response.json['recordsTotal'] == all_repo_groups
+        assert response.json['recordsFiltered'] == 0
+
+    def test_show_repo_groups_after_creating_group(self, autologin_user, xhr_header):
         fixture.create_repo_group('test_repo_group')
-        response = self.app.get(route_path('repo_groups'))
+        response = self.app.get(route_path(
+            'repo_groups_data'), extra_environ=xhr_header)
         response.mustcontain('"name_raw": "test_repo_group"')
         fixture.destroy_repo_group('test_repo_group')
 

rhodecode/apps/admin/views/repo_groups.py

@@ -17,7 +17,7 b''
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
-
+import datetime
 import logging
 import formencode
 import formencode.htmlfill
@@ -30,16 +30,16 b' from pyramid.response import Response'
 from rhodecode import events
 from rhodecode.apps._base import BaseAppView, DataGridAppView
 
-from rhodecode.lib.ext_json import json
 from rhodecode.lib.auth import (
     LoginRequired, CSRFRequired, NotAnonymous,
     HasPermissionAny, HasRepoGroupPermissionAny)
 from rhodecode.lib import helpers as h, audit_logger
-from rhodecode.lib.utils2 import safe_int, safe_unicode
+from rhodecode.lib.utils2 import safe_int, safe_unicode, datetime_to_time
 from rhodecode.model.forms import RepoGroupForm
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.scm import RepoGroupList
-from rhodecode.model.db import Session, RepoGroup
+from rhodecode.model.db import (
+    or_, count, func, in_filter_generator, Session, RepoGroup, User, Repository)
 
 log = logging.getLogger(__name__)
 
@@ -88,22 +88,168 b' class AdminRepoGroupsView(BaseAppView, D'
                 return False
         return False
 
+    # permission check in data loading of
+    # `repo_group_list_data` via RepoGroupList
     @LoginRequired()
     @NotAnonymous()
-    # perms check inside
     @view_config(
         route_name='repo_groups', request_method='GET',
         renderer='rhodecode:templates/admin/repo_groups/repo_groups.mako')
     def repo_group_list(self):
         c = self.load_default_context()
+        return self._get_template_context(c)
 
-        repo_group_list = RepoGroup.get_all_repo_groups()
-        repo_group_list_acl = RepoGroupList(
-            repo_group_list, perm_set=['group.admin'])
-        repo_group_data = RepoGroupModel().get_repo_groups_as_dict(
-            repo_group_list=repo_group_list_acl, admin=True)
-        c.data = json.dumps(repo_group_data)
-        return self._get_template_context(c)
+    # permission check inside
+    @LoginRequired()
+    @NotAnonymous()
+    @view_config(
+        route_name='repo_groups_data', request_method='GET',
+        renderer='json_ext', xhr=True)
+    def repo_group_list_data(self):
+        self.load_default_context()
+        column_map = {
+            'name_raw': 'group_name_hash',
+            'desc': 'group_description',
+            'last_change_raw': 'updated_on',
+            'top_level_repos': 'repos_total',
+            'owner': 'user_username',
+        }
+        draw, start, limit = self._extract_chunk(self.request)
+        search_q, order_by, order_dir = self._extract_ordering(
+            self.request, column_map=column_map)
+
+        _render = self.request.get_partial_renderer(
+            'rhodecode:templates/data_table/_dt_elements.mako')
+        c = _render.get_call_context()
+
+        def quick_menu(repo_group_name):
+            return _render('quick_repo_group_menu', repo_group_name)
+
+        def repo_group_lnk(repo_group_name):
+            return _render('repo_group_name', repo_group_name)
+
+        def last_change(last_change):
+            if isinstance(last_change, datetime.datetime) and not last_change.tzinfo:
+                delta = datetime.timedelta(
+                    seconds=(datetime.datetime.now() - datetime.datetime.utcnow()).seconds)
+                last_change = last_change + delta
+            return _render("last_change", last_change)
+
+        def desc(desc, personal):
+            return _render(
+                'repo_group_desc', desc, personal, c.visual.stylify_metatags)
+
+        def repo_group_actions(repo_group_id, repo_group_name, gr_count):
+            return _render(
+                'repo_group_actions', repo_group_id, repo_group_name, gr_count)
+
+        def user_profile(username):
+            return _render('user_profile', username)
+
+        auth_repo_group_list = RepoGroupList(
+            RepoGroup.query().all(), perm_set=['group.admin'])
+
+        allowed_ids = [-1]
+        for repo_group in auth_repo_group_list:
+                allowed_ids.append(repo_group.group_id)
+
+        repo_groups_data_total_count = RepoGroup.query()\
+            .filter(or_(
+                # generate multiple IN to fix limitation problems
+                *in_filter_generator(RepoGroup.group_id, allowed_ids)
+            )) \
+            .count()
+
+        repo_groups_data_total_inactive_count = RepoGroup.query()\
+            .filter(RepoGroup.group_id.in_(allowed_ids))\
+            .count()
+
+        repo_count = count(Repository.repo_id)
+        base_q = Session.query(
+            RepoGroup.group_name,
+            RepoGroup.group_name_hash,
+            RepoGroup.group_description,
+            RepoGroup.group_id,
+            RepoGroup.personal,
+            RepoGroup.updated_on,
+            User,
+            repo_count.label('repos_count')
+            ) \
+            .filter(or_(
+                # generate multiple IN to fix limitation problems
+                *in_filter_generator(RepoGroup.group_id, allowed_ids)
+            )) \
+            .outerjoin(Repository) \
+            .join(User, User.user_id == RepoGroup.user_id) \
+            .group_by(RepoGroup, User)
+
+        if search_q:
+            like_expression = u'%{}%'.format(safe_unicode(search_q))
+            base_q = base_q.filter(or_(
+                RepoGroup.group_name.ilike(like_expression),
+            ))
+
+        repo_groups_data_total_filtered_count = base_q.count()
+        # the inactive isn't really used, but we still make it same as other data grids
+        # which use inactive (users,user groups)
+        repo_groups_data_total_filtered_inactive_count = repo_groups_data_total_filtered_count
+
+        sort_defined = False
+        if order_by == 'group_name':
+            sort_col = func.lower(RepoGroup.group_name)
+            sort_defined = True
+        elif order_by == 'repos_total':
+            sort_col = repo_count
+            sort_defined = True
+        elif order_by == 'user_username':
+            sort_col = User.username
+        else:
+            sort_col = getattr(RepoGroup, order_by, None)
+
+        if sort_defined or sort_col:
+            if order_dir == 'asc':
+                sort_col = sort_col.asc()
+            else:
+                sort_col = sort_col.desc()
+
+        base_q = base_q.order_by(sort_col)
+        base_q = base_q.offset(start).limit(limit)
+
+        # authenticated access to user groups
+        auth_repo_group_list = base_q.all()
+
+        repo_groups_data = []
+        for repo_gr in auth_repo_group_list:
+            row = {
+                "menu": quick_menu(repo_gr.group_name),
+                "name": repo_group_lnk(repo_gr.group_name),
+                "name_raw": repo_gr.group_name,
+                "last_change": last_change(repo_gr.updated_on),
+                "last_change_raw": datetime_to_time(repo_gr.updated_on),
+
+                "last_changeset": "",
+                "last_changeset_raw": "",
+
+                "desc": desc(repo_gr.group_description, repo_gr.personal),
+                "owner": user_profile(repo_gr.User.username),
+                "top_level_repos": repo_gr.repos_count,
+                "action": repo_group_actions(
+                    repo_gr.group_id, repo_gr.group_name, repo_gr.repos_count),
+
+            }
+
+            repo_groups_data.append(row)
+
+        data = ({
+            'draw': draw,
+            'data': repo_groups_data,
+            'recordsTotal': repo_groups_data_total_count,
+            'recordsTotalInactive': repo_groups_data_total_inactive_count,
+            'recordsFiltered': repo_groups_data_total_filtered_count,
+            'recordsFilteredInactive': repo_groups_data_total_filtered_inactive_count,
+        })
+
+        return data
 
     @LoginRequired()
     @NotAnonymous()

rhodecode/model/db.py

@@ -25,6 +25,7 b' Database Models for RhodeCode Enterprise'
 import re
 import os
 import time
+import string
 import hashlib
 import logging
 import datetime
@@ -50,6 +51,7 b' from sqlalchemy.dialects.mysql import LO'
 from zope.cachedescriptors.property import Lazy as LazyProperty
 from pyramid import compat
 from pyramid.threadlocal import get_current_request
+from webhelpers.text import collapse, remove_formatting
 
 from rhodecode.translation import _
 from rhodecode.lib.vcs import get_vcs_instance
@@ -2469,7 +2471,8 b' class RepoGroup(Base, BaseModel):'
     CHOICES_SEPARATOR = '/'  # used to generate select2 choices for nested groups
 
     group_id = Column("group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
-    group_name = Column("group_name", String(255), nullable=False, unique=True, default=None)
+    _group_name = Column("group_name", String(255), nullable=False, unique=True, default=None)
+    group_name_hash = Column("repo_group_name_hash", String(1024), nullable=False, unique=False)
     group_parent_id = Column("group_parent_id", Integer(), ForeignKey('groups.group_id'), nullable=True, unique=None, default=None)
     group_description = Column("group_description", String(10000), nullable=True, unique=None, default=None)
     enable_locking = Column("enable_locking", Boolean(), nullable=False, unique=None, default=False)
@@ -2492,6 +2495,15 b' class RepoGroup(Base, BaseModel):'
         return u"<%s('id:%s:%s')>" % (
             self.__class__.__name__, self.group_id, self.group_name)
 
+    @hybrid_property
+    def group_name(self):
+        return self._group_name
+
+    @group_name.setter
+    def group_name(self, value):
+        self._group_name = value
+        self.group_name_hash = self.hash_repo_group_name(value)
+
     @validates('group_parent_id')
     def validate_group_parent_id(self, key, val):
         """
@@ -2508,6 +2520,18 b' class RepoGroup(Base, BaseModel):'
         return h.escape(self.group_description)
 
     @classmethod
+    def hash_repo_group_name(cls, repo_group_name):
+        val = remove_formatting(repo_group_name)
+        val = safe_str(val).lower()
+        chars = []
+        for c in val:
+            if c not in string.ascii_letters:
+                c = str(ord(c))
+            chars.append(c)
+
+        return ''.join(chars)
+
+    @classmethod
     def _generate_choice(cls, repo_group):
         from webhelpers.html import literal as _literal
         _name = lambda k: _literal(cls.CHOICES_SEPARATOR.join(k))
@@ -2770,6 +2794,13 b' class RepoGroup(Base, BaseModel):'
         }
         return data
 
+    def get_dict(self):
+        # Since we transformed `group_name` to a hybrid property, we need to
+        # keep compatibility with the code which uses `group_name` field.
+        result = super(RepoGroup, self).get_dict()
+        result['group_name'] = result.pop('_group_name', None)
+        return result
+
 
 class Permission(Base, BaseModel):
     __tablename__ = 'permissions'

rhodecode/templates/admin/repo_groups/repo_groups.mako

@@ -10,7 +10,7 b''
 
 <%def name="breadcrumbs_links()">
     <input class="q_filter_box" id="q_filter" size="15" type="text" name="filter" placeholder="${_('quick filter...')}" value=""/>
-    ${h.link_to(_('Admin'),h.route_path('admin_home'))} &raquo; <span id="repo_group_count">0</span> ${_('repository groups')}
+    ${h.link_to(_('Admin'),h.route_path('admin_home'))} &raquo; <span id="repo_group_count">0</span>
 </%def>
 
 <%def name="menu_bar_nav()">
@@ -36,15 +36,32 b''
 
 <script>
 $(document).ready(function() {
-
-    var get_datatable_count = function(){
-      var api = $('#group_list_table').dataTable().api();
-      $('#repo_group_count').text(api.page.info().recordsDisplay);
-    };
+    var $repoGroupsListTable = $('#group_list_table');
 
    // repo group list
-   $('#group_list_table').DataTable({
-      data: ${c.data|n},
+   $repoGroupsListTable.DataTable({
+      processing: true,
+      serverSide: true,
+      ajax: {
+          "url": "${h.route_path('repo_groups_data')}",
+          "dataSrc": function (json) {
+              var filteredCount = json.recordsFiltered;
+              var filteredInactiveCount = json.recordsFilteredInactive;
+              var totalInactive = json.recordsTotalInactive;
+              var total = json.recordsTotal;
+
+              var _text = _gettext(
+                      "{0} of {1} repository groups").format(
+                      filteredCount, total);
+
+              if (total === filteredCount) {
+                  _text = _gettext("{0} repository groups").format(total);
+              }
+              $('#repo_group_count').text(_text);
+              return json.data;
+          },
+      },
+
       dom: 'rtp',
       pageLength: ${c.visual.admin_grid_items},
       order: [[ 0, "asc" ]],
@@ -62,36 +79,34 b''
          { data: {"_": "owner",
                   "sort": "owner"}, title: "${_('Owner')}", className: "td-user" },
          { data: {"_": "action",
-                  "sort": "action"}, title: "${_('Action')}", className: "td-action" }
+                  "sort": "action"}, title: "${_('Action')}", className: "td-action", orderable: false }
       ],
       language: {
           paginate: DEFAULT_GRID_PAGINATION,
+          sProcessing: _gettext('loading...'),
           emptyTable: _gettext("No repository groups available yet.")
       },
-      "initComplete": function( settings, json ) {
-          get_datatable_count();
-          quick_repo_menu();
-      }
     });
 
-     // update the counter when doing search
-    $('#group_list_table').on( 'search.dt', function (e,settings) {
-      get_datatable_count();
+    $repoGroupsListTable.on('xhr.dt', function(e, settings, json, xhr){
+        $repoGroupsListTable.css('opacity', 1);
+    });
+
+    $repoGroupsListTable.on('preXhr.dt', function(e, settings, data){
+        $repoGroupsListTable.css('opacity', 0.3);
     });
 
-    // filter, filter both grids
-    $('#q_filter').on( 'keyup', function () {
-
-      var repo_group_api = $('#group_list_table').dataTable().api();
-      repo_group_api
-        .columns(0)
-        .search(this.value)
-        .draw();
+    // filter
+    $('#q_filter').on('keyup',
+        $.debounce(250, function() {
+            $repoGroupsListTable.DataTable().search(
+                $('#q_filter').val()
+            ).draw();
+        })
+    );
     });
 
-    // refilter table if page load via back button
-    $("#q_filter").trigger('keyup');
-});
 </script>
+
 </%def>
 

rhodecode/templates/admin/repos/repos.mako

@@ -66,7 +66,7 b''
          { data: {"_": "state",
                   "sort": "state"}, title: "${_('State')}", className: "td-tags td-state" },
          { data: {"_": "action",
-                  "sort": "action"}, title: "${_('Action')}", className: "td-action" }
+                  "sort": "action"}, title: "${_('Action')}", className: "td-action", orderable: false }
       ],
       language: {
           paginate: DEFAULT_GRID_PAGINATION,

rhodecode/tests/functional/test_delegated_admin.py

@@ -34,6 +34,8 b' def route_path(name, params=None, **kwar'
             ADMIN_PREFIX + '/repos',
         'repo_groups':
             ADMIN_PREFIX + '/repo_groups',
+        'repo_groups_data':
+            ADMIN_PREFIX + '/repo_groups_data',
         'user_groups':
             ADMIN_PREFIX + '/user_groups',
         'user_groups_data':
@@ -67,8 +69,9 b' class TestAdminDelegatedUser(TestControl'
         response = self.app.get(route_path('repos'), status=200)
         response.mustcontain('data: []')
 
-        response = self.app.get(route_path('repo_groups'), status=200)
-        response.mustcontain('data: []')
+        response = self.app.get(route_path('repo_groups_data'),
+                                status=200, extra_environ=xhr_header)
+        assert response.json['data'] == []
 
         response = self.app.get(route_path('user_groups_data'),
                                 status=200, extra_environ=xhr_header)
@@ -102,7 +105,8 b' class TestAdminDelegatedUser(TestControl'
         response = self.app.get(route_path('repos'), status=200)
         response.mustcontain('"name_raw": "{}"'.format(repo_name))
 
-        response = self.app.get(route_path('repo_groups'), status=200)
+        response = self.app.get(route_path('repo_groups_data'),
+                                extra_environ=xhr_header, status=200)
         response.mustcontain('"name_raw": "{}"'.format(repo_group_name))
 
         response = self.app.get(route_path('user_groups_data'),
@@ -144,7 +148,8 b' class TestAdminDelegatedUser(TestControl'
         response = self.app.get(route_path('repos'), status=200)
         response.mustcontain('"name_raw": "{}"'.format(repo_name))
 
-        response = self.app.get(route_path('repo_groups'), status=200)
+        response = self.app.get(route_path('repo_groups_data'),
+                                extra_environ=xhr_header, status=200)
         response.mustcontain('"name_raw": "{}"'.format(repo_group_name))
 
         response = self.app.get(route_path('user_groups_data'),