dashboard: main page grids async load.

- in some cases when super-admins or all repos in root loading main
page could take minutes because we loaded whole dataset.
- this change make the loading limited to first page of data.
  Rest is loaded via DB pagination
- In many cases this is major speed-up for showing 1st page fast.

rhodecode/apps/home/__init__.py

@@ -44,6 +44,14 b' def includeme(config):'
         pattern='/')
 
     config.add_route(
+        name='main_page_repos_data',
+        pattern='/_home_repos')
+
+    config.add_route(
+        name='main_page_repo_groups_data',
+        pattern='/_home_repo_groups')
+
+    config.add_route(
         name='user_autocomplete_data',
         pattern='/_users')
 

rhodecode/apps/home/views.py

@@ -22,29 +22,30 b' import re'
 import logging
 import collections
 
+from pyramid.httpexceptions import HTTPNotFound
 from pyramid.view import view_config
 
-from rhodecode.apps._base import BaseAppView
+from rhodecode.apps._base import BaseAppView, DataGridAppView
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import (
-    LoginRequired, NotAnonymous, HasRepoGroupPermissionAnyDecorator, CSRFRequired)
+    LoginRequired, NotAnonymous, HasRepoGroupPermissionAnyDecorator, CSRFRequired,
+    HasRepoGroupPermissionAny)
 from rhodecode.lib.codeblocks import filenode_as_lines_tokens
 from rhodecode.lib.index import searcher_from_config
 from rhodecode.lib.utils2 import safe_unicode, str2bool, safe_int
-from rhodecode.lib.ext_json import json
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.model.db import (
-    func, true, or_, case, in_filter_generator, Repository, RepoGroup, User, UserGroup)
+    func, true, or_, case, in_filter_generator, Session,
+    Repository, RepoGroup, User, UserGroup)
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.repo_group import RepoGroupModel
-from rhodecode.model.scm import RepoGroupList, RepoList
 from rhodecode.model.user import UserModel
 from rhodecode.model.user_group import UserGroupModel
 
 log = logging.getLogger(__name__)
 
 
-class HomeView(BaseAppView):
+class HomeView(BaseAppView, DataGridAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
@@ -673,23 +674,6 b' class HomeView(BaseAppView):'
 
         return {'suggestions': res}
 
-    def _get_groups_and_repos(self, repo_group_id=None):
-        # repo groups groups
-        repo_group_list = RepoGroup.get_all_repo_groups(group_id=repo_group_id)
-        _perms = ['group.read', 'group.write', 'group.admin']
-        repo_group_list_acl = RepoGroupList(repo_group_list, perm_set=_perms)
-        repo_group_data = RepoGroupModel().get_repo_groups_as_dict(
-            repo_group_list=repo_group_list_acl, admin=False)
-
-        # repositories
-        repo_list = Repository.get_all_repos(group_id=repo_group_id)
-        _perms = ['repository.read', 'repository.write', 'repository.admin']
-        repo_list_acl = RepoList(repo_list, perm_set=_perms)
-        repo_data = RepoModel().get_repos_as_dict(
-            repo_list=repo_list_acl, admin=False)
-
-        return repo_data, repo_group_data
-
     @LoginRequired()
     @view_config(
         route_name='home', request_method='GET',
@@ -697,13 +681,71 b' class HomeView(BaseAppView):'
     def main_page(self):
         c = self.load_default_context()
         c.repo_group = None
+        return self._get_template_context(c)
 
-        repo_data, repo_group_data = self._get_groups_and_repos()
-        # json used to render the grids
-        c.repos_data = json.dumps(repo_data)
-        c.repo_groups_data = json.dumps(repo_group_data)
+    def _main_page_repo_groups_data(self, repo_group_id):
+        column_map = {
+            'name_raw': 'group_name_hash',
+            'desc': 'group_description',
+            'last_change_raw': 'updated_on',
+            'owner': 'user_username',
+        }
+        draw, start, limit = self._extract_chunk(self.request)
+        search_q, order_by, order_dir = self._extract_ordering(
+            self.request, column_map=column_map)
+        return RepoGroupModel().get_repo_groups_data_table(
+            draw, start, limit,
+            search_q, order_by, order_dir,
+            self._rhodecode_user, repo_group_id)
+
+    def _main_page_repos_data(self, repo_group_id):
+        column_map = {
+            'name_raw': 'repo_name',
+            'desc': 'description',
+            'last_change_raw': 'updated_on',
+            'owner': 'user_username',
+        }
+        draw, start, limit = self._extract_chunk(self.request)
+        search_q, order_by, order_dir = self._extract_ordering(
+            self.request, column_map=column_map)
+        return RepoModel().get_repos_data_table(
+            draw, start, limit,
+            search_q, order_by, order_dir,
+            self._rhodecode_user, repo_group_id)
 
-        return self._get_template_context(c)
+    @LoginRequired()
+    @view_config(
+        route_name='main_page_repo_groups_data',
+        request_method='GET', renderer='json_ext', xhr=True)
+    def main_page_repo_groups_data(self):
+        self.load_default_context()
+        repo_group_id = safe_int(self.request.GET.get('repo_group_id'))
+
+        if repo_group_id:
+            group = RepoGroup.get_or_404(repo_group_id)
+            _perms = ['group.read', 'group.write', 'group.admin']
+            if not HasRepoGroupPermissionAny(*_perms)(
+                    group.group_name, 'user is allowed to list repo group children'):
+                raise HTTPNotFound()
+
+        return self._main_page_repo_groups_data(repo_group_id)
+
+    @LoginRequired()
+    @view_config(
+        route_name='main_page_repos_data',
+        request_method='GET', renderer='json_ext', xhr=True)
+    def main_page_repos_data(self):
+        self.load_default_context()
+        repo_group_id = safe_int(self.request.GET.get('repo_group_id'))
+
+        if repo_group_id:
+            group = RepoGroup.get_or_404(repo_group_id)
+            _perms = ['group.read', 'group.write', 'group.admin']
+            if not HasRepoGroupPermissionAny(*_perms)(
+                    group.group_name, 'user is allowed to list repo group children'):
+                raise HTTPNotFound()
+
+        return self._main_page_repos_data(repo_group_id)
 
     @LoginRequired()
     @HasRepoGroupPermissionAnyDecorator(
@@ -717,16 +759,6 b' class HomeView(BaseAppView):'
     def repo_group_main_page(self):
         c = self.load_default_context()
         c.repo_group = self.request.db_repo_group
-        repo_data, repo_group_data = self._get_groups_and_repos(c.repo_group.group_id)
-
-        # update every 5 min
-        if self.request.db_repo_group.last_commit_cache_update_diff > 60 * 5:
-            self.request.db_repo_group.update_commit_cache()
-
-        # json used to render the grids
-        c.repos_data = json.dumps(repo_data)
-        c.repo_groups_data = json.dumps(repo_group_data)
-
         return self._get_template_context(c)
 
     @LoginRequired()

rhodecode/lib/base.py

@@ -528,6 +528,8 b' def add_events_routes(config):'
     from rhodecode.apps._base import ADMIN_PREFIX
 
     config.add_route(name='home', pattern='/')
+    config.add_route(name='main_page_repos_data', pattern='/_home_repos')
+    config.add_route(name='main_page_repo_groups_data', pattern='/_home_repo_groups')
 
     config.add_route(name='login', pattern=ADMIN_PREFIX + '/login')
     config.add_route(name='logout', pattern=ADMIN_PREFIX + '/logout')

rhodecode/model/repo.py

@@ -281,6 +281,87 b' class RepoModel(BaseModel):'
 
         return repos_data
 
+    def get_repos_data_table(
+            self, draw, start, limit,
+            search_q, order_by, order_dir,
+            auth_user, repo_group_id):
+        from rhodecode.model.scm import RepoList
+
+        _perms = ['repository.read', 'repository.write', 'repository.admin']
+
+        repos = Repository.query() \
+            .filter(Repository.group_id == repo_group_id) \
+            .all()
+        auth_repo_list = RepoList(
+            repos, perm_set=_perms,
+            extra_kwargs=dict(user=auth_user))
+
+        allowed_ids = [-1]
+        for repo in auth_repo_list:
+            allowed_ids.append(repo.repo_id)
+
+        repos_data_total_count = Repository.query() \
+            .filter(Repository.group_id == repo_group_id) \
+            .filter(or_(
+                # generate multiple IN to fix limitation problems
+                *in_filter_generator(Repository.repo_id, allowed_ids))
+            ) \
+            .count()
+
+        base_q = Session.query(
+            Repository.repo_id,
+            Repository.repo_name,
+            Repository.description,
+            Repository.repo_type,
+            Repository.repo_state,
+            Repository.private,
+            Repository.archived,
+            Repository.fork,
+            Repository.updated_on,
+            Repository._changeset_cache,
+            User,
+            ) \
+            .filter(Repository.group_id == repo_group_id) \
+            .filter(or_(
+                # generate multiple IN to fix limitation problems
+                *in_filter_generator(Repository.repo_id, allowed_ids))
+            ) \
+            .join(User, User.user_id == Repository.user_id) \
+            .group_by(Repository, User)
+
+        repos_data_total_filtered_count = base_q.count()
+
+        sort_defined = False
+        if order_by == 'repo_name':
+            sort_col = func.lower(Repository.repo_name)
+            sort_defined = True
+        elif order_by == 'user_username':
+            sort_col = User.username
+        else:
+            sort_col = getattr(Repository, order_by, None)
+
+        if sort_defined or sort_col:
+            if order_dir == 'asc':
+                sort_col = sort_col.asc()
+            else:
+                sort_col = sort_col.desc()
+
+        base_q = base_q.order_by(sort_col)
+        base_q = base_q.offset(start).limit(limit)
+
+        repos_list = base_q.all()
+
+        repos_data = RepoModel().get_repos_as_dict(
+            repo_list=repos_list, admin=False)
+
+        data = ({
+            'draw': draw,
+            'data': repos_data,
+            'recordsTotal': repos_data_total_count,
+            'recordsFiltered': repos_data_total_filtered_count,
+        })
+        return data
+
     def _get_defaults(self, repo_name):
         """
         Gets information about repository, and returns a dict for

rhodecode/model/repo_group.py

@@ -775,6 +775,84 b' class RepoGroupModel(BaseModel):'
 
         return repo_group_data
 
+    def get_repo_groups_data_table(
+            self, draw, start, limit,
+            search_q, order_by, order_dir,
+            auth_user, repo_group_id):
+        from rhodecode.model.scm import RepoGroupList
+
+        _perms = ['group.read', 'group.write', 'group.admin']
+        repo_groups = RepoGroup.query() \
+            .filter(RepoGroup.group_parent_id == repo_group_id) \
+            .all()
+        auth_repo_group_list = RepoGroupList(
+            repo_groups, perm_set=_perms,
+            extra_kwargs=dict(user=auth_user))
+
+        allowed_ids = [-1]
+        for repo_group in auth_repo_group_list:
+            allowed_ids.append(repo_group.group_id)
+
+        repo_groups_data_total_count = RepoGroup.query() \
+            .filter(RepoGroup.group_parent_id == repo_group_id) \
+            .filter(or_(
+                # generate multiple IN to fix limitation problems
+                *in_filter_generator(RepoGroup.group_id, allowed_ids))
+            ) \
+            .count()
+
+        base_q = Session.query(
+            RepoGroup.group_name,
+            RepoGroup.group_name_hash,
+            RepoGroup.group_description,
+            RepoGroup.group_id,
+            RepoGroup.personal,
+            RepoGroup.updated_on,
+            RepoGroup._changeset_cache,
+            User,
+            ) \
+            .filter(RepoGroup.group_parent_id == repo_group_id) \
+            .filter(or_(
+                # generate multiple IN to fix limitation problems
+                *in_filter_generator(RepoGroup.group_id, allowed_ids))
+            ) \
+            .join(User, User.user_id == RepoGroup.user_id) \
+            .group_by(RepoGroup, User)
+
+        repo_groups_data_total_filtered_count = base_q.count()
+
+        sort_defined = False
+
+        if order_by == 'group_name':
+            sort_col = func.lower(RepoGroup.group_name)
+            sort_defined = True
+        elif order_by == 'user_username':
+            sort_col = User.username
+        else:
+            sort_col = getattr(RepoGroup, order_by, None)
+
+        if sort_defined or sort_col:
+            if order_dir == 'asc':
+                sort_col = sort_col.asc()
+            else:
+                sort_col = sort_col.desc()
+
+        base_q = base_q.order_by(sort_col)
+        base_q = base_q.offset(start).limit(limit)
+
+        repo_group_list = base_q.all()
+
+        repo_groups_data = RepoGroupModel().get_repo_groups_as_dict(
+            repo_group_list=repo_group_list, admin=False)
+
+        data = ({
+            'draw': draw,
+            'data': repo_groups_data,
+            'recordsTotal': repo_groups_data_total_count,
+            'recordsFiltered': repo_groups_data_total_filtered_count,
+        })
+        return data
+
     def _get_defaults(self, repo_group_name):
         repo_group = RepoGroup.get_by_group_name(repo_group_name)
 

rhodecode/public/js/rhodecode/routes.js

@@ -150,6 +150,8 b' function registerRCRoutes() {'
     pyroutes.register('reset_password', '/_admin/password_reset', []);
     pyroutes.register('reset_password_confirmation', '/_admin/password_reset_confirmation', []);
     pyroutes.register('home', '/', []);
+    pyroutes.register('main_page_repos_data', '/_home_repos', []);
+    pyroutes.register('main_page_repo_groups_data', '/_home_repo_groups', []);
     pyroutes.register('user_autocomplete_data', '/_users', []);
     pyroutes.register('user_group_autocomplete_data', '/_user_groups', []);
     pyroutes.register('repo_list_data', '/_repos', []);

rhodecode/templates/index_base.mako

@@ -15,58 +15,97 b''
 
         </div>
         <!-- end box / title -->
+        <div id="no_grid_data" class="table" style="display: none">
+            <h2 class="no-object-border">
+                 ${_('No repositories or repositories groups exists here.')}
+            </h2>
+        </div>
+
         <div class="table">
-            <div id="groups_list_wrap">
-                <table id="group_list_table" class="display" style="width: 100%"></table>
+            <div id="groups_list_wrap" style="min-height: 200px;">
+                <table id="group_list_table" class="display" style="width: 100%;"></table>
             </div>
         </div>
 
         <div class="table">
-            <div id="repos_list_wrap">
-                <table id="repo_list_table" class="display" style="width: 100%"></table>
+            <div id="repos_list_wrap" style="min-height: 200px;">
+                <table id="repo_list_table" class="display" style="width: 100%;"></table>
             </div>
         </div>
 
-       ## no repository groups and repos present, show something to the users
-       % if c.repo_groups_data == '[]' and c.repos_data == '[]':
-        <div class="table">
-            <h2 class="no-object-border">
-                 ${_('No repositories or repositories groups exists here.')}
-            </h2>
-        </div>
-       % endif
-
     </div>
     <script>
       $(document).ready(function() {
 
         // repo group list
-        % if c.repo_groups_data != '[]':
-        $('#group_list_table').DataTable({
-          data: ${c.repo_groups_data|n},
+        var $groupListTable = $('#group_list_table');
+
+        $groupListTable.DataTable({
+            processing: true,
+            serverSide: true,
+            ajax: {
+                "url": "${h.route_path('main_page_repo_groups_data')}",
+                "data": function (d) {
+                    % if c.repo_group:
+                        d.repo_group_id = ${c.repo_group.group_id}
+                    % endif
+                }
+            },
           dom: 'rtp',
           pageLength: ${c.visual.dashboard_items},
           order: [[ 0, "asc" ]],
           columns: [
-             { data: {"_": "name",
-                      "sort": "name_raw"}, title: "${_('Name')}", className: "truncate-wrap td-grid-name" },
+                {
+                    data: {
+                        "_": "name",
+                        "sort": "name_raw"
+                    }, title: "${_('Name')}", className: "truncate-wrap td-grid-name"
+                },
              { data: 'menu', "bSortable": false, className: "quick_repo_menu" },
-             { data: {"_": "desc",
-                      "sort": "desc"}, title: "${_('Description')}", className: "td-description" },
-             { data: {"_": "last_change",
+                {
+                    data: {
+                        "_": "desc",
+                        "sort": "desc"
+                    }, title: "${_('Description')}", className: "td-description"
+                },
+                {
+                    data: {
+                        "_": "last_change",
                       "sort": "last_change_raw",
-                      "type": Number}, title: "${_('Last Change')}", className: "td-time" },
-             { data: {"_": "last_changeset",
+                        "type": Number
+                    }, title: "${_('Last Change')}", className: "td-time"
+                },
+                {
+                    data: {
+                        "_": "last_changeset",
                       "sort": "last_changeset_raw",
-                      "type": Number}, title: "", className: "td-hash" },
-             { data: {"_": "owner",
-                      "sort": "owner"}, title: "${_('Owner')}", className: "td-user" }
+                        "type": Number
+                    }, title: "", className: "td-hash"
+                },
+                {
+                    data: {
+                        "_": "owner",
+                        "sort": "owner"
+                    }, title: "${_('Owner')}", className: "td-user"
+                }
           ],
           language: {
             paginate: DEFAULT_GRID_PAGINATION,
-            emptyTable: _gettext("No repository groups available yet.")
+                sProcessing: _gettext('loading...'),
+                emptyTable: _gettext("No repository groups present.")
           },
           "drawCallback": function( settings, json ) {
+                // hide grid if it's empty
+                if (settings.fnRecordsDisplay() === 0) {
+                    $('#groups_list_wrap').hide();
+                    // both hidden, show no-data
+                    if ($('#repos_list_wrap').is(':hidden')) {
+                        $('#no_grid_data').show();
+                    }
+                } else {
+                    $('#groups_list_wrap').show();
+                }
+
               timeagoActivate();
               tooltipActivate();
               quick_repo_menu();
@@ -74,37 +113,90 b''
               if (settings._iDisplayLength >= settings.fnRecordsDisplay()) {
                   $(settings.nTableWrapper).find('.dataTables_paginate').hide();
               }
-          }
+
+            },
         });
-        % endif
+
+        $groupListTable.on('xhr.dt', function (e, settings, json, xhr) {
+            $groupListTable.css('opacity', 1);
+        });
+
+        $groupListTable.on('preXhr.dt', function (e, settings, data) {
+            $groupListTable.css('opacity', 0.3);
+        });
+
 
-        // repo list
-        % if c.repos_data != '[]':
-        $('#repo_list_table').DataTable({
-          data: ${c.repos_data|n},
+        ##  // repo list
+        var $repoListTable = $('#repo_list_table');
+
+        $repoListTable.DataTable({
+            processing: true,
+            serverSide: true,
+            ajax: {
+                "url": "${h.route_path('main_page_repos_data')}",
+                "data": function (d) {
+                    % if c.repo_group:
+                        d.repo_group_id = ${c.repo_group.group_id}
+                    % endif
+                }
+            },
+            order: [[0, "asc"]],
           dom: 'rtp',
-          order: [[ 0, "asc" ]],
           pageLength: ${c.visual.dashboard_items},
           columns: [
-             { data: {"_": "name",
-                      "sort": "name_raw"}, title: "${_('Name')}", className: "truncate-wrap td-grid-name" },
-             { data: 'menu', "bSortable": false, className: "quick_repo_menu" },
-             { data: {"_": "desc",
-                      "sort": "desc"}, title: "${_('Description')}", className: "td-description" },
-             { data: {"_": "last_change",
+                {
+                    data: {
+                        "_": "name",
+                        "sort": "name_raw"
+                    }, title: "${_('Name')}", className: "truncate-wrap td-grid-name"
+                },
+                {
+                    data: 'menu', "bSortable": false, className: "quick_repo_menu"
+                },
+                {
+                    data: {
+                        "_": "desc",
+                        "sort": "desc"
+                    }, title: "${_('Description')}", className: "td-description"
+                },
+                {
+                    data: {
+                        "_": "last_change",
                       "sort": "last_change_raw",
-                      "type": Number}, title: "${_('Last Change')}", className: "td-time" },
-             { data: {"_": "last_changeset",
+                        "type": Number
+                    }, title: "${_('Last Change')}", className: "td-time", orderable: false
+                },
+                {
+                    data: {
+                        "_": "last_changeset",
                       "sort": "last_changeset_raw",
-                      "type": Number}, title: "${_('Commit')}", className: "td-hash" },
-             { data: {"_": "owner",
-                      "sort": "owner"}, title: "${_('Owner')}", className: "td-user" }
+                        "type": Number
+                    }, title: "${_('Commit')}", className: "td-hash"
+                },
+                 {
+                    data: {
+                        "_": "owner",
+                        "sort": "owner"
+                    }, title: "${_('Owner')}", className: "td-user"
+                }
           ],
           language: {
               paginate: DEFAULT_GRID_PAGINATION,
-              emptyTable: _gettext("No repositories available yet.")
+                sProcessing: _gettext('loading...'),
+                emptyTable: _gettext("No repositories present.")
           },
           "drawCallback": function( settings, json ) {
+                // hide grid if it's empty
+                if (settings.fnRecordsDisplay() == 0) {
+                    $('#repos_list_wrap').hide()
+                    // both hidden, show no-data
+                    if ($('#groups_list_wrap').is(':hidden')) {
+                        $('#no_grid_data').show()
+                    }
+                } else {
+                    $('#repos_list_wrap').show()
+                }
+
               timeagoActivate();
               tooltipActivate();
               quick_repo_menu();
@@ -112,9 +204,17 b''
               if (settings._iDisplayLength >= settings.fnRecordsDisplay()) {
                   $(settings.nTableWrapper).find('.dataTables_paginate').hide();
               }
-          }
+
+            },
         });
-        % endif
+
+        $repoListTable.on('xhr.dt', function (e, settings, json, xhr) {
+            $repoListTable.css('opacity', 1);
+        });
+
+        $repoListTable.on('preXhr.dt', function (e, settings, data) {
+            $repoListTable.css('opacity', 0.3);
+        });
 
       });
     </script>