admin-users: moved grid browsing to pyramid.

- fixed loading data to be lazy fetched.
- speeds up loading large user sets significantly

rhodecode/apps/admin/__init__.py

@@ -53,6 +53,15 b' def includeme(config):'
         name='admin_settings_sessions_cleanup',
         pattern=ADMIN_PREFIX + '/settings/sessions/cleanup')
 
+    # users admin
+    config.add_route(
+        name='users',
+        pattern=ADMIN_PREFIX + '/users')
+
+    config.add_route(
+        name='users_data',
+        pattern=ADMIN_PREFIX + '/users_data')
+
     # user auth tokens
     config.add_route(
         name='edit_user_auth_tokens',

rhodecode/apps/admin/tests/test_admin_users.py

@@ -22,18 +22,18 b' import pytest'
 
 from rhodecode.model.db import User, UserApiKeys
 
-from rhodecode.apps._base import ADMIN_PREFIX
 from rhodecode.tests import (
     TestController, TEST_USER_REGULAR_LOGIN, assert_session_flash)
 from rhodecode.tests.fixture import Fixture
-from rhodecode.tests.utils import AssertResponse
 
 fixture = Fixture()
 
 
+def route_path(name, params=None, **kwargs):
+    import urllib
+    from rhodecode.apps._base import ADMIN_PREFIX
 
-def route_path(name, **kwargs):
-    return {
+    base_url = {
         'users':
             ADMIN_PREFIX + '/users',
         'users_data':
@@ -46,9 +46,37 b' def route_path(name, **kwargs):'
             ADMIN_PREFIX + '/users/{user_id}/edit/auth_tokens/delete',
     }[name].format(**kwargs)
 
+    if params:
+        base_url = '{}?{}'.format(base_url, urllib.urlencode(params))
+    return base_url
+
 
 class TestAdminUsersView(TestController):
 
+    def test_show_users(self):
+        self.log_user()
+        self.app.get(route_path('users'))
+
+    def test_show_users_data(self, xhr_header):
+        self.log_user()
+        response = self.app.get(route_path(
+            'users_data'), extra_environ=xhr_header)
+
+        all_users = User.query().filter(
+            User.username != User.DEFAULT_USER).count()
+        assert response.json['recordsTotal'] == all_users
+
+    def test_show_users_data_filtered(self, xhr_header):
+        self.log_user()
+        response = self.app.get(route_path(
+            'users_data', params={'search[value]': 'empty_search'}),
+            extra_environ=xhr_header)
+
+        all_users = User.query().filter(
+            User.username != User.DEFAULT_USER).count()
+        assert response.json['recordsTotal'] == all_users
+        assert response.json['recordsFiltered'] == 0
+
     def test_auth_tokens_default_user(self):
         self.log_user()
         user = User.get_default_user()

rhodecode/apps/admin/views/users.py

@@ -28,9 +28,9 b' from rhodecode.lib.auth import ('
     LoginRequired, HasPermissionAllDecorator, CSRFRequired)
 from rhodecode.lib import helpers as h
 from rhodecode.lib.utils import PartialRenderer
-from rhodecode.lib.utils2 import safe_int
+from rhodecode.lib.utils2 import safe_int, safe_unicode
 from rhodecode.model.auth_token import AuthTokenModel
-from rhodecode.model.db import User
+from rhodecode.model.db import User, or_
 from rhodecode.model.meta import Session
 
 log = logging.getLogger(__name__)
@@ -58,6 +58,105 b' class AdminUsersView(BaseAppView):'
             # is a pyramid view
             raise HTTPFound('/')
 
+    def _extract_ordering(self, request):
+        column_index = safe_int(request.GET.get('order[0][column]'))
+        order_dir = request.GET.get(
+            'order[0][dir]', 'desc')
+        order_by = request.GET.get(
+            'columns[%s][data][sort]' % column_index, 'name_raw')
+
+        # translate datatable to DB columns
+        order_by = {
+            'first_name': 'name',
+            'last_name': 'lastname',
+            'last_activity': ''
+        }.get(order_by) or order_by
+
+        search_q = request.GET.get('search[value]')
+        return search_q, order_by, order_dir
+
+    def _extract_chunk(self, request):
+        start = safe_int(request.GET.get('start'), 0)
+        length = safe_int(request.GET.get('length'), 25)
+        draw = safe_int(request.GET.get('draw'))
+        return draw, start, length
+
+    @HasPermissionAllDecorator('hg.admin')
+    @view_config(
+        route_name='users', request_method='GET',
+        renderer='rhodecode:templates/admin/users/users.mako')
+    def users_list(self):
+        c = self.load_default_context()
+        return self._get_template_context(c)
+
+    @HasPermissionAllDecorator('hg.admin')
+    @view_config(
+        # renderer defined below
+        route_name='users_data', request_method='GET', renderer='json',
+        xhr=True)
+    def users_list_data(self):
+        draw, start, limit = self._extract_chunk(self.request)
+        search_q, order_by, order_dir = self._extract_ordering(self.request)
+
+        _render = PartialRenderer('data_table/_dt_elements.mako')
+
+        def user_actions(user_id, username):
+            return _render("user_actions", user_id, username)
+
+        users_data_total_count = User.query()\
+            .filter(User.username != User.DEFAULT_USER) \
+            .count()
+
+        # json generate
+        base_q = User.query().filter(User.username != User.DEFAULT_USER)
+
+        if search_q:
+            like_expression = u'{}%'.format(safe_unicode(search_q))
+            base_q = base_q.filter(or_(
+                User.username.ilike(like_expression),
+                User._email.ilike(like_expression),
+                User.name.ilike(like_expression),
+                User.lastname.ilike(like_expression),
+            ))
+
+        users_data_total_filtered_count = base_q.count()
+
+        sort_col = getattr(User, order_by, None)
+        if sort_col and order_dir == 'asc':
+            base_q = base_q.order_by(sort_col.asc())
+        elif sort_col:
+            base_q = base_q.order_by(sort_col.desc())
+
+        base_q = base_q.offset(start).limit(limit)
+        users_list = base_q.all()
+
+        users_data = []
+        for user in users_list:
+            users_data.append({
+                "username": h.gravatar_with_user(user.username),
+                "email": user.email,
+                "first_name": h.escape(user.name),
+                "last_name": h.escape(user.lastname),
+                "last_login": h.format_date(user.last_login),
+                "last_activity": h.format_date(
+                    h.time_to_datetime(user.user_data.get('last_activity', 0))),
+                "active": h.bool2icon(user.active),
+                "active_raw": user.active,
+                "admin": h.bool2icon(user.admin),
+                "extern_type": user.extern_type,
+                "extern_name": user.extern_name,
+                "action": user_actions(user.user_id, user.username),
+            })
+
+        data = ({
+            'draw': draw,
+            'data': users_data,
+            'recordsTotal': users_data_total_count,
+            'recordsFiltered': users_data_total_filtered_count,
+        })
+
+        return data
+
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(

rhodecode/config/routing.py

@@ -292,8 +292,6 b' def make_map(config):'
                         controller='admin/users') as m:
         m.connect('users', '/users',
                   action='create', conditions={'method': ['POST']})
-        m.connect('users', '/users',
-                  action='index', conditions={'method': ['GET']})
         m.connect('new_user', '/users/new',
                   action='new', conditions={'method': ['GET']})
         m.connect('update_user', '/users/{user_id}',

rhodecode/controllers/admin/my_account.py

@@ -77,7 +77,7 b' class MyAccountController(BaseController'
         if c.user.username == User.DEFAULT_USER:
             h.flash(_("You can't edit this user since it's"
                       " crucial for entire application"), category='warning')
-            return redirect(url('users'))
+            return redirect(h.route_path('users'))
 
         c.auth_user = AuthUser(
             user_id=c.rhodecode_user.user_id, ip_addr=self.ip_addr)

rhodecode/controllers/admin/users.py

@@ -50,7 +50,6 b' from rhodecode.model.user import UserMod'
 from rhodecode.model.meta import Session
 from rhodecode.model.permission import PermissionModel
 from rhodecode.lib.utils import action_logger
-from rhodecode.lib.ext_json import json
 from rhodecode.lib.utils2 import datetime_to_time, safe_int, AttributeDict
 
 log = logging.getLogger(__name__)
@@ -76,51 +75,6 b' class UsersController(BaseController):'
         ]
         PermissionModel().set_global_permission_choices(c, gettext_translator=_)
 
-    @HasPermissionAllDecorator('hg.admin')
-    def index(self):
-        """GET /users: All items in the collection"""
-        # url('users')
-
-        from rhodecode.lib.utils import PartialRenderer
-        _render = PartialRenderer('data_table/_dt_elements.mako')
-
-        def username(user_id, username):
-            return _render("user_name", user_id, username)
-
-        def user_actions(user_id, username):
-            return _render("user_actions", user_id, username)
-
-        # json generate
-        c.users_list = User.query()\
-            .filter(User.username != User.DEFAULT_USER) \
-            .all()
-
-        users_data = []
-        for user in c.users_list:
-            users_data.append({
-                "username": h.gravatar_with_user(user.username),
-                "username_raw": user.username,
-                "email": user.email,
-                "first_name": h.escape(user.name),
-                "last_name": h.escape(user.lastname),
-                "last_login": h.format_date(user.last_login),
-                "last_login_raw": datetime_to_time(user.last_login),
-                "last_activity": h.format_date(
-                    h.time_to_datetime(user.user_data.get('last_activity', 0))),
-                "last_activity_raw": user.user_data.get('last_activity', 0),
-                "active": h.bool2icon(user.active),
-                "active_raw": user.active,
-                "admin": h.bool2icon(user.admin),
-                "admin_raw": user.admin,
-                "extern_type": user.extern_type,
-                "extern_name": user.extern_name,
-                "action": user_actions(user.user_id, user.username),
-            })
-
-
-        c.data = json.dumps(users_data)
-        return render('admin/users/users.mako')
-
     def _get_personal_repo_group_template_vars(self):
         DummyUser = AttributeDict({
             'username': '${username}',
@@ -135,7 +89,6 b' class UsersController(BaseController):'
     @auth.CSRFRequired()
     def create(self):
         """POST /users: Create a new item"""
-        # url('users')
         c.default_extern_type = auth_rhodecode.RhodeCodeAuthPlugin.name
         user_model = UserModel()
         user_form = UserForm()()
@@ -168,7 +121,7 b' class UsersController(BaseController):'
             log.exception("Exception creation of user")
             h.flash(_('Error occurred during creation of user %s')
                     % request.POST.get('username'), category='error')
-        return redirect(url('users'))
+        return redirect(h.route_path('users'))
 
     @HasPermissionAllDecorator('hg.admin')
     def new(self):
@@ -312,7 +265,7 b' class UsersController(BaseController):'
             log.exception("Exception during deletion of user")
             h.flash(_('An error occurred during deletion of user'),
                     category='error')
-        return redirect(url('users'))
+        return redirect(h.route_path('users'))
 
     @HasPermissionAllDecorator('hg.admin')
     @auth.CSRFRequired()
@@ -404,7 +357,7 b' class UsersController(BaseController):'
         c.user = User.get_or_404(user_id)
         if c.user.username == User.DEFAULT_USER:
             h.flash(_("You can't edit this user"), category='warning')
-            return redirect(url('users'))
+            return redirect(h.route_path('users'))
 
         c.active = 'profile'
         c.extern_type = c.user.extern_type
@@ -425,7 +378,7 b' class UsersController(BaseController):'
         user = c.user = User.get_or_404(user_id)
         if user.username == User.DEFAULT_USER:
             h.flash(_("You can't edit this user"), category='warning')
-            return redirect(url('users'))
+            return redirect(h.route_path('users'))
 
         c.active = 'advanced'
         c.perm_user = AuthUser(user_id=user_id, ip_addr=self.ip_addr)
@@ -457,7 +410,7 b' class UsersController(BaseController):'
         c.user = User.get_or_404(user_id)
         if c.user.username == User.DEFAULT_USER:
             h.flash(_("You can't edit this user"), category='warning')
-            return redirect(url('users'))
+            return redirect(h.route_path('users'))
 
         c.active = 'global_perms'
 
@@ -531,7 +484,7 b' class UsersController(BaseController):'
         c.user = User.get_or_404(user_id)
         if c.user.username == User.DEFAULT_USER:
             h.flash(_("You can't edit this user"), category='warning')
-            return redirect(url('users'))
+            return redirect(h.route_path('users'))
 
         c.active = 'perms_summary'
         c.perm_user = AuthUser(user_id=user_id, ip_addr=self.ip_addr)
@@ -544,7 +497,7 b' class UsersController(BaseController):'
         c.user = User.get_or_404(user_id)
         if c.user.username == User.DEFAULT_USER:
             h.flash(_("You can't edit this user"), category='warning')
-            return redirect(url('users'))
+            return redirect(h.route_path('users'))
 
         c.active = 'emails'
         c.user_email_map = UserEmailMap.query() \
@@ -602,7 +555,7 b' class UsersController(BaseController):'
         c.user = User.get_or_404(user_id)
         if c.user.username == User.DEFAULT_USER:
             h.flash(_("You can't edit this user"), category='warning')
-            return redirect(url('users'))
+            return redirect(h.route_path('users'))
 
         c.active = 'ips'
         c.user_ip_map = UserIpMap.query() \

rhodecode/templates/admin/users/user_add.mako

@@ -10,7 +10,7 b''
 <%def name="breadcrumbs_links()">
     ${h.link_to(_('Admin'),h.url('admin_home'))}
     &raquo;
-    ${h.link_to(_('Users'),h.url('users'))}
+    ${h.link_to(_('Users'),h.route_path('users'))}
     &raquo;
     ${_('Add User')}
 </%def>

rhodecode/templates/admin/users/user_edit.mako

@@ -11,7 +11,7 b''
 <%def name="breadcrumbs_links()">
     ${h.link_to(_('Admin'),h.url('admin_home'))}
     &raquo;
-    ${h.link_to(_('Users'),h.url('users'))}
+    ${h.link_to(_('Users'),h.route_path('users'))}
     &raquo;
     ${c.user.username}
 </%def>

rhodecode/templates/admin/users/users.mako

@@ -34,60 +34,31 b''
     </div>
 </div>
 
-<script>
+<script type="text/javascript">
+
 $(document).ready(function() {
 
-    var get_datatable_count = function(){
-      var datatable = $('#user_list_table').dataTable();
-      var api = datatable.api();
-      var total = api.page.info().recordsDisplay;
-      var active = datatable.fnGetFilteredData();
-      var _text = _gettext("{0} active out of {1} users").format(active, total);
-      $('#user_count').text(_text);
-    };
-
-    // custom filter that filters by username OR email
-    $.fn.dataTable.ext.search.push(
-        function( settings, data, dataIndex ) {
-            var query = $('#q_filter').val();
-
-            var username = data[0];
-            var email = data[1];
-            var first_name = data[2];
-            var last_name = data[3];
+    var getDatatableCount = function(){
+      var table = $('#user_list_table').dataTable();
+      var page = table.api().page.info();
+      var  active = page.recordsDisplay;
+      var  total = page.recordsTotal;
 
-            var query_str = username + " " +
-                            email + " " +
-                            first_name + " " +
-                            last_name;
-
-            if ((query_str).indexOf(query) !== -1) {
-                return true;
-            }
-            return false;
-        }
-    );
-    // filtered data plugin
-    $.fn.dataTableExt.oApi.fnGetFilteredData = function ( oSettings ) {
-        var res = [];
-        for ( var i=0, iLen=oSettings.fnRecordsDisplay() ; i<iLen ; i++ ) {
-            var record = oSettings.aoData[i]._aData;
-            if(record['active_raw']){
-                res.push(record);
-            }
-        }
-        return res.length;
+      var _text = _gettext("{0} out of {1} users").format(active, total);
+      $('#user_count').text(_text);
     };
 
     // user list
     $('#user_list_table').DataTable({
-      data: ${c.data|n},
+      processing: false,
+      serverSide: true,
+      ajax: "${h.route_path('users_data')}",
       dom: 'rtp',
       pageLength: ${c.visual.admin_grid_items},
-      order: [[ 1, "asc" ]],
+      order: [[ 0, "asc" ]],
       columns: [
          { data: {"_": "username",
-                  "sort": "username_raw"}, title: "${_('Username')}", className: "td-user" },
+                  "sort": "username"}, title: "${_('Username')}", className: "td-user" },
          { data: {"_": "email",
                   "sort": "email"}, title: "${_('Email')}", className: "td-email"  },
          { data: {"_": "first_name",
@@ -95,12 +66,12 b''
          { data: {"_": "last_name",
                   "sort": "last_name"}, title: "${_('Last Name')}", className: "td-user" },
          { data: {"_": "last_activity",
-                  "sort": "last_activity_raw",
-                  "type": Number}, title: "${_('Last activity')}", className: "td-time" },
+                  "sort": "last_activity",
+                  "type": Number}, title: "${_('Last activity')}", className: "td-time", orderable: false },
          { data: {"_": "active",
-                  "sort": "active_raw"}, title: "${_('Active')}", className: "td-active" },
+                  "sort": "active"}, title: "${_('Active')}", className: "td-active" },
          { data: {"_": "admin",
-                  "sort": "admin_raw"}, title: "${_('Admin')}", className: "td-admin" },
+                  "sort": "admin"}, title: "${_('Admin')}", className: "td-admin" },
          { data: {"_": "extern_type",
                   "sort": "extern_type"}, title: "${_('Auth type')}", className: "td-type"  },
          { data: {"_": "action",
@@ -110,9 +81,7 b''
           paginate: DEFAULT_GRID_PAGINATION,
           emptyTable: _gettext("No users available yet.")
       },
-      "initComplete": function( settings, json ) {
-          get_datatable_count();
-      },
+
       "createdRow": function ( row, data, index ) {
           if (!data['active_raw']){
             $(row).addClass('closed')
@@ -120,23 +89,29 b''
       }
     });
 
-    // update the counter when doing search
-    $('#user_list_table').on( 'search.dt', function (e,settings) {
-      get_datatable_count();
+    $('#user_list_table').on('xhr.dt', function(e, settings, json, xhr){
+        $('#user_list_table').css('opacity', 1);
+    });
+
+    $('#user_list_table').on('preXhr.dt', function(e, settings, data){
+        $('#user_list_table').css('opacity', 0.3);
     });
 
-    // filter, filter both grids
-    $('#q_filter').on( 'keyup', function () {
-      var user_api = $('#user_list_table').dataTable().api();
-      user_api
-        .draw();
+    // refresh counters on draw
+    $('#user_list_table').on('draw.dt', function(){
+        getDatatableCount();
     });
 
-    // refilter table if page load via back button
-    $("#q_filter").trigger('keyup');
+    // filter
+    $('#q_filter').on('keyup',
+        $.debounce(250, function() {
+            $('#user_list_table').DataTable().search(
+                $('#q_filter').val()
+            ).draw();
+        })
+    );
 
   });
-
 </script>
 
 </%def>

rhodecode/templates/base/base.mako

@@ -75,7 +75,7 b''
       <li><a href="${h.url('admin_home')}">${_('Admin journal')}</a></li>
       <li><a href="${h.url('repos')}">${_('Repositories')}</a></li>
       <li><a href="${h.url('repo_groups')}">${_('Repository groups')}</a></li>
-      <li><a href="${h.url('users')}">${_('Users')}</a></li>
+      <li><a href="${h.route_path('users')}">${_('Users')}</a></li>
       <li><a href="${h.url('users_groups')}">${_('User groups')}</a></li>
       <li><a href="${h.url('admin_permissions_application')}">${_('Permissions')}</a></li>
       <li><a href="${h.route_path('auth_home', traverse='')}">${_('Authentication')}</a></li>

rhodecode/tests/functional/test_admin_users.py

@@ -36,6 +36,20 b' from rhodecode.tests.utils import Assert'
 fixture = Fixture()
 
 
+def route_path(name, params=None, **kwargs):
+    import urllib
+    from rhodecode.apps._base import ADMIN_PREFIX
+
+    base_url = {
+        'users_data':
+            ADMIN_PREFIX + '/users_data',
+    }[name].format(**kwargs)
+
+    if params:
+        base_url = '{}?{}'.format(base_url, urllib.urlencode(params))
+    return base_url
+
+
 class TestAdminUsersController(TestController):
     test_user_1 = 'testme'
     destroy_users = set()
@@ -44,7 +58,7 b' class TestAdminUsersController(TestContr'
     def teardown_method(cls, method):
         fixture.destroy_users(cls.destroy_users)
 
-    def test_create(self):
+    def test_create(self, xhr_header):
         self.log_user()
         username = 'newtestuser'
         password = 'test12'
@@ -53,7 +67,7 b' class TestAdminUsersController(TestContr'
         lastname = 'lastname'
         email = '[e-mail]'
 
-        response = self.app.get(url('new_user'))
+        self.app.get(url('new_user'))
 
         response = self.app.post(url('users'), params={
             'username': username,
@@ -81,8 +95,8 b' class TestAdminUsersController(TestContr'
         assert new_user.lastname == lastname
         assert new_user.email == email
 
-        response.follow()
-        response = response.follow()
+        response = self.app.get(route_path('users_data'),
+                                extra_environ=xhr_header)
         response.mustcontain(username)
 
     def test_create_err(self):
@@ -93,7 +107,7 b' class TestAdminUsersController(TestContr'
         lastname = 'lastname'
         email = 'errmail.com'
 
-        response = self.app.get(url('new_user'))
+        self.app.get(url('new_user'))
 
         response = self.app.post(url('users'), params={
             'username': username,