admin: made all grids use same partial loading logic

- dont load all results for repositories in admin
- speed up loading admin repos page by narrowing results based on permissions
- some small speed ups and code optimizations on permission loading

rhodecode/apps/admin/__init__.py

@@ -140,7 +140,6 b' def admin_routes(config):'
         name='admin_settings_visual_update',
         pattern='/settings/visual/update')
 
-
     config.add_route(
         name='admin_settings_issuetracker',
         pattern='/settings/issue-tracker')
@@ -415,6 +414,10 b' def admin_routes(config):'
         pattern='/repos')
 
     config.add_route(
+        name='repos_data',
+        pattern='/repos_data')
+
+    config.add_route(
         name='repo_new',
         pattern='/repos/new')
 

rhodecode/apps/admin/views/repo_groups.py

@@ -65,7 +65,7 b' class AdminRepoGroupsView(BaseAppView, D'
         # and display only those we have ADMIN right
         groups_with_admin_rights = RepoGroupList(
             RepoGroup.query().all(),
-            perm_set=['group.admin'])
+            perm_set=['group.admin'], extra_kwargs=dict(user=self._rhodecode_user))
         c.repo_groups = RepoGroup.groups_choices(
             groups=groups_with_admin_rights,
             show_empty_group=allow_empty_group)
@@ -150,12 +150,8 b' class AdminRepoGroupsView(BaseAppView, D'
         def user_profile(username):
             return _render('user_profile', username)
 
-        auth_repo_group_list = RepoGroupList(
-            RepoGroup.query().all(), perm_set=['group.admin'])
-
-        allowed_ids = [-1]
-        for repo_group in auth_repo_group_list:
-                allowed_ids.append(repo_group.group_id)
+        _perms = ['group.admin']
+        allowed_ids = [-1] + self._rhodecode_user.repo_group_acl_ids_from_stack(_perms)
 
         repo_groups_data_total_count = RepoGroup.query()\
             .filter(or_(

rhodecode/apps/admin/views/repositories.py

@@ -31,7 +31,6 b' from rhodecode import events'
 from rhodecode.apps._base import BaseAppView, DataGridAppView
 from rhodecode.lib.celerylib.utils import get_task_id
 
-from rhodecode.lib.ext_json import json
 from rhodecode.lib.auth import (
     LoginRequired, CSRFRequired, NotAnonymous,
     HasPermissionAny, HasRepoGroupPermissionAny)
@@ -43,7 +42,8 b' from rhodecode.model.permission import P'
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.scm import RepoList, RepoGroupList, ScmModel
 from rhodecode.model.settings import SettingsModel
-from rhodecode.model.db import Repository, RepoGroup
+from rhodecode.model.db import (
+    in_filter_generator, or_, func, Session, Repository, RepoGroup, User)
 
 log = logging.getLogger(__name__)
 
@@ -70,15 +70,94 b' class AdminReposView(BaseAppView, DataGr'
         renderer='rhodecode:templates/admin/repos/repos.mako')
     def repository_list(self):
         c = self.load_default_context()
+        return self._get_template_context(c)
 
-        repo_list = Repository.get_all_repos()
-        c.repo_list = RepoList(repo_list, perm_set=['repository.admin'])
+    @LoginRequired()
+    @NotAnonymous()
+    # perms check inside
+    @view_config(
+        route_name='repos_data', request_method='GET',
+        renderer='json_ext', xhr=True)
+    def repository_list_data(self):
+        self.load_default_context()
+        column_map = {
+            'name_raw': 'repo_name',
+            'desc': 'description',
+            'last_change_raw': 'updated_on',
+            'owner': 'user_username',
+        }
+        draw, start, limit = self._extract_chunk(self.request)
+        search_q, order_by, order_dir = self._extract_ordering(
+            self.request, column_map=column_map)
+
+        _perms = ['repository.admin']
+        allowed_ids = [-1] + self._rhodecode_user.repo_acl_ids_from_stack(_perms)
+
+        repos_data_total_count = Repository.query() \
+            .filter(or_(
+                # generate multiple IN to fix limitation problems
+                *in_filter_generator(Repository.repo_id, allowed_ids))
+            ) \
+            .count()
+
+        base_q = Session.query(
+            Repository.repo_id,
+            Repository.repo_name,
+            Repository.description,
+            Repository.repo_type,
+            Repository.repo_state,
+            Repository.private,
+            Repository.archived,
+            Repository.fork,
+            Repository.updated_on,
+            Repository._changeset_cache,
+            User,
+            ) \
+            .filter(or_(
+                # generate multiple IN to fix limitation problems
+                *in_filter_generator(Repository.repo_id, allowed_ids))
+            ) \
+            .join(User, User.user_id == Repository.user_id) \
+            .group_by(Repository, User)
+
+        if search_q:
+            like_expression = u'%{}%'.format(safe_unicode(search_q))
+            base_q = base_q.filter(or_(
+                Repository.repo_name.ilike(like_expression),
+            ))
+
+        repos_data_total_filtered_count = base_q.count()
+
+        sort_defined = False
+        if order_by == 'repo_name':
+            sort_col = func.lower(Repository.repo_name)
+            sort_defined = True
+        elif order_by == 'user_username':
+            sort_col = User.username
+        else:
+            sort_col = getattr(Repository, order_by, None)
+
+        if sort_defined or sort_col:
+            if order_dir == 'asc':
+                sort_col = sort_col.asc()
+            else:
+                sort_col = sort_col.desc()
+
+        base_q = base_q.order_by(sort_col)
+        base_q = base_q.offset(start).limit(limit)
+
+        repos_list = base_q.all()
+
         repos_data = RepoModel().get_repos_as_dict(
-            repo_list=c.repo_list, admin=True, super_user_actions=True)
-        # json used to render the grid
-        c.data = json.dumps(repos_data)
+            repo_list=repos_list, admin=True, super_user_actions=True)
 
-        return self._get_template_context(c)
+        data = ({
+            'draw': draw,
+            'data': repos_data,
+            'recordsTotal': repos_data_total_count,
+            'recordsFiltered': repos_data_total_filtered_count,
+        })
+        return data
 
     @LoginRequired()
     @NotAnonymous()

rhodecode/apps/admin/views/user_groups.py

@@ -99,12 +99,8 b' class AdminUserGroupsView(BaseAppView, D'
         def user_profile(username):
             return _render('user_profile', username)
 
-        auth_user_group_list = UserGroupList(
-            UserGroup.query().all(), perm_set=['usergroup.admin'])
-
-        allowed_ids = [-1]
-        for user_group in auth_user_group_list:
-                allowed_ids.append(user_group.users_group_id)
+        _perms = ['usergroup.admin']
+        allowed_ids = [-1] + self._rhodecode_user.user_group_acl_ids_from_stack(_perms)
 
         user_groups_data_total_count = UserGroup.query()\
             .filter(or_(

rhodecode/lib/auth.py

@@ -352,23 +352,23 b' class PermOriginDict(dict):'
     `.perm_origin_stack` will return the stack of (perm, origin) set per key
 
     >>> perms = PermOriginDict()
-    >>> perms['resource'] = 'read', 'default'
+    >>> perms['resource'] = 'read', 'default', 1
     >>> perms['resource']
     'read'
-    >>> perms['resource'] = 'write', 'admin'
+    >>> perms['resource'] = 'write', 'admin', 2
     >>> perms['resource']
     'write'
     >>> perms.perm_origin_stack
-    {'resource': [('read', 'default'), ('write', 'admin')]}
+    {'resource': [('read', 'default', 1), ('write', 'admin', 2)]}
     """
 
     def __init__(self, *args, **kw):
         dict.__init__(self, *args, **kw)
         self.perm_origin_stack = collections.OrderedDict()
 
-    def __setitem__(self, key, (perm, origin)):
+    def __setitem__(self, key, (perm, origin, obj_id)):
         self.perm_origin_stack.setdefault(key, []).append(
-            (perm, origin))
+            (perm, origin, obj_id))
         dict.__setitem__(self, key, perm)
 
 
@@ -463,26 +463,29 b' class PermissionCalculator(object):'
         # repositories
         for perm in self.default_repo_perms:
             r_k = perm.UserRepoToPerm.repository.repo_name
+            obj_id = perm.UserRepoToPerm.repository.repo_id
             archived = perm.UserRepoToPerm.repository.archived
             p = 'repository.admin'
-            self.permissions_repositories[r_k] = p, PermOrigin.SUPER_ADMIN
+            self.permissions_repositories[r_k] = p, PermOrigin.SUPER_ADMIN, obj_id
             # special case for archived repositories, which we block still even for
             # super admins
             if archived:
                 p = 'repository.read'
-                self.permissions_repositories[r_k] = p, PermOrigin.ARCHIVED
+                self.permissions_repositories[r_k] = p, PermOrigin.ARCHIVED, obj_id
 
         # repository groups
         for perm in self.default_repo_groups_perms:
             rg_k = perm.UserRepoGroupToPerm.group.group_name
+            obj_id = perm.UserRepoGroupToPerm.group.group_id
             p = 'group.admin'
-            self.permissions_repository_groups[rg_k] = p, PermOrigin.SUPER_ADMIN
+            self.permissions_repository_groups[rg_k] = p, PermOrigin.SUPER_ADMIN, obj_id
 
         # user groups
         for perm in self.default_user_group_perms:
             u_k = perm.UserUserGroupToPerm.user_group.users_group_name
+            obj_id = perm.UserUserGroupToPerm.user_group.users_group_id
             p = 'usergroup.admin'
-            self.permissions_user_groups[u_k] = p, PermOrigin.SUPER_ADMIN
+            self.permissions_user_groups[u_k] = p, PermOrigin.SUPER_ADMIN, obj_id
 
         # branch permissions
         # since super-admin also can have custom rule permissions
@@ -578,10 +581,11 b' class PermissionCalculator(object):'
     def _calculate_default_permissions_repositories(self, user_inherit_object_permissions):
         for perm in self.default_repo_perms:
             r_k = perm.UserRepoToPerm.repository.repo_name
+            obj_id = perm.UserRepoToPerm.repository.repo_id
             archived = perm.UserRepoToPerm.repository.archived
             p = perm.Permission.permission_name
             o = PermOrigin.REPO_DEFAULT
-            self.permissions_repositories[r_k] = p, o
+            self.permissions_repositories[r_k] = p, o, obj_id
 
             # if we decide this user isn't inheriting permissions from
             # default user we set him to .none so only explicit
@@ -589,25 +593,25 b' class PermissionCalculator(object):'
             if not user_inherit_object_permissions:
                 p = 'repository.none'
                 o = PermOrigin.REPO_DEFAULT_NO_INHERIT
-                self.permissions_repositories[r_k] = p, o
+                self.permissions_repositories[r_k] = p, o, obj_id
 
             if perm.Repository.private and not (
                     perm.Repository.user_id == self.user_id):
                 # disable defaults for private repos,
                 p = 'repository.none'
                 o = PermOrigin.REPO_PRIVATE
-                self.permissions_repositories[r_k] = p, o
+                self.permissions_repositories[r_k] = p, o, obj_id
 
             elif perm.Repository.user_id == self.user_id:
                 # set admin if owner
                 p = 'repository.admin'
                 o = PermOrigin.REPO_OWNER
-                self.permissions_repositories[r_k] = p, o
+                self.permissions_repositories[r_k] = p, o, obj_id
 
             if self.user_is_admin:
                 p = 'repository.admin'
                 o = PermOrigin.SUPER_ADMIN
-                self.permissions_repositories[r_k] = p, o
+                self.permissions_repositories[r_k] = p, o, obj_id
 
             # finally in case of archived repositories, we downgrade  higher
             # permissions to read
@@ -616,7 +620,7 b' class PermissionCalculator(object):'
                 if current_perm in ['repository.write', 'repository.admin']:
                     p = 'repository.read'
                     o = PermOrigin.ARCHIVED
-                    self.permissions_repositories[r_k] = p, o
+                    self.permissions_repositories[r_k] = p, o, obj_id
 
     def _calculate_default_permissions_repository_branches(self, user_inherit_object_permissions):
         for perm in self.default_branch_repo_perms:
@@ -641,52 +645,54 b' class PermissionCalculator(object):'
     def _calculate_default_permissions_repository_groups(self, user_inherit_object_permissions):
         for perm in self.default_repo_groups_perms:
             rg_k = perm.UserRepoGroupToPerm.group.group_name
+            obj_id = perm.UserRepoGroupToPerm.group.group_id
             p = perm.Permission.permission_name
             o = PermOrigin.REPOGROUP_DEFAULT
-            self.permissions_repository_groups[rg_k] = p, o
+            self.permissions_repository_groups[rg_k] = p, o, obj_id
 
             # if we decide this user isn't inheriting permissions from default
             # user we set him to .none so only explicit permissions work
             if not user_inherit_object_permissions:
                 p = 'group.none'
                 o = PermOrigin.REPOGROUP_DEFAULT_NO_INHERIT
-                self.permissions_repository_groups[rg_k] = p, o
+                self.permissions_repository_groups[rg_k] = p, o, obj_id
 
             if perm.RepoGroup.user_id == self.user_id:
                 # set admin if owner
                 p = 'group.admin'
                 o = PermOrigin.REPOGROUP_OWNER
-                self.permissions_repository_groups[rg_k] = p, o
+                self.permissions_repository_groups[rg_k] = p, o, obj_id
 
             if self.user_is_admin:
                 p = 'group.admin'
                 o = PermOrigin.SUPER_ADMIN
-                self.permissions_repository_groups[rg_k] = p, o
+                self.permissions_repository_groups[rg_k] = p, o, obj_id
 
     def _calculate_default_permissions_user_groups(self, user_inherit_object_permissions):
         for perm in self.default_user_group_perms:
             u_k = perm.UserUserGroupToPerm.user_group.users_group_name
+            obj_id = perm.UserUserGroupToPerm.user_group.users_group_id
             p = perm.Permission.permission_name
             o = PermOrigin.USERGROUP_DEFAULT
-            self.permissions_user_groups[u_k] = p, o
+            self.permissions_user_groups[u_k] = p, o, obj_id
 
             # if we decide this user isn't inheriting permissions from default
             # user we set him to .none so only explicit permissions work
             if not user_inherit_object_permissions:
                 p = 'usergroup.none'
                 o = PermOrigin.USERGROUP_DEFAULT_NO_INHERIT
-                self.permissions_user_groups[u_k] = p, o
+                self.permissions_user_groups[u_k] = p, o, obj_id
 
             if perm.UserGroup.user_id == self.user_id:
                 # set admin if owner
                 p = 'usergroup.admin'
                 o = PermOrigin.USERGROUP_OWNER
-                self.permissions_user_groups[u_k] = p, o
+                self.permissions_user_groups[u_k] = p, o, obj_id
 
             if self.user_is_admin:
                 p = 'usergroup.admin'
                 o = PermOrigin.SUPER_ADMIN
-                self.permissions_user_groups[u_k] = p, o
+                self.permissions_user_groups[u_k] = p, o, obj_id
 
     def _calculate_default_permissions(self):
         """
@@ -738,6 +744,7 b' class PermissionCalculator(object):'
         multiple_counter = collections.defaultdict(int)
         for perm in user_repo_perms_from_user_group:
             r_k = perm.UserGroupRepoToPerm.repository.repo_name
+            obj_id = perm.UserGroupRepoToPerm.repository.repo_id
             multiple_counter[r_k] += 1
             p = perm.Permission.permission_name
             o = PermOrigin.REPO_USERGROUP % perm.UserGroupRepoToPerm\
@@ -747,18 +754,18 b' class PermissionCalculator(object):'
                 cur_perm = self.permissions_repositories[r_k]
                 p = self._choose_permission(p, cur_perm)
 
-            self.permissions_repositories[r_k] = p, o
+            self.permissions_repositories[r_k] = p, o, obj_id
 
             if perm.Repository.user_id == self.user_id:
                 # set admin if owner
                 p = 'repository.admin'
                 o = PermOrigin.REPO_OWNER
-                self.permissions_repositories[r_k] = p, o
+                self.permissions_repositories[r_k] = p, o, obj_id
 
             if self.user_is_admin:
                 p = 'repository.admin'
                 o = PermOrigin.SUPER_ADMIN
-                self.permissions_repositories[r_k] = p, o
+                self.permissions_repositories[r_k] = p, o, obj_id
 
         # user explicit permissions for repositories, overrides any specified
         # by the group permission
@@ -766,6 +773,7 b' class PermissionCalculator(object):'
             self.user_id, self.scope_repo_id)
         for perm in user_repo_perms:
             r_k = perm.UserRepoToPerm.repository.repo_name
+            obj_id = perm.UserRepoToPerm.repository.repo_id
             p = perm.Permission.permission_name
             o = PermOrigin.REPO_USER % perm.UserRepoToPerm.user.username
 
@@ -774,18 +782,18 b' class PermissionCalculator(object):'
                     r_k, 'repository.none')
                 p = self._choose_permission(p, cur_perm)
 
-            self.permissions_repositories[r_k] = p, o
+            self.permissions_repositories[r_k] = p, o, obj_id
 
             if perm.Repository.user_id == self.user_id:
                 # set admin if owner
                 p = 'repository.admin'
                 o = PermOrigin.REPO_OWNER
-                self.permissions_repositories[r_k] = p, o
+                self.permissions_repositories[r_k] = p, o, obj_id
 
             if self.user_is_admin:
                 p = 'repository.admin'
                 o = PermOrigin.SUPER_ADMIN
-                self.permissions_repositories[r_k] = p, o
+                self.permissions_repositories[r_k] = p, o, obj_id
 
     def _calculate_repository_branch_permissions(self):
         # user group for repositories permissions
@@ -847,6 +855,7 b' class PermissionCalculator(object):'
         multiple_counter = collections.defaultdict(int)
         for perm in user_repo_group_perms_from_user_group:
             rg_k = perm.UserGroupRepoGroupToPerm.group.group_name
+            obj_id = perm.UserGroupRepoGroupToPerm.group.group_id
             multiple_counter[rg_k] += 1
             o = PermOrigin.REPOGROUP_USERGROUP % perm.UserGroupRepoGroupToPerm\
                 .users_group.users_group_name
@@ -855,24 +864,25 b' class PermissionCalculator(object):'
             if multiple_counter[rg_k] > 1:
                 cur_perm = self.permissions_repository_groups[rg_k]
                 p = self._choose_permission(p, cur_perm)
-            self.permissions_repository_groups[rg_k] = p, o
+            self.permissions_repository_groups[rg_k] = p, o, obj_id
 
             if perm.RepoGroup.user_id == self.user_id:
                 # set admin if owner, even for member of other user group
                 p = 'group.admin'
                 o = PermOrigin.REPOGROUP_OWNER
-                self.permissions_repository_groups[rg_k] = p, o
+                self.permissions_repository_groups[rg_k] = p, o, obj_id
 
             if self.user_is_admin:
                 p = 'group.admin'
                 o = PermOrigin.SUPER_ADMIN
-                self.permissions_repository_groups[rg_k] = p, o
+                self.permissions_repository_groups[rg_k] = p, o, obj_id
 
         # user explicit permissions for repository groups
         user_repo_groups_perms = Permission.get_default_group_perms(
             self.user_id, self.scope_repo_group_id)
         for perm in user_repo_groups_perms:
             rg_k = perm.UserRepoGroupToPerm.group.group_name
+            obj_id = perm.UserRepoGroupToPerm.group.group_id
             o = PermOrigin.REPOGROUP_USER % perm.UserRepoGroupToPerm\
                 .user.username
             p = perm.Permission.permission_name
@@ -881,18 +891,18 b' class PermissionCalculator(object):'
                 cur_perm = self.permissions_repository_groups.get(rg_k, 'group.none')
                 p = self._choose_permission(p, cur_perm)
 
-            self.permissions_repository_groups[rg_k] = p, o
+            self.permissions_repository_groups[rg_k] = p, o, obj_id
 
             if perm.RepoGroup.user_id == self.user_id:
                 # set admin if owner
                 p = 'group.admin'
                 o = PermOrigin.REPOGROUP_OWNER
-                self.permissions_repository_groups[rg_k] = p, o
+                self.permissions_repository_groups[rg_k] = p, o, obj_id
 
             if self.user_is_admin:
                 p = 'group.admin'
                 o = PermOrigin.SUPER_ADMIN
-                self.permissions_repository_groups[rg_k] = p, o
+                self.permissions_repository_groups[rg_k] = p, o, obj_id
 
     def _calculate_user_group_permissions(self):
         """
@@ -905,8 +915,8 b' class PermissionCalculator(object):'
 
         multiple_counter = collections.defaultdict(int)
         for perm in user_group_from_user_group:
-            ug_k = perm.UserGroupUserGroupToPerm\
-                .target_user_group.users_group_name
+            ug_k = perm.UserGroupUserGroupToPerm.target_user_group.users_group_name
+            obj_id = perm.UserGroupUserGroupToPerm.target_user_group.users_group_id
             multiple_counter[ug_k] += 1
             o = PermOrigin.USERGROUP_USERGROUP % perm.UserGroupUserGroupToPerm\
                 .user_group.users_group_name
@@ -916,24 +926,25 b' class PermissionCalculator(object):'
                 cur_perm = self.permissions_user_groups[ug_k]
                 p = self._choose_permission(p, cur_perm)
 
-            self.permissions_user_groups[ug_k] = p, o
+            self.permissions_user_groups[ug_k] = p, o, obj_id
 
             if perm.UserGroup.user_id == self.user_id:
                 # set admin if owner, even for member of other user group
                 p = 'usergroup.admin'
                 o = PermOrigin.USERGROUP_OWNER
-                self.permissions_user_groups[ug_k] = p, o
+                self.permissions_user_groups[ug_k] = p, o, obj_id
 
             if self.user_is_admin:
                 p = 'usergroup.admin'
                 o = PermOrigin.SUPER_ADMIN
-                self.permissions_user_groups[ug_k] = p, o
+                self.permissions_user_groups[ug_k] = p, o, obj_id
 
         # user explicit permission for user groups
         user_user_groups_perms = Permission.get_default_user_group_perms(
             self.user_id, self.scope_user_group_id)
         for perm in user_user_groups_perms:
             ug_k = perm.UserUserGroupToPerm.user_group.users_group_name
+            obj_id = perm.UserUserGroupToPerm.user_group.users_group_id
             o = PermOrigin.USERGROUP_USER % perm.UserUserGroupToPerm\
                 .user.username
             p = perm.Permission.permission_name
@@ -942,18 +953,18 b' class PermissionCalculator(object):'
                 cur_perm = self.permissions_user_groups.get(ug_k, 'usergroup.none')
                 p = self._choose_permission(p, cur_perm)
 
-            self.permissions_user_groups[ug_k] = p, o
+            self.permissions_user_groups[ug_k] = p, o, obj_id
 
             if perm.UserGroup.user_id == self.user_id:
                 # set admin if owner
                 p = 'usergroup.admin'
                 o = PermOrigin.USERGROUP_OWNER
-                self.permissions_user_groups[ug_k] = p, o
+                self.permissions_user_groups[ug_k] = p, o, obj_id
 
             if self.user_is_admin:
                 p = 'usergroup.admin'
                 o = PermOrigin.SUPER_ADMIN
-                self.permissions_user_groups[ug_k] = p, o
+                self.permissions_user_groups[ug_k] = p, o, obj_id
 
     def _choose_permission(self, new_perm, cur_perm):
         new_perm_val = Permission.PERM_WEIGHTS[new_perm]
@@ -1277,6 +1288,18 b' class AuthUser(object):'
             x[0] for x in self.permissions['user_groups'].items()
             if x[1] == 'usergroup.admin']
 
+    def repo_acl_ids_from_stack(self, perms=None, prefix_filter=None, cache=False):
+        if not perms:
+            perms = ['repository.read', 'repository.write', 'repository.admin']
+        allowed_ids = []
+        for k, stack_data in self.permissions['repositories'].perm_origin_stack.items():
+            perm, origin, obj_id = stack_data[-1]  # last item is the current permission
+            if prefix_filter and not k.startswith(prefix_filter):
+                continue
+            if perm in perms:
+                allowed_ids.append(obj_id)
+        return allowed_ids
+
     def repo_acl_ids(self, perms=None, name_filter=None, cache=False):
         """
         Returns list of repository ids that user have access to based on given
@@ -1285,8 +1308,7 b' class AuthUser(object):'
         """
         from rhodecode.model.scm import RepoList
         if not perms:
-            perms = [
-                'repository.read', 'repository.write', 'repository.admin']
+            perms = ['repository.read', 'repository.write', 'repository.admin']
 
         def _cached_repo_acl(user_id, perm_def, _name_filter):
             qry = Repository.query()
@@ -1296,10 +1318,22 b' class AuthUser(object):'
                     Repository.repo_name.ilike(ilike_expression))
 
             return [x.repo_id for x in
-                    RepoList(qry, perm_set=perm_def)]
+                    RepoList(qry, perm_set=perm_def, extra_kwargs={'user': self})]
 
         return _cached_repo_acl(self.user_id, perms, name_filter)
 
+    def repo_group_acl_ids_from_stack(self, perms=None, prefix_filter=None, cache=False):
+        if not perms:
+            perms = ['group.read', 'group.write', 'group.admin']
+        allowed_ids = []
+        for k, stack_data in self.permissions['repositories_groups'].perm_origin_stack.items():
+            perm, origin, obj_id = stack_data[-1]  # last item is the current permission
+            if prefix_filter and not k.startswith(prefix_filter):
+                continue
+            if perm in perms:
+                allowed_ids.append(obj_id)
+        return allowed_ids
+
     def repo_group_acl_ids(self, perms=None, name_filter=None, cache=False):
         """
         Returns list of repository group ids that user have access to based on given
@@ -1308,8 +1342,7 b' class AuthUser(object):'
         """
         from rhodecode.model.scm import RepoGroupList
         if not perms:
-            perms = [
-                'group.read', 'group.write', 'group.admin']
+            perms = ['group.read', 'group.write', 'group.admin']
 
         def _cached_repo_group_acl(user_id, perm_def, _name_filter):
             qry = RepoGroup.query()
@@ -1319,10 +1352,20 b' class AuthUser(object):'
                     RepoGroup.group_name.ilike(ilike_expression))
 
             return [x.group_id for x in
-                    RepoGroupList(qry, perm_set=perm_def)]
+                    RepoGroupList(qry, perm_set=perm_def, extra_kwargs={'user': self})]
 
         return _cached_repo_group_acl(self.user_id, perms, name_filter)
 
+    def user_group_acl_ids_from_stack(self, perms=None, cache=False):
+        if not perms:
+            perms = ['usergroup.read', 'usergroup.write', 'usergroup.admin']
+        allowed_ids = []
+        for k, stack_data in self.permissions['user_groups'].perm_origin_stack.items():
+            perm, origin, obj_id = stack_data[-1]  # last item is the current permission
+            if perm in perms:
+                allowed_ids.append(obj_id)
+        return allowed_ids
+
     def user_group_acl_ids(self, perms=None, name_filter=None, cache=False):
         """
         Returns list of user group ids that user have access to based on given
@@ -1331,8 +1374,7 b' class AuthUser(object):'
         """
         from rhodecode.model.scm import UserGroupList
         if not perms:
-            perms = [
-                'usergroup.read', 'usergroup.write', 'usergroup.admin']
+            perms = ['usergroup.read', 'usergroup.write', 'usergroup.admin']
 
         def _cached_user_group_acl(user_id, perm_def, name_filter):
             qry = UserGroup.query()
@@ -1342,7 +1384,7 b' class AuthUser(object):'
                     UserGroup.users_group_name.ilike(ilike_expression))
 
             return [x.users_group_id for x in
-                    UserGroupList(qry, perm_set=perm_def)]
+                    UserGroupList(qry, perm_set=perm_def, extra_kwargs={'user': self})]
 
         return _cached_user_group_acl(self.user_id, perms, name_filter)
 

rhodecode/model/db.py

@@ -1646,7 +1646,7 b' class Repository(Base, BaseModel):'
         primary_key=True)
     _repo_name = Column(
         "repo_name", Text(), nullable=False, default=None)
-    _repo_name_hash = Column(
+    repo_name_hash = Column(
         "repo_name_hash", String(255), nullable=False, unique=True)
     repo_state = Column("repo_state", String(255), nullable=True)
 
@@ -1772,22 +1772,26 b' class Repository(Base, BaseModel):'
         else:
             self._locked = None
 
-    @hybrid_property
-    def changeset_cache(self):
+    @classmethod
+    def _load_changeset_cache(cls, repo_id, changeset_cache_raw):
         from rhodecode.lib.vcs.backends.base import EmptyCommit
         dummy = EmptyCommit().__json__()
-        if not self._changeset_cache:
-            dummy['source_repo_id'] = self.repo_id
+        if not changeset_cache_raw:
+            dummy['source_repo_id'] = repo_id
             return json.loads(json.dumps(dummy))
 
         try:
-            return json.loads(self._changeset_cache)
+            return json.loads(changeset_cache_raw)
         except TypeError:
             return dummy
         except Exception:
             log.error(traceback.format_exc())
             return dummy
 
+    @hybrid_property
+    def changeset_cache(self):
+        return self._load_changeset_cache(self.repo_id, self._changeset_cache)
+
     @changeset_cache.setter
     def changeset_cache(self, val):
         try:
@@ -1802,7 +1806,7 b' class Repository(Base, BaseModel):'
     @repo_name.setter
     def repo_name(self, value):
         self._repo_name = value
-        self._repo_name_hash = hashlib.sha1(safe_str(value)).hexdigest()
+        self.repo_name_hash = hashlib.sha1(safe_str(value)).hexdigest()
 
     @classmethod
     def normalize_repo_name(cls, repo_name):
@@ -2239,17 +2243,21 b' class Repository(Base, BaseModel):'
     def last_commit_cache_update_diff(self):
         return time.time() - (safe_int(self.changeset_cache.get('updated_on')) or 0)
 
-    @property
-    def last_commit_change(self):
+    @classmethod
+    def _load_commit_change(cls, last_commit_cache):
         from rhodecode.lib.vcs.utils.helpers import parse_datetime
         empty_date = datetime.datetime.fromtimestamp(0)
-        date_latest = self.changeset_cache.get('date', empty_date)
+        date_latest = last_commit_cache.get('date', empty_date)
         try:
             return parse_datetime(date_latest)
         except Exception:
             return empty_date
 
     @property
+    def last_commit_change(self):
+        return self._load_commit_change(self.changeset_cache)
+
+    @property
     def last_db_change(self):
         return self.updated_on
 
@@ -2598,8 +2606,7 b' class RepoGroup(Base, BaseModel):'
     created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     updated_on = Column('updated_on', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now)
     personal = Column('personal', Boolean(), nullable=True, unique=None, default=None)
-    _changeset_cache = Column(
-        "changeset_cache", LargeBinary(), nullable=True)  # JSON data
+    _changeset_cache = Column("changeset_cache", LargeBinary(), nullable=True)  # JSON data
 
     repo_group_to_perm = relationship('UserRepoGroupToPerm', cascade='all', order_by='UserRepoGroupToPerm.group_to_perm_id')
     users_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all')
@@ -2627,22 +2634,26 b' class RepoGroup(Base, BaseModel):'
         self._group_name = value
         self.group_name_hash = self.hash_repo_group_name(value)
 
-    @hybrid_property
-    def changeset_cache(self):
+    @classmethod
+    def _load_changeset_cache(cls, repo_id, changeset_cache_raw):
         from rhodecode.lib.vcs.backends.base import EmptyCommit
         dummy = EmptyCommit().__json__()
-        if not self._changeset_cache:
-            dummy['source_repo_id'] = ''
+        if not changeset_cache_raw:
+            dummy['source_repo_id'] = repo_id
             return json.loads(json.dumps(dummy))
 
         try:
-            return json.loads(self._changeset_cache)
+            return json.loads(changeset_cache_raw)
         except TypeError:
             return dummy
         except Exception:
             log.error(traceback.format_exc())
             return dummy
 
+    @hybrid_property
+    def changeset_cache(self):
+        return self._load_changeset_cache('', self._changeset_cache)
+
     @changeset_cache.setter
     def changeset_cache(self, val):
         try:
@@ -2771,17 +2782,21 b' class RepoGroup(Base, BaseModel):'
     def last_commit_cache_update_diff(self):
         return time.time() - (safe_int(self.changeset_cache.get('updated_on')) or 0)
 
-    @property
-    def last_commit_change(self):
+    @classmethod
+    def _load_commit_change(cls, last_commit_cache):
         from rhodecode.lib.vcs.utils.helpers import parse_datetime
         empty_date = datetime.datetime.fromtimestamp(0)
-        date_latest = self.changeset_cache.get('date', empty_date)
+        date_latest = last_commit_cache.get('date', empty_date)
         try:
             return parse_datetime(date_latest)
         except Exception:
             return empty_date
 
     @property
+    def last_commit_change(self):
+        return self._load_commit_change(self.changeset_cache)
+
+    @property
     def last_db_change(self):
         return self.updated_on
 

rhodecode/model/scm.py

@@ -149,8 +149,7 b' class RepoList(_PermCheckIterator):'
 
     def __init__(self, db_repo_list, perm_set=None, extra_kwargs=None):
         if not perm_set:
-            perm_set = [
-                'repository.read', 'repository.write', 'repository.admin']
+            perm_set = ['repository.read', 'repository.write', 'repository.admin']
 
         super(RepoList, self).__init__(
             obj_list=db_repo_list,

rhodecode/public/js/rhodecode/routes.js

@@ -133,6 +133,7 b' function registerRCRoutes() {'
     pyroutes.register('user_groups_new', '/_admin/user_groups/new', []);
     pyroutes.register('user_groups_create', '/_admin/user_groups/create', []);
     pyroutes.register('repos', '/_admin/repos', []);
+    pyroutes.register('repos_data', '/_admin/repos_data', []);
     pyroutes.register('repo_new', '/_admin/repos/new', []);
     pyroutes.register('repo_create', '/_admin/repos/create', []);
     pyroutes.register('repo_groups', '/_admin/repo_groups', []);

rhodecode/templates/admin/repo_groups/repo_groups.mako

@@ -23,7 +23,7 b''
 
     <div class="title">
         <input class="q_filter_box" id="q_filter" size="15" type="text" name="filter" placeholder="${_('quick filter...')}" value=""/>
-        <span id="repo_group_count">0</span> ${_('repository groups')}
+        <span id="repo_group_count"></span>
 
         <ul class="links">
         %if c.can_create_repo_group:
@@ -108,6 +108,7 b''
             ).draw();
         })
     );
+
 });
 
 </script>

rhodecode/templates/admin/repos/repos.mako

@@ -23,7 +23,7 b''
 
     <div class="title">
         <input class="q_filter_box" id="q_filter" size="15" type="text" name="filter" placeholder="${_('quick filter...')}" value=""/>
-        <span id="repo_count">0</span> ${_('repositories')}
+        <span id="repo_count"></span>
 
         <ul class="links">
         %if c.can_create_repo:
@@ -36,68 +36,112 b''
     <div id="repos_list_wrap">
         <table id="repo_list_table" class="display"></table>
     </div>
+
 </div>
 
 <script>
 $(document).ready(function() {
-
-    var get_datatable_count = function(){
-      var api = $('#repo_list_table').dataTable().api();
-      $('#repo_count').text(api.page.info().recordsDisplay);
-    };
-
+    var $repoListTable = $('#repo_list_table');
 
     // repo list
-    $('#repo_list_table').DataTable({
-      data: ${c.data|n},
+    $repoListTable.DataTable({
+        processing: true,
+        serverSide: true,
+        ajax: {
+            "url": "${h.route_path('repos_data')}",
+            "dataSrc": function (json) {
+                var filteredCount = json.recordsFiltered;
+                var total = json.recordsTotal;
+
+                var _text = _gettext(
+                        "{0} of {1} repositories").format(
+                        filteredCount, total);
+
+                if (total === filteredCount) {
+                    _text = _gettext("{0} repositories").format(total);
+                }
+                $('#repo_count').text(_text);
+
+                return json.data;
+            },
+        },
       dom: 'rtp',
       pageLength: ${c.visual.admin_grid_items},
       order: [[ 0, "asc" ]],
       columns: [
-         { data: {"_": "name",
-                  "sort": "name_raw"}, title: "${_('Name')}", className: "td-componentname" },
-         { data: 'menu', "bSortable": false, className: "quick_repo_menu" },
-         { data: {"_": "desc",
-                  "sort": "desc"}, title: "${_('Description')}", className: "td-description" },
-         { data: {"_": "last_change",
+            {
+                data: {
+                    "_": "name",
+                    "sort": "name_raw"
+                }, title: "${_('Name')}", className: "td-componentname"
+            },
+            {
+                data: 'menu', "bSortable": false, className: "quick_repo_menu"},
+            {
+                data: {
+                    "_": "desc",
+                    "sort": "desc"
+                }, title: "${_('Description')}", className: "td-description"
+            },
+            {
+                data: {
+                    "_": "last_change",
                   "sort": "last_change_raw",
-                  "type": Number}, title: "${_('Last Change')}", className: "td-time" },
-         { data: {"_": "last_changeset",
+                    "type": Number
+                }, title: "${_('Last Change')}", className: "td-time"
+            },
+            {
+                data: {
+                    "_": "last_changeset",
                   "sort": "last_changeset_raw",
-                  "type": Number}, title: "${_('Commit')}", className: "td-commit" },
-         { data: {"_": "owner",
-                  "sort": "owner"}, title: "${_('Owner')}", className: "td-user" },
-         { data: {"_": "state",
-                  "sort": "state"}, title: "${_('State')}", className: "td-tags td-state" },
-         { data: {"_": "action",
-                  "sort": "action"}, title: "${_('Action')}", className: "td-action", orderable: false }
+                    "type": Number
+                }, title: "${_('Commit')}", className: "td-commit", orderable: false
+            },
+            {
+                data: {
+                    "_": "owner",
+                    "sort": "owner"
+                }, title: "${_('Owner')}", className: "td-user"
+            },
+            {
+                data: {
+                    "_": "state",
+                    "sort": "state"
+                }, title: "${_('State')}", className: "td-tags td-state"
+            },
+            {
+                data: {
+                    "_": "action",
+                    "sort": "action"
+                }, title: "${_('Action')}", className: "td-action", orderable: false
+            }
       ],
       language: {
           paginate: DEFAULT_GRID_PAGINATION,
-          emptyTable:_gettext("No repositories available yet.")
+          sProcessing: _gettext('loading...'),
+          emptyTable:_gettext("No repositories present.")
       },
       "initComplete": function( settings, json ) {
-          get_datatable_count();
           quick_repo_menu();
       }
     });
 
-    // update the counter when doing search
-    $('#repo_list_table').on( 'search.dt', function (e,settings) {
-      get_datatable_count();
+    $repoListTable.on('xhr.dt', function(e, settings, json, xhr){
+        $repoListTable.css('opacity', 1);
+    });
+
+    $repoListTable.on('preXhr.dt', function(e, settings, data){
+        $repoListTable.css('opacity', 0.3);
     });
 
-    // filter, filter both grids
-    $('#q_filter').on( 'keyup', function () {
-      var repo_api = $('#repo_list_table').dataTable().api();
-      repo_api
-        .columns(0)
-        .search(this.value)
-        .draw();
-    });
+    $('#q_filter').on('keyup',
+        $.debounce(250, function() {
+            $repoListTable.DataTable().search(
+                $('#q_filter').val()
+            ).draw();
+        })
+    );
 
-    // refilter table if page load via back button
-    $("#q_filter").trigger('keyup');
   });
 
 </script>

rhodecode/templates/base/perms_summary.mako

@@ -278,7 +278,7 b''
                     <td class="td-tags">
                       %if hasattr(permissions[section], 'perm_origin_stack'):
                          <div>
-                         %for i, (perm, origin) in enumerate(reversed(permissions[section].perm_origin_stack[k])):
+                         %for i, (perm, origin, obj_id) in enumerate(reversed(permissions[section].perm_origin_stack[k])):
                          <% _css_class =  i > 0 and 'perm_overriden' or '' %>
                          % if i > 0:
                              <div style="color: #979797">

rhodecode/tests/lib/test_auth.py

@@ -35,29 +35,29 b' from rhodecode.model.user_group import U'
 
 def test_perm_origin_dict():
     pod = auth.PermOriginDict()
-    pod['thing'] = 'read', 'default'
+    pod['thing'] = 'read', 'default', 1
     assert pod['thing'] == 'read'
 
     assert pod.perm_origin_stack == {
-        'thing': [('read', 'default')]}
+        'thing': [('read', 'default', 1)]}
 
-    pod['thing'] = 'write', 'admin'
+    pod['thing'] = 'write', 'admin', 1
     assert pod['thing'] == 'write'
 
     assert pod.perm_origin_stack == {
-        'thing': [('read', 'default'), ('write', 'admin')]}
+        'thing': [('read', 'default', 1), ('write', 'admin', 1)]}
 
-    pod['other'] = 'write', 'default'
+    pod['other'] = 'write', 'default', 8
 
     assert pod.perm_origin_stack == {
-        'other': [('write', 'default')],
-        'thing': [('read', 'default'), ('write', 'admin')]}
+        'other': [('write', 'default', 8)],
+        'thing': [('read', 'default', 1), ('write', 'admin', 1)]}
 
-    pod['other'] = 'none', 'override'
+    pod['other'] = 'none', 'override', 8
 
     assert pod.perm_origin_stack == {
-        'other': [('write', 'default'), ('none', 'override')],
-        'thing': [('read', 'default'), ('write', 'admin')]}
+        'other': [('write', 'default', 8), ('none', 'override', 8)],
+        'thing': [('read', 'default', 1), ('write', 'admin', 1)]}
 
     with pytest.raises(ValueError):
         pod['thing'] = 'read'