grids: columns and sorting fixes

rhodecode/apps/admin/views/repo_groups.py

@@ -1,361 +1,360 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import datetime
 import logging
 import time
 
 import formencode
 import formencode.htmlfill
 
 from pyramid.httpexceptions import HTTPFound, HTTPForbidden
 from pyramid.view import view_config
 from pyramid.renderers import render
 from pyramid.response import Response
 
 from rhodecode import events
 from rhodecode.apps._base import BaseAppView, DataGridAppView
 
 from rhodecode.lib.auth import (
     LoginRequired, CSRFRequired, NotAnonymous,
     HasPermissionAny, HasRepoGroupPermissionAny)
 from rhodecode.lib import helpers as h, audit_logger
 from rhodecode.lib.utils2 import safe_int, safe_unicode, datetime_to_time
 from rhodecode.model.forms import RepoGroupForm
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.scm import RepoGroupList
 from rhodecode.model.db import (
     or_, count, func, in_filter_generator, Session, RepoGroup, User, Repository)
 
 log = logging.getLogger(__name__)
 
 
 class AdminRepoGroupsView(BaseAppView, DataGridAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
 
         return c
 
     def _load_form_data(self, c):
         allow_empty_group = False
 
         if self._can_create_repo_group():
             # we're global admin, we're ok and we can create TOP level groups
             allow_empty_group = True
 
         # override the choices for this form, we need to filter choices
         # and display only those we have ADMIN right
         groups_with_admin_rights = RepoGroupList(
             RepoGroup.query().all(),
             perm_set=['group.admin'], extra_kwargs=dict(user=self._rhodecode_user))
         c.repo_groups = RepoGroup.groups_choices(
             groups=groups_with_admin_rights,
             show_empty_group=allow_empty_group)
 
     def _can_create_repo_group(self, parent_group_id=None):
         is_admin = HasPermissionAny('hg.admin')('group create controller')
         create_repo_group = HasPermissionAny(
             'hg.repogroup.create.true')('group create controller')
         if is_admin or (create_repo_group and not parent_group_id):
             # we're global admin, or we have global repo group create
             # permission
             # we're ok and we can create TOP level groups
             return True
         elif parent_group_id:
             # we check the permission if we can write to parent group
             group = RepoGroup.get(parent_group_id)
             group_name = group.group_name if group else None
             if HasRepoGroupPermissionAny('group.admin')(
                     group_name, 'check if user is an admin of group'):
                 # we're an admin of passed in group, we're ok.
                 return True
             else:
                 return False
         return False
 
     # permission check in data loading of
     # `repo_group_list_data` via RepoGroupList
     @LoginRequired()
     @NotAnonymous()
     @view_config(
         route_name='repo_groups', request_method='GET',
         renderer='rhodecode:templates/admin/repo_groups/repo_groups.mako')
     def repo_group_list(self):
         c = self.load_default_context()
         return self._get_template_context(c)
 
     # permission check inside
     @LoginRequired()
     @NotAnonymous()
     @view_config(
         route_name='repo_groups_data', request_method='GET',
         renderer='json_ext', xhr=True)
     def repo_group_list_data(self):
         self.load_default_context()
         column_map = {
-            'name_raw': 'group_name_hash',
+            'name': 'group_name_hash',
             'desc': 'group_description',
-            'last_change_raw': 'updated_on',
+            'last_change': 'updated_on',
             'top_level_repos': 'repos_total',
             'owner': 'user_username',
         }
         draw, start, limit = self._extract_chunk(self.request)
         search_q, order_by, order_dir = self._extract_ordering(
             self.request, column_map=column_map)
 
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/data_table/_dt_elements.mako')
         c = _render.get_call_context()
 
         def quick_menu(repo_group_name):
             return _render('quick_repo_group_menu', repo_group_name)
 
         def repo_group_lnk(repo_group_name):
             return _render('repo_group_name', repo_group_name)
 
         def last_change(last_change):
             if isinstance(last_change, datetime.datetime) and not last_change.tzinfo:
                 ts = time.time()
                 utc_offset = (datetime.datetime.fromtimestamp(ts)
                               - datetime.datetime.utcfromtimestamp(ts)).total_seconds()
                 last_change = last_change + datetime.timedelta(seconds=utc_offset)
             return _render("last_change", last_change)
 
         def desc(desc, personal):
             return _render(
                 'repo_group_desc', desc, personal, c.visual.stylify_metatags)
 
         def repo_group_actions(repo_group_id, repo_group_name, gr_count):
             return _render(
                 'repo_group_actions', repo_group_id, repo_group_name, gr_count)
 
         def user_profile(username):
             return _render('user_profile', username)
 
         _perms = ['group.admin']
         allowed_ids = [-1] + self._rhodecode_user.repo_group_acl_ids_from_stack(_perms)
 
         repo_groups_data_total_count = RepoGroup.query()\
             .filter(or_(
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(RepoGroup.group_id, allowed_ids)
             )) \
             .count()
 
         repo_groups_data_total_inactive_count = RepoGroup.query()\
             .filter(RepoGroup.group_id.in_(allowed_ids))\
             .count()
 
         repo_count = count(Repository.repo_id)
         base_q = Session.query(
             RepoGroup.group_name,
             RepoGroup.group_name_hash,
             RepoGroup.group_description,
             RepoGroup.group_id,
             RepoGroup.personal,
             RepoGroup.updated_on,
             User,
             repo_count.label('repos_count')
             ) \
             .filter(or_(
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(RepoGroup.group_id, allowed_ids)
             )) \
             .outerjoin(Repository,  Repository.group_id == RepoGroup.group_id) \
             .join(User, User.user_id == RepoGroup.user_id) \
             .group_by(RepoGroup, User)
 
         if search_q:
             like_expression = u'%{}%'.format(safe_unicode(search_q))
             base_q = base_q.filter(or_(
                 RepoGroup.group_name.ilike(like_expression),
             ))
 
         repo_groups_data_total_filtered_count = base_q.count()
         # the inactive isn't really used, but we still make it same as other data grids
         # which use inactive (users,user groups)
         repo_groups_data_total_filtered_inactive_count = repo_groups_data_total_filtered_count
 
         sort_defined = False
         if order_by == 'group_name':
             sort_col = func.lower(RepoGroup.group_name)
             sort_defined = True
         elif order_by == 'repos_total':
             sort_col = repo_count
             sort_defined = True
         elif order_by == 'user_username':
             sort_col = User.username
         else:
             sort_col = getattr(RepoGroup, order_by, None)
 
         if sort_defined or sort_col:
             if order_dir == 'asc':
                 sort_col = sort_col.asc()
             else:
                 sort_col = sort_col.desc()
 
         base_q = base_q.order_by(sort_col)
         base_q = base_q.offset(start).limit(limit)
 
         # authenticated access to user groups
         auth_repo_group_list = base_q.all()
 
         repo_groups_data = []
         for repo_gr in auth_repo_group_list:
             row = {
                 "menu": quick_menu(repo_gr.group_name),
                 "name": repo_group_lnk(repo_gr.group_name),
-                "name_raw": repo_gr.group_name,
+
                 "last_change": last_change(repo_gr.updated_on),
-                "last_change_raw": datetime_to_time(repo_gr.updated_on),
 
                 "last_changeset": "",
                 "last_changeset_raw": "",
 
                 "desc": desc(repo_gr.group_description, repo_gr.personal),
                 "owner": user_profile(repo_gr.User.username),
                 "top_level_repos": repo_gr.repos_count,
                 "action": repo_group_actions(
                     repo_gr.group_id, repo_gr.group_name, repo_gr.repos_count),
 
             }
 
             repo_groups_data.append(row)
 
         data = ({
             'draw': draw,
             'data': repo_groups_data,
             'recordsTotal': repo_groups_data_total_count,
             'recordsTotalInactive': repo_groups_data_total_inactive_count,
             'recordsFiltered': repo_groups_data_total_filtered_count,
             'recordsFilteredInactive': repo_groups_data_total_filtered_inactive_count,
         })
 
         return data
 
     @LoginRequired()
     @NotAnonymous()
     # perm checks inside
     @view_config(
         route_name='repo_group_new', request_method='GET',
         renderer='rhodecode:templates/admin/repo_groups/repo_group_add.mako')
     def repo_group_new(self):
         c = self.load_default_context()
 
         # perm check for admin, create_group perm or admin of parent_group
         parent_group_id = safe_int(self.request.GET.get('parent_group'))
         if not self._can_create_repo_group(parent_group_id):
             raise HTTPForbidden()
 
         self._load_form_data(c)
 
         defaults = {}  # Future proof for default of repo group
         data = render(
             'rhodecode:templates/admin/repo_groups/repo_group_add.mako',
             self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     # perm checks inside
     @view_config(
         route_name='repo_group_create', request_method='POST',
         renderer='rhodecode:templates/admin/repo_groups/repo_group_add.mako')
     def repo_group_create(self):
         c = self.load_default_context()
         _ = self.request.translate
 
         parent_group_id = safe_int(self.request.POST.get('group_parent_id'))
         can_create = self._can_create_repo_group(parent_group_id)
 
         self._load_form_data(c)
         # permissions for can create group based on parent_id are checked
         # here in the Form
         available_groups = map(lambda k: safe_unicode(k[0]), c.repo_groups)
         repo_group_form = RepoGroupForm(
             self.request.translate, available_groups=available_groups,
             can_create_in_root=can_create)()
 
         repo_group_name = self.request.POST.get('group_name')
         try:
             owner = self._rhodecode_user
             form_result = repo_group_form.to_python(dict(self.request.POST))
             copy_permissions = form_result.get('group_copy_permissions')
             repo_group = RepoGroupModel().create(
                 group_name=form_result['group_name_full'],
                 group_description=form_result['group_description'],
                 owner=owner.user_id,
                 copy_permissions=form_result['group_copy_permissions']
             )
             Session().flush()
 
             repo_group_data = repo_group.get_api_data()
             audit_logger.store_web(
                 'repo_group.create', action_data={'data': repo_group_data},
                 user=self._rhodecode_user)
 
             Session().commit()
 
             _new_group_name = form_result['group_name_full']
 
             repo_group_url = h.link_to(
                 _new_group_name,
                 h.route_path('repo_group_home', repo_group_name=_new_group_name))
             h.flash(h.literal(_('Created repository group %s')
                     % repo_group_url), category='success')
 
         except formencode.Invalid as errors:
             data = render(
                 'rhodecode:templates/admin/repo_groups/repo_group_add.mako',
                 self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
         except Exception:
             log.exception("Exception during creation of repository group")
             h.flash(_('Error occurred during creation of repository group %s')
                     % repo_group_name, category='error')
             raise HTTPFound(h.route_path('home'))
 
         affected_user_ids = [self._rhodecode_user.user_id]
         if copy_permissions:
             user_group_perms = repo_group.permissions(expand_from_user_groups=True)
             copy_perms = [perm['user_id'] for perm in user_group_perms]
             # also include those newly created by copy
             affected_user_ids.extend(copy_perms)
         PermissionModel().trigger_permission_flush(affected_user_ids)
 
         raise HTTPFound(
             h.route_path('repo_group_home',
                          repo_group_name=form_result['group_name_full']))

rhodecode/apps/admin/views/repositories.py

@@ -1,266 +1,266 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import formencode
 import formencode.htmlfill
 
 from pyramid.httpexceptions import HTTPFound, HTTPForbidden
 from pyramid.view import view_config
 from pyramid.renderers import render
 from pyramid.response import Response
 
 from rhodecode import events
 from rhodecode.apps._base import BaseAppView, DataGridAppView
 from rhodecode.lib.celerylib.utils import get_task_id
 
 from rhodecode.lib.auth import (
     LoginRequired, CSRFRequired, NotAnonymous,
     HasPermissionAny, HasRepoGroupPermissionAny)
 from rhodecode.lib import helpers as h
 from rhodecode.lib.utils import repo_name_slug
 from rhodecode.lib.utils2 import safe_int, safe_unicode
 from rhodecode.model.forms import RepoForm
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.scm import RepoList, RepoGroupList, ScmModel
 from rhodecode.model.settings import SettingsModel
 from rhodecode.model.db import (
     in_filter_generator, or_, func, Session, Repository, RepoGroup, User)
 
 log = logging.getLogger(__name__)
 
 
 class AdminReposView(BaseAppView, DataGridAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
 
         return c
 
     def _load_form_data(self, c):
         acl_groups = RepoGroupList(RepoGroup.query().all(),
                                    perm_set=['group.write', 'group.admin'])
         c.repo_groups = RepoGroup.groups_choices(groups=acl_groups)
         c.repo_groups_choices = map(lambda k: safe_unicode(k[0]), c.repo_groups)
         c.personal_repo_group = self._rhodecode_user.personal_repo_group
 
     @LoginRequired()
     @NotAnonymous()
     # perms check inside
     @view_config(
         route_name='repos', request_method='GET',
         renderer='rhodecode:templates/admin/repos/repos.mako')
     def repository_list(self):
         c = self.load_default_context()
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     # perms check inside
     @view_config(
         route_name='repos_data', request_method='GET',
         renderer='json_ext', xhr=True)
     def repository_list_data(self):
         self.load_default_context()
         column_map = {
-            'name_raw': 'repo_name',
+            'name': 'repo_name',
             'desc': 'description',
-            'last_change_raw': 'updated_on',
+            'last_change': 'updated_on',
             'owner': 'user_username',
         }
         draw, start, limit = self._extract_chunk(self.request)
         search_q, order_by, order_dir = self._extract_ordering(
             self.request, column_map=column_map)
 
         _perms = ['repository.admin']
         allowed_ids = [-1] + self._rhodecode_user.repo_acl_ids_from_stack(_perms)
 
         repos_data_total_count = Repository.query() \
             .filter(or_(
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(Repository.repo_id, allowed_ids))
             ) \
             .count()
 
         base_q = Session.query(
             Repository.repo_id,
             Repository.repo_name,
             Repository.description,
             Repository.repo_type,
             Repository.repo_state,
             Repository.private,
             Repository.archived,
             Repository.fork,
             Repository.updated_on,
             Repository._changeset_cache,
             User,
             ) \
             .filter(or_(
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(Repository.repo_id, allowed_ids))
             ) \
             .join(User, User.user_id == Repository.user_id) \
             .group_by(Repository, User)
 
         if search_q:
             like_expression = u'%{}%'.format(safe_unicode(search_q))
             base_q = base_q.filter(or_(
                 Repository.repo_name.ilike(like_expression),
             ))
 
         repos_data_total_filtered_count = base_q.count()
 
         sort_defined = False
         if order_by == 'repo_name':
             sort_col = func.lower(Repository.repo_name)
             sort_defined = True
         elif order_by == 'user_username':
             sort_col = User.username
         else:
             sort_col = getattr(Repository, order_by, None)
 
         if sort_defined or sort_col:
             if order_dir == 'asc':
                 sort_col = sort_col.asc()
             else:
                 sort_col = sort_col.desc()
 
         base_q = base_q.order_by(sort_col)
         base_q = base_q.offset(start).limit(limit)
 
         repos_list = base_q.all()
 
         repos_data = RepoModel().get_repos_as_dict(
             repo_list=repos_list, admin=True, super_user_actions=True)
 
         data = ({
             'draw': draw,
             'data': repos_data,
             'recordsTotal': repos_data_total_count,
             'recordsFiltered': repos_data_total_filtered_count,
         })
         return data
 
     @LoginRequired()
     @NotAnonymous()
     # perms check inside
     @view_config(
         route_name='repo_new', request_method='GET',
         renderer='rhodecode:templates/admin/repos/repo_add.mako')
     def repository_new(self):
         c = self.load_default_context()
 
         new_repo = self.request.GET.get('repo', '')
         parent_group = safe_int(self.request.GET.get('parent_group'))
         _gr = RepoGroup.get(parent_group)
 
         if not HasPermissionAny('hg.admin', 'hg.create.repository')():
             # you're not super admin nor have global create permissions,
             # but maybe you have at least write permission to a parent group ?
 
             gr_name = _gr.group_name if _gr else None
             # create repositories with write permission on group is set to true
             create_on_write = HasPermissionAny('hg.create.write_on_repogroup.true')()
             group_admin = HasRepoGroupPermissionAny('group.admin')(group_name=gr_name)
             group_write = HasRepoGroupPermissionAny('group.write')(group_name=gr_name)
             if not (group_admin or (group_write and create_on_write)):
                 raise HTTPForbidden()
 
         self._load_form_data(c)
         c.new_repo = repo_name_slug(new_repo)
 
         # apply the defaults from defaults page
         defaults = SettingsModel().get_default_repo_settings(strip_prefix=True)
         # set checkbox to autochecked
         defaults['repo_copy_permissions'] = True
 
         parent_group_choice = '-1'
         if not self._rhodecode_user.is_admin and self._rhodecode_user.personal_repo_group:
             parent_group_choice = self._rhodecode_user.personal_repo_group
 
         if parent_group and _gr:
             if parent_group in [x[0] for x in c.repo_groups]:
                 parent_group_choice = safe_unicode(parent_group)
 
         defaults.update({'repo_group': parent_group_choice})
 
         data = render('rhodecode:templates/admin/repos/repo_add.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     # perms check inside
     @view_config(
         route_name='repo_create', request_method='POST',
         renderer='rhodecode:templates/admin/repos/repos.mako')
     def repository_create(self):
         c = self.load_default_context()
 
         form_result = {}
         self._load_form_data(c)
 
         try:
             # CanWriteToGroup validators checks permissions of this POST
             form = RepoForm(
                 self.request.translate, repo_groups=c.repo_groups_choices)()
             form_result = form.to_python(dict(self.request.POST))
             copy_permissions = form_result.get('repo_copy_permissions')
             # create is done sometimes async on celery, db transaction
             # management is handled there.
             task = RepoModel().create(form_result, self._rhodecode_user.user_id)
             task_id = get_task_id(task)
         except formencode.Invalid as errors:
             data = render('rhodecode:templates/admin/repos/repo_add.mako',
                           self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
 
         except Exception as e:
             msg = self._log_creation_exception(e, form_result.get('repo_name'))
             h.flash(msg, category='error')
             raise HTTPFound(h.route_path('home'))
 
         repo_name = form_result.get('repo_name_full')
 
         affected_user_ids = [self._rhodecode_user.user_id]
         if copy_permissions:
             # permission flush is done in repo creating
             pass
         PermissionModel().trigger_permission_flush(affected_user_ids)
 
         raise HTTPFound(
             h.route_path('repo_creating', repo_name=repo_name,
                          _query=dict(task_id=task_id)))

rhodecode/apps/admin/views/user_groups.py

@@ -1,269 +1,268 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 
 import formencode
 import formencode.htmlfill
 
 from pyramid.httpexceptions import HTTPFound
 from pyramid.view import view_config
 from pyramid.response import Response
 from pyramid.renderers import render
 
 from rhodecode import events
 from rhodecode.apps._base import BaseAppView, DataGridAppView
 from rhodecode.lib.auth import (
     LoginRequired, NotAnonymous, CSRFRequired, HasPermissionAnyDecorator)
 from rhodecode.lib import helpers as h, audit_logger
 from rhodecode.lib.utils2 import safe_unicode
 
 from rhodecode.model.forms import UserGroupForm
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.scm import UserGroupList
 from rhodecode.model.db import (
     or_, count, User, UserGroup, UserGroupMember, in_filter_generator)
 from rhodecode.model.meta import Session
 from rhodecode.model.user_group import UserGroupModel
 from rhodecode.model.db import true
 
 log = logging.getLogger(__name__)
 
 
 class AdminUserGroupsView(BaseAppView, DataGridAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
 
         PermissionModel().set_global_permission_choices(
             c, gettext_translator=self.request.translate)
 
         return c
 
     # permission check in data loading of
     # `user_groups_list_data` via UserGroupList
     @LoginRequired()
     @NotAnonymous()
     @view_config(
         route_name='user_groups', request_method='GET',
         renderer='rhodecode:templates/admin/user_groups/user_groups.mako')
     def user_groups_list(self):
         c = self.load_default_context()
         return self._get_template_context(c)
 
     # permission check inside
     @LoginRequired()
     @NotAnonymous()
     @view_config(
         route_name='user_groups_data', request_method='GET',
         renderer='json_ext', xhr=True)
     def user_groups_list_data(self):
         self.load_default_context()
         column_map = {
             'active': 'users_group_active',
             'description': 'user_group_description',
             'members': 'members_total',
             'owner': 'user_username',
             'sync': 'group_data'
         }
         draw, start, limit = self._extract_chunk(self.request)
         search_q, order_by, order_dir = self._extract_ordering(
             self.request, column_map=column_map)
 
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/data_table/_dt_elements.mako')
 
         def user_group_name(user_group_name):
             return _render("user_group_name", user_group_name)
 
         def user_group_actions(user_group_id, user_group_name):
             return _render("user_group_actions", user_group_id, user_group_name)
 
         def user_profile(username):
             return _render('user_profile', username)
 
         _perms = ['usergroup.admin']
         allowed_ids = [-1] + self._rhodecode_user.user_group_acl_ids_from_stack(_perms)
 
         user_groups_data_total_count = UserGroup.query()\
             .filter(or_(
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(UserGroup.users_group_id, allowed_ids)
             ))\
             .count()
 
         user_groups_data_total_inactive_count = UserGroup.query()\
             .filter(or_(
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(UserGroup.users_group_id, allowed_ids)
             ))\
             .filter(UserGroup.users_group_active != true()).count()
 
         member_count = count(UserGroupMember.user_id)
         base_q = Session.query(
             UserGroup.users_group_name,
             UserGroup.user_group_description,
             UserGroup.users_group_active,
             UserGroup.users_group_id,
             UserGroup.group_data,
             User,
             member_count.label('member_count')
             ) \
             .filter(or_(
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(UserGroup.users_group_id, allowed_ids)
             )) \
             .outerjoin(UserGroupMember, UserGroupMember.users_group_id == UserGroup.users_group_id) \
             .join(User, User.user_id == UserGroup.user_id) \
             .group_by(UserGroup, User)
 
         base_q_inactive = base_q.filter(UserGroup.users_group_active != true())
 
         if search_q:
             like_expression = u'%{}%'.format(safe_unicode(search_q))
             base_q = base_q.filter(or_(
                 UserGroup.users_group_name.ilike(like_expression),
             ))
             base_q_inactive = base_q.filter(UserGroup.users_group_active != true())
 
         user_groups_data_total_filtered_count = base_q.count()
         user_groups_data_total_filtered_inactive_count = base_q_inactive.count()
 
         sort_defined = False
         if order_by == 'members_total':
             sort_col = member_count
             sort_defined = True
         elif order_by == 'user_username':
             sort_col = User.username
         else:
             sort_col = getattr(UserGroup, order_by, None)
 
         if sort_defined or sort_col:
             if order_dir == 'asc':
                 sort_col = sort_col.asc()
             else:
                 sort_col = sort_col.desc()
 
         base_q = base_q.order_by(sort_col)
         base_q = base_q.offset(start).limit(limit)
 
         # authenticated access to user groups
         auth_user_group_list = base_q.all()
 
         user_groups_data = []
         for user_gr in auth_user_group_list:
             row = {
                 "users_group_name": user_group_name(user_gr.users_group_name),
-                "name_raw": h.escape(user_gr.users_group_name),
                 "description": h.escape(user_gr.user_group_description),
                 "members": user_gr.member_count,
                 # NOTE(marcink): because of advanced query we
                 # need to load it like that
                 "sync": UserGroup._load_sync(
                     UserGroup._load_group_data(user_gr.group_data)),
                 "active": h.bool2icon(user_gr.users_group_active),
                 "owner": user_profile(user_gr.User.username),
                 "action": user_group_actions(
                     user_gr.users_group_id, user_gr.users_group_name)
             }
             user_groups_data.append(row)
 
         data = ({
             'draw': draw,
             'data': user_groups_data,
             'recordsTotal': user_groups_data_total_count,
             'recordsTotalInactive': user_groups_data_total_inactive_count,
             'recordsFiltered': user_groups_data_total_filtered_count,
             'recordsFilteredInactive': user_groups_data_total_filtered_inactive_count,
         })
 
         return data
 
     @LoginRequired()
     @HasPermissionAnyDecorator('hg.admin', 'hg.usergroup.create.true')
     @view_config(
         route_name='user_groups_new', request_method='GET',
         renderer='rhodecode:templates/admin/user_groups/user_group_add.mako')
     def user_groups_new(self):
         c = self.load_default_context()
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAnyDecorator('hg.admin', 'hg.usergroup.create.true')
     @CSRFRequired()
     @view_config(
         route_name='user_groups_create', request_method='POST',
         renderer='rhodecode:templates/admin/user_groups/user_group_add.mako')
     def user_groups_create(self):
         _ = self.request.translate
         c = self.load_default_context()
         users_group_form = UserGroupForm(self.request.translate)()
 
         user_group_name = self.request.POST.get('users_group_name')
         try:
             form_result = users_group_form.to_python(dict(self.request.POST))
             user_group = UserGroupModel().create(
                 name=form_result['users_group_name'],
                 description=form_result['user_group_description'],
                 owner=self._rhodecode_user.user_id,
                 active=form_result['users_group_active'])
             Session().flush()
             creation_data = user_group.get_api_data()
             user_group_name = form_result['users_group_name']
 
             audit_logger.store_web(
                 'user_group.create', action_data={'data': creation_data},
                 user=self._rhodecode_user)
 
             user_group_link = h.link_to(
                 h.escape(user_group_name),
                 h.route_path(
                     'edit_user_group', user_group_id=user_group.users_group_id))
             h.flash(h.literal(_('Created user group %(user_group_link)s')
                               % {'user_group_link': user_group_link}),
                     category='success')
             Session().commit()
             user_group_id = user_group.users_group_id
         except formencode.Invalid as errors:
 
             data = render(
                 'rhodecode:templates/admin/user_groups/user_group_add.mako',
                 self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
 
         except Exception:
             log.exception("Exception creating user group")
             h.flash(_('Error occurred during creation of user group %s') \
                     % user_group_name, category='error')
             raise HTTPFound(h.route_path('user_groups_new'))
 
         affected_user_ids = [self._rhodecode_user.user_id]
         PermissionModel().trigger_permission_flush(affected_user_ids)
 
         raise HTTPFound(
             h.route_path('edit_user_group', user_group_id=user_group_id))

rhodecode/apps/home/views.py

@@ -1,823 +1,823 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import re
 import logging
 import collections
 
 from pyramid.httpexceptions import HTTPNotFound
 from pyramid.view import view_config
 
 from rhodecode.apps._base import BaseAppView, DataGridAppView
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import (
     LoginRequired, NotAnonymous, HasRepoGroupPermissionAnyDecorator, CSRFRequired,
     HasRepoGroupPermissionAny)
 from rhodecode.lib.codeblocks import filenode_as_lines_tokens
 from rhodecode.lib.index import searcher_from_config
 from rhodecode.lib.utils2 import safe_unicode, str2bool, safe_int
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.model.db import (
     func, true, or_, case, in_filter_generator, Session,
     Repository, RepoGroup, User, UserGroup)
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.user import UserModel
 from rhodecode.model.user_group import UserGroupModel
 
 log = logging.getLogger(__name__)
 
 
 class HomeView(BaseAppView, DataGridAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         c.user = c.auth_user.get_instance()
 
         return c
 
     @LoginRequired()
     @view_config(
         route_name='user_autocomplete_data', request_method='GET',
         renderer='json_ext', xhr=True)
     def user_autocomplete_data(self):
         self.load_default_context()
         query = self.request.GET.get('query')
         active = str2bool(self.request.GET.get('active') or True)
         include_groups = str2bool(self.request.GET.get('user_groups'))
         expand_groups = str2bool(self.request.GET.get('user_groups_expand'))
         skip_default_user = str2bool(self.request.GET.get('skip_default_user'))
 
         log.debug('generating user list, query:%s, active:%s, with_groups:%s',
                   query, active, include_groups)
 
         _users = UserModel().get_users(
             name_contains=query, only_active=active)
 
         def maybe_skip_default_user(usr):
             if skip_default_user and usr['username'] == UserModel.cls.DEFAULT_USER:
                 return False
             return True
         _users = filter(maybe_skip_default_user, _users)
 
         if include_groups:
             # extend with user groups
             _user_groups = UserGroupModel().get_user_groups(
                 name_contains=query, only_active=active,
                 expand_groups=expand_groups)
             _users = _users + _user_groups
 
         return {'suggestions': _users}
 
     @LoginRequired()
     @NotAnonymous()
     @view_config(
         route_name='user_group_autocomplete_data', request_method='GET',
         renderer='json_ext', xhr=True)
     def user_group_autocomplete_data(self):
         self.load_default_context()
         query = self.request.GET.get('query')
         active = str2bool(self.request.GET.get('active') or True)
         expand_groups = str2bool(self.request.GET.get('user_groups_expand'))
 
         log.debug('generating user group list, query:%s, active:%s',
                   query, active)
 
         _user_groups = UserGroupModel().get_user_groups(
             name_contains=query, only_active=active,
             expand_groups=expand_groups)
         _user_groups = _user_groups
 
         return {'suggestions': _user_groups}
 
     def _get_repo_list(self, name_contains=None, repo_type=None, repo_group_name='', limit=20):
         org_query = name_contains
         allowed_ids = self._rhodecode_user.repo_acl_ids(
             ['repository.read', 'repository.write', 'repository.admin'],
             cache=False, name_filter=name_contains) or [-1]
 
         query = Repository.query()\
             .filter(Repository.archived.isnot(true()))\
             .filter(or_(
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(Repository.repo_id, allowed_ids)
             ))
 
         query = query.order_by(case(
             [
                 (Repository.repo_name.startswith(repo_group_name), repo_group_name+'/'),
             ],
         ))
         query = query.order_by(func.length(Repository.repo_name))
         query = query.order_by(Repository.repo_name)
 
         if repo_type:
             query = query.filter(Repository.repo_type == repo_type)
 
         if name_contains:
             ilike_expression = u'%{}%'.format(safe_unicode(name_contains))
             query = query.filter(
                 Repository.repo_name.ilike(ilike_expression))
             query = query.limit(limit)
 
         acl_iter = query
 
         return [
             {
                 'id': obj.repo_name,
                 'value': org_query,
                 'value_display': obj.repo_name,
                 'text': obj.repo_name,
                 'type': 'repo',
                 'repo_id': obj.repo_id,
                 'repo_type': obj.repo_type,
                 'private': obj.private,
                 'url': h.route_path('repo_summary', repo_name=obj.repo_name)
             }
             for obj in acl_iter]
 
     def _get_repo_group_list(self, name_contains=None, repo_group_name='', limit=20):
         org_query = name_contains
         allowed_ids = self._rhodecode_user.repo_group_acl_ids(
             ['group.read', 'group.write', 'group.admin'],
             cache=False, name_filter=name_contains) or [-1]
 
         query = RepoGroup.query()\
             .filter(or_(
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(RepoGroup.group_id, allowed_ids)
             ))
 
         query = query.order_by(case(
             [
                 (RepoGroup.group_name.startswith(repo_group_name), repo_group_name+'/'),
             ],
         ))
         query = query.order_by(func.length(RepoGroup.group_name))
         query = query.order_by(RepoGroup.group_name)
 
         if name_contains:
             ilike_expression = u'%{}%'.format(safe_unicode(name_contains))
             query = query.filter(
                 RepoGroup.group_name.ilike(ilike_expression))
             query = query.limit(limit)
 
         acl_iter = query
 
         return [
             {
                 'id': obj.group_name,
                 'value': org_query,
                 'value_display': obj.group_name,
                 'text': obj.group_name,
                 'type': 'repo_group',
                 'repo_group_id': obj.group_id,
                 'url': h.route_path(
                     'repo_group_home', repo_group_name=obj.group_name)
             }
             for obj in acl_iter]
 
     def _get_user_list(self, name_contains=None, limit=20):
         org_query = name_contains
         if not name_contains:
             return [], False
 
         # TODO(marcink): should all logged in users be allowed to search others?
         allowed_user_search = self._rhodecode_user.username != User.DEFAULT_USER
         if not allowed_user_search:
             return [], False
 
         name_contains = re.compile('(?:user:[ ]?)(.+)').findall(name_contains)
         if len(name_contains) != 1:
             return [], False
 
         name_contains = name_contains[0]
 
         query = User.query()\
             .order_by(func.length(User.username))\
             .order_by(User.username) \
             .filter(User.username != User.DEFAULT_USER)
 
         if name_contains:
             ilike_expression = u'%{}%'.format(safe_unicode(name_contains))
             query = query.filter(
                 User.username.ilike(ilike_expression))
             query = query.limit(limit)
 
         acl_iter = query
 
         return [
             {
                 'id': obj.user_id,
                 'value': org_query,
                 'value_display': 'user: `{}`'.format(obj.username),
                 'type': 'user',
                 'icon_link': h.gravatar_url(obj.email, 30),
                 'url': h.route_path(
                     'user_profile', username=obj.username)
             }
             for obj in acl_iter], True
 
     def _get_user_groups_list(self, name_contains=None, limit=20):
         org_query = name_contains
         if not name_contains:
             return [], False
 
         # TODO(marcink): should all logged in users be allowed to search others?
         allowed_user_search = self._rhodecode_user.username != User.DEFAULT_USER
         if not allowed_user_search:
             return [], False
 
         name_contains = re.compile('(?:user_group:[ ]?)(.+)').findall(name_contains)
         if len(name_contains) != 1:
             return [], False
 
         name_contains = name_contains[0]
 
         query = UserGroup.query()\
             .order_by(func.length(UserGroup.users_group_name))\
             .order_by(UserGroup.users_group_name)
 
         if name_contains:
             ilike_expression = u'%{}%'.format(safe_unicode(name_contains))
             query = query.filter(
                 UserGroup.users_group_name.ilike(ilike_expression))
             query = query.limit(limit)
 
         acl_iter = query
 
         return [
             {
                 'id': obj.users_group_id,
                 'value': org_query,
                 'value_display': 'user_group: `{}`'.format(obj.users_group_name),
                 'type': 'user_group',
                 'url': h.route_path(
                     'user_group_profile', user_group_name=obj.users_group_name)
             }
             for obj in acl_iter], True
 
     def _get_hash_commit_list(self, auth_user, searcher, query, repo=None, repo_group=None):
         repo_name = repo_group_name = None
         if repo:
             repo_name = repo.repo_name
         if repo_group:
             repo_group_name = repo_group.group_name
 
         org_query = query
         if not query or len(query) < 3 or not searcher:
             return [], False
 
         commit_hashes = re.compile('(?:commit:[ ]?)([0-9a-f]{2,40})').findall(query)
 
         if len(commit_hashes) != 1:
             return [], False
 
         commit_hash = commit_hashes[0]
 
         result = searcher.search(
             'commit_id:{}*'.format(commit_hash), 'commit', auth_user,
             repo_name, repo_group_name, raise_on_exc=False)
 
         commits = []
         for entry in result['results']:
             repo_data = {
                 'repository_id': entry.get('repository_id'),
                 'repository_type': entry.get('repo_type'),
                 'repository_name': entry.get('repository'),
             }
 
             commit_entry = {
                 'id': entry['commit_id'],
                 'value': org_query,
                 'value_display': '`{}` commit: {}'.format(
                     entry['repository'], entry['commit_id']),
                 'type': 'commit',
                 'repo': entry['repository'],
                 'repo_data': repo_data,
 
                 'url': h.route_path(
                     'repo_commit',
                     repo_name=entry['repository'], commit_id=entry['commit_id'])
             }
 
             commits.append(commit_entry)
         return commits, True
 
     def _get_path_list(self, auth_user, searcher, query, repo=None, repo_group=None):
         repo_name = repo_group_name = None
         if repo:
             repo_name = repo.repo_name
         if repo_group:
             repo_group_name = repo_group.group_name
 
         org_query = query
         if not query or len(query) < 3 or not searcher:
             return [], False
 
         paths_re = re.compile('(?:file:[ ]?)(.+)').findall(query)
         if len(paths_re) != 1:
             return [], False
 
         file_path = paths_re[0]
 
         search_path = searcher.escape_specials(file_path)
         result = searcher.search(
             'file.raw:*{}*'.format(search_path), 'path', auth_user,
             repo_name, repo_group_name, raise_on_exc=False)
 
         files = []
         for entry in result['results']:
             repo_data = {
                 'repository_id': entry.get('repository_id'),
                 'repository_type': entry.get('repo_type'),
                 'repository_name': entry.get('repository'),
             }
 
             file_entry = {
                 'id': entry['commit_id'],
                 'value': org_query,
                 'value_display': '`{}` file: {}'.format(
                     entry['repository'], entry['file']),
                 'type': 'file',
                 'repo': entry['repository'],
                 'repo_data': repo_data,
 
                 'url': h.route_path(
                     'repo_files',
                     repo_name=entry['repository'], commit_id=entry['commit_id'],
                     f_path=entry['file'])
             }
 
             files.append(file_entry)
         return files, True
 
     @LoginRequired()
     @view_config(
         route_name='repo_list_data', request_method='GET',
         renderer='json_ext', xhr=True)
     def repo_list_data(self):
         _ = self.request.translate
         self.load_default_context()
 
         query = self.request.GET.get('query')
         repo_type = self.request.GET.get('repo_type')
         log.debug('generating repo list, query:%s, repo_type:%s',
                   query, repo_type)
 
         res = []
         repos = self._get_repo_list(query, repo_type=repo_type)
         if repos:
             res.append({
                 'text': _('Repositories'),
                 'children': repos
             })
 
         data = {
             'more': False,
             'results': res
         }
         return data
 
     @LoginRequired()
     @view_config(
         route_name='repo_group_list_data', request_method='GET',
         renderer='json_ext', xhr=True)
     def repo_group_list_data(self):
         _ = self.request.translate
         self.load_default_context()
 
         query = self.request.GET.get('query')
 
         log.debug('generating repo group list, query:%s',
                   query)
 
         res = []
         repo_groups = self._get_repo_group_list(query)
         if repo_groups:
             res.append({
                 'text': _('Repository Groups'),
                 'children': repo_groups
             })
 
         data = {
             'more': False,
             'results': res
         }
         return data
 
     def _get_default_search_queries(self, search_context, searcher, query):
         if not searcher:
             return []
 
         is_es_6 = searcher.is_es_6
 
         queries = []
         repo_group_name, repo_name, repo_context = None, None, None
 
         # repo group context
         if search_context.get('search_context[repo_group_name]'):
             repo_group_name = search_context.get('search_context[repo_group_name]')
         if search_context.get('search_context[repo_name]'):
             repo_name = search_context.get('search_context[repo_name]')
             repo_context = search_context.get('search_context[repo_view_type]')
 
         if is_es_6 and repo_name:
             # files
             def query_modifier():
                 qry = query
                 return {'q': qry, 'type': 'content'}
 
             label = u'File search for `{}`'.format(h.escape(query))
             file_qry = {
                 'id': -10,
                 'value': query,
                 'value_display': label,
                 'value_icon': '<i class="icon-code"></i>',
                 'type': 'search',
                 'subtype': 'repo',
                 'url': h.route_path('search_repo',
                                     repo_name=repo_name,
                                     _query=query_modifier())
                 }
 
             # commits
             def query_modifier():
                 qry = query
                 return {'q': qry, 'type': 'commit'}
 
             label = u'Commit search for `{}`'.format(h.escape(query))
             commit_qry = {
                 'id': -20,
                 'value': query,
                 'value_display': label,
                 'value_icon': '<i class="icon-history"></i>',
                 'type': 'search',
                 'subtype': 'repo',
                 'url': h.route_path('search_repo',
                                     repo_name=repo_name,
                                     _query=query_modifier())
                 }
 
             if repo_context in ['commit', 'commits']:
                 queries.extend([commit_qry, file_qry])
             elif repo_context in ['files', 'summary']:
                 queries.extend([file_qry, commit_qry])
             else:
                 queries.extend([commit_qry, file_qry])
 
         elif is_es_6 and repo_group_name:
             # files
             def query_modifier():
                 qry = query
                 return {'q': qry, 'type': 'content'}
 
             label = u'File search for `{}`'.format(query)
             file_qry = {
                 'id': -30,
                 'value': query,
                 'value_display': label,
                 'value_icon': '<i class="icon-code"></i>',
                 'type': 'search',
                 'subtype': 'repo_group',
                 'url': h.route_path('search_repo_group',
                                     repo_group_name=repo_group_name,
                                     _query=query_modifier())
                 }
 
             # commits
             def query_modifier():
                 qry = query
                 return {'q': qry, 'type': 'commit'}
 
             label = u'Commit search for `{}`'.format(query)
             commit_qry = {
                 'id': -40,
                 'value': query,
                 'value_display': label,
                 'value_icon': '<i class="icon-history"></i>',
                 'type': 'search',
                 'subtype': 'repo_group',
                 'url': h.route_path('search_repo_group',
                                     repo_group_name=repo_group_name,
                                     _query=query_modifier())
                 }
 
             if repo_context in ['commit', 'commits']:
                 queries.extend([commit_qry, file_qry])
             elif repo_context in ['files', 'summary']:
                 queries.extend([file_qry, commit_qry])
             else:
                 queries.extend([commit_qry, file_qry])
 
         # Global, not scoped
         if not queries:
             queries.append(
                 {
                     'id': -1,
                     'value': query,
                     'value_display': u'File search for: `{}`'.format(query),
                     'value_icon': '<i class="icon-code"></i>',
                     'type': 'search',
                     'subtype': 'global',
                     'url': h.route_path('search',
                                         _query={'q': query, 'type': 'content'})
                 })
             queries.append(
                 {
                     'id': -2,
                     'value': query,
                     'value_display': u'Commit search for: `{}`'.format(query),
                     'value_icon': '<i class="icon-history"></i>',
                     'type': 'search',
                     'subtype': 'global',
                     'url': h.route_path('search',
                                         _query={'q': query, 'type': 'commit'})
                 })
 
         return queries
 
     @LoginRequired()
     @view_config(
         route_name='goto_switcher_data', request_method='GET',
         renderer='json_ext', xhr=True)
     def goto_switcher_data(self):
         c = self.load_default_context()
 
         _ = self.request.translate
 
         query = self.request.GET.get('query')
         log.debug('generating main filter data, query %s', query)
 
         res = []
         if not query:
             return {'suggestions': res}
 
         def no_match(name):
             return {
                 'id': -1,
                 'value': "",
                 'value_display': name,
                 'type': 'text',
                 'url': ""
             }
         searcher = searcher_from_config(self.request.registry.settings)
         has_specialized_search = False
 
         # set repo context
         repo = None
         repo_id = safe_int(self.request.GET.get('search_context[repo_id]'))
         if repo_id:
             repo = Repository.get(repo_id)
 
         # set group context
         repo_group = None
         repo_group_id = safe_int(self.request.GET.get('search_context[repo_group_id]'))
         if repo_group_id:
             repo_group = RepoGroup.get(repo_group_id)
         prefix_match = False
 
         # user: type search
         if not prefix_match:
             users, prefix_match = self._get_user_list(query)
             if users:
                 has_specialized_search = True
                 for serialized_user in users:
                     res.append(serialized_user)
             elif prefix_match:
                 has_specialized_search = True
                 res.append(no_match('No matching users found'))
 
         # user_group: type search
         if not prefix_match:
             user_groups, prefix_match = self._get_user_groups_list(query)
             if user_groups:
                 has_specialized_search = True
                 for serialized_user_group in user_groups:
                     res.append(serialized_user_group)
             elif prefix_match:
                 has_specialized_search = True
                 res.append(no_match('No matching user groups found'))
 
         # FTS commit: type search
         if not prefix_match:
             commits, prefix_match = self._get_hash_commit_list(
                 c.auth_user, searcher, query, repo, repo_group)
             if commits:
                 has_specialized_search = True
                 unique_repos = collections.OrderedDict()
                 for commit in commits:
                     repo_name = commit['repo']
                     unique_repos.setdefault(repo_name, []).append(commit)
 
                 for _repo, commits in unique_repos.items():
                     for commit in commits:
                         res.append(commit)
             elif prefix_match:
                 has_specialized_search = True
                 res.append(no_match('No matching commits found'))
 
         # FTS file: type search
         if not prefix_match:
             paths, prefix_match = self._get_path_list(
                 c.auth_user, searcher, query, repo, repo_group)
             if paths:
                 has_specialized_search = True
                 unique_repos = collections.OrderedDict()
                 for path in paths:
                     repo_name = path['repo']
                     unique_repos.setdefault(repo_name, []).append(path)
 
                 for repo, paths in unique_repos.items():
                     for path in paths:
                         res.append(path)
             elif prefix_match:
                 has_specialized_search = True
                 res.append(no_match('No matching files found'))
 
         # main suggestions
         if not has_specialized_search:
             repo_group_name = ''
             if repo_group:
                 repo_group_name = repo_group.group_name
 
             for _q in self._get_default_search_queries(self.request.GET, searcher, query):
                 res.append(_q)
 
             repo_groups = self._get_repo_group_list(query, repo_group_name=repo_group_name)
             for serialized_repo_group in repo_groups:
                 res.append(serialized_repo_group)
 
             repos = self._get_repo_list(query, repo_group_name=repo_group_name)
             for serialized_repo in repos:
                 res.append(serialized_repo)
 
             if not repos and not repo_groups:
                 res.append(no_match('No matches found'))
 
         return {'suggestions': res}
 
     @LoginRequired()
     @view_config(
         route_name='home', request_method='GET',
         renderer='rhodecode:templates/index.mako')
     def main_page(self):
         c = self.load_default_context()
         c.repo_group = None
         return self._get_template_context(c)
 
     def _main_page_repo_groups_data(self, repo_group_id):
         column_map = {
-            'name_raw': 'group_name_hash',
+            'name': 'group_name_hash',
             'desc': 'group_description',
-            'last_change_raw': 'updated_on',
+            'last_change': 'updated_on',
             'owner': 'user_username',
         }
         draw, start, limit = self._extract_chunk(self.request)
         search_q, order_by, order_dir = self._extract_ordering(
             self.request, column_map=column_map)
         return RepoGroupModel().get_repo_groups_data_table(
             draw, start, limit,
             search_q, order_by, order_dir,
             self._rhodecode_user, repo_group_id)
 
     def _main_page_repos_data(self, repo_group_id):
         column_map = {
-            'name_raw': 'repo_name',
+            'name': 'repo_name',
             'desc': 'description',
-            'last_change_raw': 'updated_on',
+            'last_change': 'updated_on',
             'owner': 'user_username',
         }
         draw, start, limit = self._extract_chunk(self.request)
         search_q, order_by, order_dir = self._extract_ordering(
             self.request, column_map=column_map)
         return RepoModel().get_repos_data_table(
             draw, start, limit,
             search_q, order_by, order_dir,
             self._rhodecode_user, repo_group_id)
 
     @LoginRequired()
     @view_config(
         route_name='main_page_repo_groups_data',
         request_method='GET', renderer='json_ext', xhr=True)
     def main_page_repo_groups_data(self):
         self.load_default_context()
         repo_group_id = safe_int(self.request.GET.get('repo_group_id'))
 
         if repo_group_id:
             group = RepoGroup.get_or_404(repo_group_id)
             _perms = ['group.read', 'group.write', 'group.admin']
             if not HasRepoGroupPermissionAny(*_perms)(
                     group.group_name, 'user is allowed to list repo group children'):
                 raise HTTPNotFound()
 
         return self._main_page_repo_groups_data(repo_group_id)
 
     @LoginRequired()
     @view_config(
         route_name='main_page_repos_data',
         request_method='GET', renderer='json_ext', xhr=True)
     def main_page_repos_data(self):
         self.load_default_context()
         repo_group_id = safe_int(self.request.GET.get('repo_group_id'))
 
         if repo_group_id:
             group = RepoGroup.get_or_404(repo_group_id)
             _perms = ['group.read', 'group.write', 'group.admin']
             if not HasRepoGroupPermissionAny(*_perms)(
                     group.group_name, 'user is allowed to list repo group children'):
                 raise HTTPNotFound()
 
         return self._main_page_repos_data(repo_group_id)
 
     @LoginRequired()
     @HasRepoGroupPermissionAnyDecorator(
         'group.read', 'group.write', 'group.admin')
     @view_config(
         route_name='repo_group_home', request_method='GET',
         renderer='rhodecode:templates/index_repo_group.mako')
     @view_config(
         route_name='repo_group_home_slash', request_method='GET',
         renderer='rhodecode:templates/index_repo_group.mako')
     def repo_group_main_page(self):
         c = self.load_default_context()
         c.repo_group = self.request.db_repo_group
         return self._get_template_context(c)
 
     @LoginRequired()
     @CSRFRequired()
     @view_config(
         route_name='markup_preview', request_method='POST',
         renderer='string', xhr=True)
     def markup_preview(self):
         # Technically a CSRF token is not needed as no state changes with this
         # call. However, as this is a POST is better to have it, so automated
         # tools don't flag it as potential CSRF.
         # Post is required because the payload could be bigger than the maximum
         # allowed by GET.
 
         text = self.request.POST.get('text')
         renderer = self.request.POST.get('renderer') or 'rst'
         if text:
             return h.render(text, renderer=renderer, mentions=True)
         return ''
 
     @LoginRequired()
     @CSRFRequired()
     @view_config(
         route_name='file_preview', request_method='POST',
         renderer='string', xhr=True)
     def file_preview(self):
         # Technically a CSRF token is not needed as no state changes with this
         # call. However, as this is a POST is better to have it, so automated
         # tools don't flag it as potential CSRF.
         # Post is required because the payload could be bigger than the maximum
         # allowed by GET.
 
         text = self.request.POST.get('text')
         file_path = self.request.POST.get('file_path')
 
         renderer = h.renderer_from_filename(file_path)
 
         if renderer:
             return h.render(text, renderer=renderer, mentions=True)
         else:
             self.load_default_context()
             _render = self.request.get_partial_renderer(
                 'rhodecode:templates/files/file_content.mako')
 
             lines = filenode_as_lines_tokens(FileNode(file_path, text))
 
             return _render('render_lines', lines)
 
     @LoginRequired()
     @CSRFRequired()
     @view_config(
         route_name='store_user_session_value', request_method='POST',
         renderer='string', xhr=True)
     def store_user_session_attr(self):
         key = self.request.POST.get('key')
         val = self.request.POST.get('val')
 
         existing_value = self.request.session.get(key)
         if existing_value != val:
             self.request.session[key] = val
 
         return 'stored:{}:{}'.format(key, val)

rhodecode/model/repo.py

@@ -1,1160 +1,1158 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import re
 import shutil
 import time
 import logging
 import traceback
 import datetime
 
 from pyramid.threadlocal import get_current_request
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 from rhodecode import events
 from rhodecode.lib.auth import HasUserGroupPermissionAny
 from rhodecode.lib.caching_query import FromCache
 from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError
 from rhodecode.lib.hooks_base import log_delete_repository
 from rhodecode.lib.user_log_filter import user_log_filter
 from rhodecode.lib.utils import make_db_config
 from rhodecode.lib.utils2 import (
     safe_str, safe_unicode, remove_prefix, obfuscate_url_pw,
     get_current_rhodecode_user, safe_int, datetime_to_time,
     action_logger_generic)
 from rhodecode.lib.vcs.backends import get_backend
 from rhodecode.model import BaseModel
 from rhodecode.model.db import (
     _hash_key, func, case, joinedload, or_, in_filter_generator,
     Session, Repository, UserRepoToPerm, UserGroupRepoToPerm,
     UserRepoGroupToPerm, UserGroupRepoGroupToPerm, User, Permission,
     Statistics, UserGroup, RepoGroup, RepositoryField, UserLog)
 from rhodecode.model.settings import VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
 
 class RepoModel(BaseModel):
 
     cls = Repository
 
     def _get_user_group(self, users_group):
         return self._get_instance(UserGroup, users_group,
                                   callback=UserGroup.get_by_group_name)
 
     def _get_repo_group(self, repo_group):
         return self._get_instance(RepoGroup, repo_group,
                                   callback=RepoGroup.get_by_group_name)
 
     def _create_default_perms(self, repository, private):
         # create default permission
         default = 'repository.read'
         def_user = User.get_default_user()
         for p in def_user.user_perms:
             if p.permission.permission_name.startswith('repository.'):
                 default = p.permission.permission_name
                 break
 
         default_perm = 'repository.none' if private else default
 
         repo_to_perm = UserRepoToPerm()
         repo_to_perm.permission = Permission.get_by_key(default_perm)
 
         repo_to_perm.repository = repository
         repo_to_perm.user_id = def_user.user_id
 
         return repo_to_perm
 
     @LazyProperty
     def repos_path(self):
         """
         Gets the repositories root path from database
         """
         settings_model = VcsSettingsModel(sa=self.sa)
         return settings_model.get_repos_location()
 
     def get(self, repo_id):
         repo = self.sa.query(Repository) \
             .filter(Repository.repo_id == repo_id)
 
         return repo.scalar()
 
     def get_repo(self, repository):
         return self._get_repo(repository)
 
     def get_by_repo_name(self, repo_name, cache=False):
         repo = self.sa.query(Repository) \
             .filter(Repository.repo_name == repo_name)
 
         if cache:
             name_key = _hash_key(repo_name)
             repo = repo.options(
                 FromCache("sql_cache_short", "get_repo_%s" % name_key))
         return repo.scalar()
 
     def _extract_id_from_repo_name(self, repo_name):
         if repo_name.startswith('/'):
             repo_name = repo_name.lstrip('/')
         by_id_match = re.match(r'^_(\d{1,})', repo_name)
         if by_id_match:
             return by_id_match.groups()[0]
 
     def get_repo_by_id(self, repo_name):
         """
         Extracts repo_name by id from special urls.
         Example url is _11/repo_name
 
         :param repo_name:
         :return: repo object if matched else None
         """
 
         try:
             _repo_id = self._extract_id_from_repo_name(repo_name)
             if _repo_id:
                 return self.get(_repo_id)
         except Exception:
             log.exception('Failed to extract repo_name from URL')
 
         return None
 
     def get_repos_for_root(self, root, traverse=False):
         if traverse:
             like_expression = u'{}%'.format(safe_unicode(root))
             repos = Repository.query().filter(
                 Repository.repo_name.like(like_expression)).all()
         else:
             if root and not isinstance(root, RepoGroup):
                 raise ValueError(
                     'Root must be an instance '
                     'of RepoGroup, got:{} instead'.format(type(root)))
             repos = Repository.query().filter(Repository.group == root).all()
         return repos
 
     def get_url(self, repo, request=None, permalink=False):
         if not request:
             request = get_current_request()
 
         if not request:
             return
 
         if permalink:
             return request.route_url(
                 'repo_summary', repo_name='_{}'.format(safe_str(repo.repo_id)))
         else:
             return request.route_url(
                 'repo_summary', repo_name=safe_str(repo.repo_name))
 
     def get_commit_url(self, repo, commit_id, request=None, permalink=False):
         if not request:
             request = get_current_request()
 
         if not request:
             return
 
         if permalink:
             return request.route_url(
                 'repo_commit', repo_name=safe_str(repo.repo_id),
                 commit_id=commit_id)
 
         else:
             return request.route_url(
                 'repo_commit', repo_name=safe_str(repo.repo_name),
                 commit_id=commit_id)
 
     def get_repo_log(self, repo, filter_term):
         repo_log = UserLog.query()\
             .filter(or_(UserLog.repository_id == repo.repo_id,
                         UserLog.repository_name == repo.repo_name))\
             .options(joinedload(UserLog.user))\
             .options(joinedload(UserLog.repository))\
             .order_by(UserLog.action_date.desc())
 
         repo_log = user_log_filter(repo_log, filter_term)
         return repo_log
 
     @classmethod
     def update_commit_cache(cls, repositories=None):
         if not repositories:
             repositories = Repository.getAll()
         for repo in repositories:
             repo.update_commit_cache()
 
     def get_repos_as_dict(self, repo_list=None, admin=False,
                           super_user_actions=False, short_name=None):
         _render = get_current_request().get_partial_renderer(
             'rhodecode:templates/data_table/_dt_elements.mako')
         c = _render.get_call_context()
 
         def quick_menu(repo_name):
             return _render('quick_menu', repo_name)
 
         def repo_lnk(name, rtype, rstate, private, archived, fork_of):
             if short_name is not None:
                 short_name_var = short_name
             else:
                 short_name_var = not admin
             return _render('repo_name', name, rtype, rstate, private, archived, fork_of,
                            short_name=short_name_var, admin=False)
 
         def last_change(last_change):
             if admin and isinstance(last_change, datetime.datetime) and not last_change.tzinfo:
                 ts = time.time()
                 utc_offset = (datetime.datetime.fromtimestamp(ts)
                               - datetime.datetime.utcfromtimestamp(ts)).total_seconds()
                 last_change = last_change + datetime.timedelta(seconds=utc_offset)
 
             return _render("last_change", last_change)
 
         def rss_lnk(repo_name):
             return _render("rss", repo_name)
 
         def atom_lnk(repo_name):
             return _render("atom", repo_name)
 
         def last_rev(repo_name, cs_cache):
             return _render('revision', repo_name, cs_cache.get('revision'),
                            cs_cache.get('raw_id'), cs_cache.get('author'),
                            cs_cache.get('message'), cs_cache.get('date'))
 
         def desc(desc):
             return _render('repo_desc', desc, c.visual.stylify_metatags)
 
         def state(repo_state):
             return _render("repo_state", repo_state)
 
         def repo_actions(repo_name):
             return _render('repo_actions', repo_name, super_user_actions)
 
         def user_profile(username):
             return _render('user_profile', username)
 
         repos_data = []
         for repo in repo_list:
             # NOTE(marcink): because we use only raw column we need to load it like that
             changeset_cache = Repository._load_changeset_cache(
                 repo.repo_id, repo._changeset_cache)
-            last_commit_change = Repository._load_commit_change(changeset_cache)
 
             row = {
                 "menu": quick_menu(repo.repo_name),
 
                 "name": repo_lnk(repo.repo_name, repo.repo_type, repo.repo_state,
                                  repo.private, repo.archived, repo.fork),
-                "name_raw": repo.repo_name.lower(),
+
                 "desc": desc(repo.description),
 
-                "last_change": last_change(last_commit_change),
-                "last_change_raw": datetime_to_time(last_commit_change),
+                "last_change": last_change(repo.updated_on),
 
                 "last_changeset": last_rev(repo.repo_name, changeset_cache),
                 "last_changeset_raw": changeset_cache.get('revision'),
 
                 "owner": user_profile(repo.User.username),
 
                 "state": state(repo.repo_state),
                 "rss": rss_lnk(repo.repo_name),
                 "atom": atom_lnk(repo.repo_name),
             }
             if admin:
                 row.update({
                     "action": repo_actions(repo.repo_name),
                 })
             repos_data.append(row)
 
         return repos_data
 
     def get_repos_data_table(
             self, draw, start, limit,
             search_q, order_by, order_dir,
             auth_user, repo_group_id):
         from rhodecode.model.scm import RepoList
 
         _perms = ['repository.read', 'repository.write', 'repository.admin']
 
         repos = Repository.query() \
             .filter(Repository.group_id == repo_group_id) \
             .all()
         auth_repo_list = RepoList(
             repos, perm_set=_perms,
             extra_kwargs=dict(user=auth_user))
 
         allowed_ids = [-1]
         for repo in auth_repo_list:
             allowed_ids.append(repo.repo_id)
 
         repos_data_total_count = Repository.query() \
             .filter(Repository.group_id == repo_group_id) \
             .filter(or_(
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(Repository.repo_id, allowed_ids))
             ) \
             .count()
 
         base_q = Session.query(
             Repository.repo_id,
             Repository.repo_name,
             Repository.description,
             Repository.repo_type,
             Repository.repo_state,
             Repository.private,
             Repository.archived,
             Repository.fork,
             Repository.updated_on,
             Repository._changeset_cache,
             User,
             ) \
             .filter(Repository.group_id == repo_group_id) \
             .filter(or_(
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(Repository.repo_id, allowed_ids))
             ) \
             .join(User, User.user_id == Repository.user_id) \
             .group_by(Repository, User)
 
         repos_data_total_filtered_count = base_q.count()
 
         sort_defined = False
         if order_by == 'repo_name':
             sort_col = func.lower(Repository.repo_name)
             sort_defined = True
         elif order_by == 'user_username':
             sort_col = User.username
         else:
             sort_col = getattr(Repository, order_by, None)
 
         if sort_defined or sort_col:
             if order_dir == 'asc':
                 sort_col = sort_col.asc()
             else:
                 sort_col = sort_col.desc()
 
         base_q = base_q.order_by(sort_col)
         base_q = base_q.offset(start).limit(limit)
 
         repos_list = base_q.all()
 
         repos_data = RepoModel().get_repos_as_dict(
             repo_list=repos_list, admin=False)
 
         data = ({
             'draw': draw,
             'data': repos_data,
             'recordsTotal': repos_data_total_count,
             'recordsFiltered': repos_data_total_filtered_count,
         })
         return data
 
     def _get_defaults(self, repo_name):
         """
         Gets information about repository, and returns a dict for
         usage in forms
 
         :param repo_name:
         """
 
         repo_info = Repository.get_by_repo_name(repo_name)
 
         if repo_info is None:
             return None
 
         defaults = repo_info.get_dict()
         defaults['repo_name'] = repo_info.just_name
 
         groups = repo_info.groups_with_parents
         parent_group = groups[-1] if groups else None
 
         # we use -1 as this is how in HTML, we mark an empty group
         defaults['repo_group'] = getattr(parent_group, 'group_id', -1)
 
         keys_to_process = (
             {'k': 'repo_type', 'strip': False},
             {'k': 'repo_enable_downloads', 'strip': True},
             {'k': 'repo_description', 'strip': True},
             {'k': 'repo_enable_locking', 'strip': True},
             {'k': 'repo_landing_rev', 'strip': True},
             {'k': 'clone_uri', 'strip': False},
             {'k': 'push_uri', 'strip': False},
             {'k': 'repo_private', 'strip': True},
             {'k': 'repo_enable_statistics', 'strip': True}
         )
 
         for item in keys_to_process:
             attr = item['k']
             if item['strip']:
                 attr = remove_prefix(item['k'], 'repo_')
 
             val = defaults[attr]
             if item['k'] == 'repo_landing_rev':
                 val = ':'.join(defaults[attr])
             defaults[item['k']] = val
             if item['k'] == 'clone_uri':
                 defaults['clone_uri_hidden'] = repo_info.clone_uri_hidden
             if item['k'] == 'push_uri':
                 defaults['push_uri_hidden'] = repo_info.push_uri_hidden
 
         # fill owner
         if repo_info.user:
             defaults.update({'user': repo_info.user.username})
         else:
             replacement_user = User.get_first_super_admin().username
             defaults.update({'user': replacement_user})
 
         return defaults
 
     def update(self, repo, **kwargs):
         try:
             cur_repo = self._get_repo(repo)
             source_repo_name = cur_repo.repo_name
             if 'user' in kwargs:
                 cur_repo.user = User.get_by_username(kwargs['user'])
 
             if 'repo_group' in kwargs:
                 cur_repo.group = RepoGroup.get(kwargs['repo_group'])
             log.debug('Updating repo %s with params:%s', cur_repo, kwargs)
 
             update_keys = [
                 (1, 'repo_description'),
                 (1, 'repo_landing_rev'),
                 (1, 'repo_private'),
                 (1, 'repo_enable_downloads'),
                 (1, 'repo_enable_locking'),
                 (1, 'repo_enable_statistics'),
                 (0, 'clone_uri'),
                 (0, 'push_uri'),
                 (0, 'fork_id')
             ]
             for strip, k in update_keys:
                 if k in kwargs:
                     val = kwargs[k]
                     if strip:
                         k = remove_prefix(k, 'repo_')
 
                     setattr(cur_repo, k, val)
 
             new_name = cur_repo.get_new_name(kwargs['repo_name'])
             cur_repo.repo_name = new_name
 
             # if private flag is set, reset default permission to NONE
             if kwargs.get('repo_private'):
                 EMPTY_PERM = 'repository.none'
                 RepoModel().grant_user_permission(
                     repo=cur_repo, user=User.DEFAULT_USER, perm=EMPTY_PERM
                 )
 
             # handle extra fields
             for field in filter(lambda k: k.startswith(RepositoryField.PREFIX), kwargs):
                 k = RepositoryField.un_prefix_key(field)
                 ex_field = RepositoryField.get_by_key_name(
                     key=k, repo=cur_repo)
                 if ex_field:
                     ex_field.field_value = kwargs[field]
                     self.sa.add(ex_field)
 
             self.sa.add(cur_repo)
 
             if source_repo_name != new_name:
                 # rename repository
                 self._rename_filesystem_repo(
                     old=source_repo_name, new=new_name)
 
             return cur_repo
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def _create_repo(self, repo_name, repo_type, description, owner,
                      private=False, clone_uri=None, repo_group=None,
                      landing_rev='rev:tip', fork_of=None,
                      copy_fork_permissions=False, enable_statistics=False,
                      enable_locking=False, enable_downloads=False,
                      copy_group_permissions=False,
                      state=Repository.STATE_PENDING):
         """
         Create repository inside database with PENDING state, this should be
         only executed by create() repo. With exception of importing existing
         repos
         """
         from rhodecode.model.scm import ScmModel
 
         owner = self._get_user(owner)
         fork_of = self._get_repo(fork_of)
         repo_group = self._get_repo_group(safe_int(repo_group))
 
         try:
             repo_name = safe_unicode(repo_name)
             description = safe_unicode(description)
             # repo name is just a name of repository
             # while repo_name_full is a full qualified name that is combined
             # with name and path of group
             repo_name_full = repo_name
             repo_name = repo_name.split(Repository.NAME_SEP)[-1]
 
             new_repo = Repository()
             new_repo.repo_state = state
             new_repo.enable_statistics = False
             new_repo.repo_name = repo_name_full
             new_repo.repo_type = repo_type
             new_repo.user = owner
             new_repo.group = repo_group
             new_repo.description = description or repo_name
             new_repo.private = private
             new_repo.archived = False
             new_repo.clone_uri = clone_uri
             new_repo.landing_rev = landing_rev
 
             new_repo.enable_statistics = enable_statistics
             new_repo.enable_locking = enable_locking
             new_repo.enable_downloads = enable_downloads
 
             if repo_group:
                 new_repo.enable_locking = repo_group.enable_locking
 
             if fork_of:
                 parent_repo = fork_of
                 new_repo.fork = parent_repo
 
             events.trigger(events.RepoPreCreateEvent(new_repo))
 
             self.sa.add(new_repo)
 
             EMPTY_PERM = 'repository.none'
             if fork_of and copy_fork_permissions:
                 repo = fork_of
                 user_perms = UserRepoToPerm.query() \
                     .filter(UserRepoToPerm.repository == repo).all()
                 group_perms = UserGroupRepoToPerm.query() \
                     .filter(UserGroupRepoToPerm.repository == repo).all()
 
                 for perm in user_perms:
                     UserRepoToPerm.create(
                         perm.user, new_repo, perm.permission)
 
                 for perm in group_perms:
                     UserGroupRepoToPerm.create(
                         perm.users_group, new_repo, perm.permission)
                 # in case we copy permissions and also set this repo to private
                 # override the default user permission to make it a private repo
                 if private:
                     RepoModel(self.sa).grant_user_permission(
                         repo=new_repo, user=User.DEFAULT_USER, perm=EMPTY_PERM)
 
             elif repo_group and copy_group_permissions:
                 user_perms = UserRepoGroupToPerm.query() \
                     .filter(UserRepoGroupToPerm.group == repo_group).all()
 
                 group_perms = UserGroupRepoGroupToPerm.query() \
                     .filter(UserGroupRepoGroupToPerm.group == repo_group).all()
 
                 for perm in user_perms:
                     perm_name = perm.permission.permission_name.replace(
                         'group.', 'repository.')
                     perm_obj = Permission.get_by_key(perm_name)
                     UserRepoToPerm.create(perm.user, new_repo, perm_obj)
 
                 for perm in group_perms:
                     perm_name = perm.permission.permission_name.replace(
                         'group.', 'repository.')
                     perm_obj = Permission.get_by_key(perm_name)
                     UserGroupRepoToPerm.create(perm.users_group, new_repo, perm_obj)
 
                 if private:
                     RepoModel(self.sa).grant_user_permission(
                         repo=new_repo, user=User.DEFAULT_USER, perm=EMPTY_PERM)
 
             else:
                 perm_obj = self._create_default_perms(new_repo, private)
                 self.sa.add(perm_obj)
 
             # now automatically start following this repository as owner
             ScmModel(self.sa).toggle_following_repo(new_repo.repo_id, owner.user_id)
 
             # we need to flush here, in order to check if database won't
             # throw any exceptions, create filesystem dirs at the very end
             self.sa.flush()
             events.trigger(events.RepoCreateEvent(new_repo))
             return new_repo
 
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def create(self, form_data, cur_user):
         """
         Create repository using celery tasks
 
         :param form_data:
         :param cur_user:
         """
         from rhodecode.lib.celerylib import tasks, run_task
         return run_task(tasks.create_repo, form_data, cur_user)
 
     def update_permissions(self, repo, perm_additions=None, perm_updates=None,
                            perm_deletions=None, check_perms=True,
                            cur_user=None):
         if not perm_additions:
             perm_additions = []
         if not perm_updates:
             perm_updates = []
         if not perm_deletions:
             perm_deletions = []
 
         req_perms = ('usergroup.read', 'usergroup.write', 'usergroup.admin')
 
         changes = {
             'added': [],
             'updated': [],
             'deleted': []
         }
         # update permissions
         for member_id, perm, member_type in perm_updates:
             member_id = int(member_id)
             if member_type == 'user':
                 member_name = User.get(member_id).username
                 # this updates also current one if found
                 self.grant_user_permission(
                     repo=repo, user=member_id, perm=perm)
             elif member_type == 'user_group':
                 # check if we have permissions to alter this usergroup
                 member_name = UserGroup.get(member_id).users_group_name
                 if not check_perms or HasUserGroupPermissionAny(
                         *req_perms)(member_name, user=cur_user):
                     self.grant_user_group_permission(
                         repo=repo, group_name=member_id, perm=perm)
             else:
                 raise ValueError("member_type must be 'user' or 'user_group' "
                                  "got {} instead".format(member_type))
             changes['updated'].append({'type': member_type, 'id': member_id,
                                        'name': member_name, 'new_perm': perm})
 
         # set new permissions
         for member_id, perm, member_type in perm_additions:
             member_id = int(member_id)
             if member_type == 'user':
                 member_name = User.get(member_id).username
                 self.grant_user_permission(
                     repo=repo, user=member_id, perm=perm)
             elif member_type == 'user_group':
                 # check if we have permissions to alter this usergroup
                 member_name = UserGroup.get(member_id).users_group_name
                 if not check_perms or HasUserGroupPermissionAny(
                         *req_perms)(member_name, user=cur_user):
                     self.grant_user_group_permission(
                         repo=repo, group_name=member_id, perm=perm)
             else:
                 raise ValueError("member_type must be 'user' or 'user_group' "
                                  "got {} instead".format(member_type))
 
             changes['added'].append({'type': member_type, 'id': member_id,
                                      'name': member_name, 'new_perm': perm})
         # delete permissions
         for member_id, perm, member_type in perm_deletions:
             member_id = int(member_id)
             if member_type == 'user':
                 member_name = User.get(member_id).username
                 self.revoke_user_permission(repo=repo, user=member_id)
             elif member_type == 'user_group':
                 # check if we have permissions to alter this usergroup
                 member_name = UserGroup.get(member_id).users_group_name
                 if not check_perms or HasUserGroupPermissionAny(
                         *req_perms)(member_name, user=cur_user):
                     self.revoke_user_group_permission(
                         repo=repo, group_name=member_id)
             else:
                 raise ValueError("member_type must be 'user' or 'user_group' "
                                  "got {} instead".format(member_type))
 
             changes['deleted'].append({'type': member_type, 'id': member_id,
                                        'name': member_name, 'new_perm': perm})
         return changes
 
     def create_fork(self, form_data, cur_user):
         """
         Simple wrapper into executing celery task for fork creation
 
         :param form_data:
         :param cur_user:
         """
         from rhodecode.lib.celerylib import tasks, run_task
         return run_task(tasks.create_repo_fork, form_data, cur_user)
 
     def archive(self, repo):
         """
         Archive given repository. Set archive flag.
 
         :param repo:
         """
         repo = self._get_repo(repo)
         if repo:
 
             try:
                 repo.archived = True
                 self.sa.add(repo)
                 self.sa.commit()
             except Exception:
                 log.error(traceback.format_exc())
                 raise
 
     def delete(self, repo, forks=None, pull_requests=None, fs_remove=True, cur_user=None):
         """
         Delete given repository, forks parameter defines what do do with
         attached forks. Throws AttachedForksError if deleted repo has attached
         forks
 
         :param repo:
         :param forks: str 'delete' or 'detach'
         :param pull_requests: str 'delete' or None
         :param fs_remove: remove(archive) repo from filesystem
         """
         if not cur_user:
             cur_user = getattr(get_current_rhodecode_user(), 'username', None)
         repo = self._get_repo(repo)
         if repo:
             if forks == 'detach':
                 for r in repo.forks:
                     r.fork = None
                     self.sa.add(r)
             elif forks == 'delete':
                 for r in repo.forks:
                     self.delete(r, forks='delete')
             elif [f for f in repo.forks]:
                 raise AttachedForksError()
 
             # check for pull requests
             pr_sources = repo.pull_requests_source
             pr_targets = repo.pull_requests_target
             if pull_requests != 'delete' and (pr_sources or pr_targets):
                 raise AttachedPullRequestsError()
 
             old_repo_dict = repo.get_dict()
             events.trigger(events.RepoPreDeleteEvent(repo))
             try:
                 self.sa.delete(repo)
                 if fs_remove:
                     self._delete_filesystem_repo(repo)
                 else:
                     log.debug('skipping removal from filesystem')
                 old_repo_dict.update({
                     'deleted_by': cur_user,
                     'deleted_on': time.time(),
                 })
                 log_delete_repository(**old_repo_dict)
                 events.trigger(events.RepoDeleteEvent(repo))
             except Exception:
                 log.error(traceback.format_exc())
                 raise
 
     def grant_user_permission(self, repo, user, perm):
         """
         Grant permission for user on given repository, or update existing one
         if found
 
         :param repo: Instance of Repository, repository_id, or repository name
         :param user: Instance of User, user_id or username
         :param perm: Instance of Permission, or permission_name
         """
         user = self._get_user(user)
         repo = self._get_repo(repo)
         permission = self._get_perm(perm)
 
         # check if we have that permission already
         obj = self.sa.query(UserRepoToPerm) \
             .filter(UserRepoToPerm.user == user) \
             .filter(UserRepoToPerm.repository == repo) \
             .scalar()
         if obj is None:
             # create new !
             obj = UserRepoToPerm()
         obj.repository = repo
         obj.user = user
         obj.permission = permission
         self.sa.add(obj)
         log.debug('Granted perm %s to %s on %s', perm, user, repo)
         action_logger_generic(
             'granted permission: {} to user: {} on repo: {}'.format(
                 perm, user, repo), namespace='security.repo')
         return obj
 
     def revoke_user_permission(self, repo, user):
         """
         Revoke permission for user on given repository
 
         :param repo: Instance of Repository, repository_id, or repository name
         :param user: Instance of User, user_id or username
         """
 
         user = self._get_user(user)
         repo = self._get_repo(repo)
 
         obj = self.sa.query(UserRepoToPerm) \
             .filter(UserRepoToPerm.repository == repo) \
             .filter(UserRepoToPerm.user == user) \
             .scalar()
         if obj:
             self.sa.delete(obj)
             log.debug('Revoked perm on %s on %s', repo, user)
             action_logger_generic(
                 'revoked permission from user: {} on repo: {}'.format(
                     user, repo), namespace='security.repo')
 
     def grant_user_group_permission(self, repo, group_name, perm):
         """
         Grant permission for user group on given repository, or update
         existing one if found
 
         :param repo: Instance of Repository, repository_id, or repository name
         :param group_name: Instance of UserGroup, users_group_id,
             or user group name
         :param perm: Instance of Permission, or permission_name
         """
         repo = self._get_repo(repo)
         group_name = self._get_user_group(group_name)
         permission = self._get_perm(perm)
 
         # check if we have that permission already
         obj = self.sa.query(UserGroupRepoToPerm) \
             .filter(UserGroupRepoToPerm.users_group == group_name) \
             .filter(UserGroupRepoToPerm.repository == repo) \
             .scalar()
 
         if obj is None:
             # create new
             obj = UserGroupRepoToPerm()
 
         obj.repository = repo
         obj.users_group = group_name
         obj.permission = permission
         self.sa.add(obj)
         log.debug('Granted perm %s to %s on %s', perm, group_name, repo)
         action_logger_generic(
             'granted permission: {} to usergroup: {} on repo: {}'.format(
                 perm, group_name, repo), namespace='security.repo')
 
         return obj
 
     def revoke_user_group_permission(self, repo, group_name):
         """
         Revoke permission for user group on given repository
 
         :param repo: Instance of Repository, repository_id, or repository name
         :param group_name: Instance of UserGroup, users_group_id,
             or user group name
         """
         repo = self._get_repo(repo)
         group_name = self._get_user_group(group_name)
 
         obj = self.sa.query(UserGroupRepoToPerm) \
             .filter(UserGroupRepoToPerm.repository == repo) \
             .filter(UserGroupRepoToPerm.users_group == group_name) \
             .scalar()
         if obj:
             self.sa.delete(obj)
             log.debug('Revoked perm to %s on %s', repo, group_name)
             action_logger_generic(
                 'revoked permission from usergroup: {} on repo: {}'.format(
                     group_name, repo), namespace='security.repo')
 
     def delete_stats(self, repo_name):
         """
         removes stats for given repo
 
         :param repo_name:
         """
         repo = self._get_repo(repo_name)
         try:
             obj = self.sa.query(Statistics) \
                 .filter(Statistics.repository == repo).scalar()
             if obj:
                 self.sa.delete(obj)
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def add_repo_field(self, repo_name, field_key, field_label, field_value='',
                        field_type='str', field_desc=''):
 
         repo = self._get_repo(repo_name)
 
         new_field = RepositoryField()
         new_field.repository = repo
         new_field.field_key = field_key
         new_field.field_type = field_type  # python type
         new_field.field_value = field_value
         new_field.field_desc = field_desc
         new_field.field_label = field_label
         self.sa.add(new_field)
         return new_field
 
     def delete_repo_field(self, repo_name, field_key):
         repo = self._get_repo(repo_name)
         field = RepositoryField.get_by_key_name(field_key, repo)
         if field:
             self.sa.delete(field)
 
     def _create_filesystem_repo(self, repo_name, repo_type, repo_group,
                                 clone_uri=None, repo_store_location=None,
                                 use_global_config=False, install_hooks=True):
         """
         makes repository on filesystem. It's group aware means it'll create
         a repository within a group, and alter the paths accordingly of
         group location
 
         :param repo_name:
         :param alias:
         :param parent:
         :param clone_uri:
         :param repo_store_location:
         """
         from rhodecode.lib.utils import is_valid_repo, is_valid_repo_group
         from rhodecode.model.scm import ScmModel
 
         if Repository.NAME_SEP in repo_name:
             raise ValueError(
                 'repo_name must not contain groups got `%s`' % repo_name)
 
         if isinstance(repo_group, RepoGroup):
             new_parent_path = os.sep.join(repo_group.full_path_splitted)
         else:
             new_parent_path = repo_group or ''
 
         if repo_store_location:
             _paths = [repo_store_location]
         else:
             _paths = [self.repos_path, new_parent_path, repo_name]
             # we need to make it str for mercurial
         repo_path = os.path.join(*map(lambda x: safe_str(x), _paths))
 
         # check if this path is not a repository
         if is_valid_repo(repo_path, self.repos_path):
             raise Exception('This path %s is a valid repository' % repo_path)
 
         # check if this path is a group
         if is_valid_repo_group(repo_path, self.repos_path):
             raise Exception('This path %s is a valid group' % repo_path)
 
         log.info('creating repo %s in %s from url: `%s`',
                  repo_name, safe_unicode(repo_path),
                  obfuscate_url_pw(clone_uri))
 
         backend = get_backend(repo_type)
 
         config_repo = None if use_global_config else repo_name
         if config_repo and new_parent_path:
             config_repo = Repository.NAME_SEP.join(
                 (new_parent_path, config_repo))
         config = make_db_config(clear_session=False, repo=config_repo)
         config.set('extensions', 'largefiles', '')
 
         # patch and reset hooks section of UI config to not run any
         # hooks on creating remote repo
         config.clear_section('hooks')
 
         # TODO: johbo: Unify this, hardcoded "bare=True" does not look nice
         if repo_type == 'git':
             repo = backend(
                 repo_path, config=config, create=True, src_url=clone_uri, bare=True,
                 with_wire={"cache": False})
         else:
             repo = backend(
                 repo_path, config=config, create=True, src_url=clone_uri,
                 with_wire={"cache": False})
 
         if install_hooks:
             repo.install_hooks()
 
         log.debug('Created repo %s with %s backend',
                   safe_unicode(repo_name), safe_unicode(repo_type))
         return repo
 
     def _rename_filesystem_repo(self, old, new):
         """
         renames repository on filesystem
 
         :param old: old name
         :param new: new name
         """
         log.info('renaming repo from %s to %s', old, new)
 
         old_path = os.path.join(self.repos_path, old)
         new_path = os.path.join(self.repos_path, new)
         if os.path.isdir(new_path):
             raise Exception(
                 'Was trying to rename to already existing dir %s' % new_path
             )
         shutil.move(old_path, new_path)
 
     def _delete_filesystem_repo(self, repo):
         """
         removes repo from filesystem, the removal is acctually made by
         added rm__ prefix into dir, and rename internat .hg/.git dirs so this
         repository is no longer valid for rhodecode, can be undeleted later on
         by reverting the renames on this repository
 
         :param repo: repo object
         """
         rm_path = os.path.join(self.repos_path, repo.repo_name)
         repo_group = repo.group
         log.info("Removing repository %s", rm_path)
         # disable hg/git internal that it doesn't get detected as repo
         alias = repo.repo_type
 
         config = make_db_config(clear_session=False)
         config.set('extensions', 'largefiles', '')
         bare = getattr(repo.scm_instance(config=config), 'bare', False)
 
         # skip this for bare git repos
         if not bare:
             # disable VCS repo
             vcs_path = os.path.join(rm_path, '.%s' % alias)
             if os.path.exists(vcs_path):
                 shutil.move(vcs_path, os.path.join(rm_path, 'rm__.%s' % alias))
 
         _now = datetime.datetime.now()
         _ms = str(_now.microsecond).rjust(6, '0')
         _d = 'rm__%s__%s' % (_now.strftime('%Y%m%d_%H%M%S_' + _ms),
                              repo.just_name)
         if repo_group:
             # if repository is in group, prefix the removal path with the group
             args = repo_group.full_path_splitted + [_d]
             _d = os.path.join(*args)
 
         if os.path.isdir(rm_path):
             shutil.move(rm_path, os.path.join(self.repos_path, _d))
 
         # finally cleanup diff-cache if it exists
         cached_diffs_dir = repo.cached_diffs_dir
         if os.path.isdir(cached_diffs_dir):
             shutil.rmtree(cached_diffs_dir)
 
 
 class ReadmeFinder:
     """
     Utility which knows how to find a readme for a specific commit.
 
     The main idea is that this is a configurable algorithm. When creating an
     instance you can define parameters, currently only the `default_renderer`.
     Based on this configuration the method :meth:`search` behaves slightly
     different.
     """
 
     readme_re = re.compile(r'^readme(\.[^\.]+)?$', re.IGNORECASE)
     path_re = re.compile(r'^docs?', re.IGNORECASE)
 
     default_priorities = {
         None: 0,
         '.text': 2,
         '.txt': 3,
         '.rst': 1,
         '.rest': 2,
         '.md': 1,
         '.mkdn': 2,
         '.mdown': 3,
         '.markdown': 4,
     }
 
     path_priority = {
         'doc': 0,
         'docs': 1,
     }
 
     FALLBACK_PRIORITY = 99
 
     RENDERER_TO_EXTENSION = {
         'rst': ['.rst', '.rest'],
         'markdown': ['.md', 'mkdn', '.mdown', '.markdown'],
     }
 
     def __init__(self, default_renderer=None):
         self._default_renderer = default_renderer
         self._renderer_extensions = self.RENDERER_TO_EXTENSION.get(
             default_renderer, [])
 
     def search(self, commit, path='/'):
         """
         Find a readme in the given `commit`.
         """
         nodes = commit.get_nodes(path)
         matches = self._match_readmes(nodes)
         matches = self._sort_according_to_priority(matches)
         if matches:
             return matches[0].node
 
         paths = self._match_paths(nodes)
         paths = self._sort_paths_according_to_priority(paths)
         for path in paths:
             match = self.search(commit, path=path)
             if match:
                 return match
 
         return None
 
     def _match_readmes(self, nodes):
         for node in nodes:
             if not node.is_file():
                 continue
             path = node.path.rsplit('/', 1)[-1]
             match = self.readme_re.match(path)
             if match:
                 extension = match.group(1)
                 yield ReadmeMatch(node, match, self._priority(extension))
 
     def _match_paths(self, nodes):
         for node in nodes:
             if not node.is_dir():
                 continue
             match = self.path_re.match(node.path)
             if match:
                 yield node.path
 
     def _priority(self, extension):
         renderer_priority = (
             0 if extension in self._renderer_extensions else 1)
         extension_priority = self.default_priorities.get(
             extension, self.FALLBACK_PRIORITY)
         return (renderer_priority, extension_priority)
 
     def _sort_according_to_priority(self, matches):
 
         def priority_and_path(match):
             return (match.priority, match.path)
 
         return sorted(matches, key=priority_and_path)
 
     def _sort_paths_according_to_priority(self, paths):
 
         def priority_and_path(path):
             return (self.path_priority.get(path, self.FALLBACK_PRIORITY), path)
 
         return sorted(paths, key=priority_and_path)
 
 
 class ReadmeMatch:
 
     def __init__(self, node, match, priority):
         self.node = node
         self._match = match
         self.priority = priority
 
     @property
     def path(self):
         return self.node.path
 
     def __repr__(self):
         return '<ReadmeMatch {} priority={}'.format(self.path, self.priority)

rhodecode/model/repo_group.py

@@ -1,877 +1,876 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 repo group model for RhodeCode
 """
 
 import os
 import datetime
 import itertools
 import logging
 import shutil
 import time
 import traceback
 import string
 
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 from rhodecode import events
 from rhodecode.model import BaseModel
 from rhodecode.model.db import (_hash_key, func, or_, in_filter_generator,
     Session, RepoGroup, UserRepoGroupToPerm, User, Permission, UserGroupRepoGroupToPerm,
     UserGroup, Repository)
 from rhodecode.model.settings import VcsSettingsModel, SettingsModel
 from rhodecode.lib.caching_query import FromCache
 from rhodecode.lib.utils2 import action_logger_generic, datetime_to_time
 
 log = logging.getLogger(__name__)
 
 
 class RepoGroupModel(BaseModel):
 
     cls = RepoGroup
     PERSONAL_GROUP_DESC = 'personal repo group of user `%(username)s`'
     PERSONAL_GROUP_PATTERN = '${username}'  # default
 
     def _get_user_group(self, users_group):
         return self._get_instance(UserGroup, users_group,
                                   callback=UserGroup.get_by_group_name)
 
     def _get_repo_group(self, repo_group):
         return self._get_instance(RepoGroup, repo_group,
                                   callback=RepoGroup.get_by_group_name)
 
     @LazyProperty
     def repos_path(self):
         """
         Gets the repositories root path from database
         """
 
         settings_model = VcsSettingsModel(sa=self.sa)
         return settings_model.get_repos_location()
 
     def get_by_group_name(self, repo_group_name, cache=None):
         repo = self.sa.query(RepoGroup) \
             .filter(RepoGroup.group_name == repo_group_name)
 
         if cache:
             name_key = _hash_key(repo_group_name)
             repo = repo.options(
                 FromCache("sql_cache_short", "get_repo_group_%s" % name_key))
         return repo.scalar()
 
     def get_default_create_personal_repo_group(self):
         value = SettingsModel().get_setting_by_name(
             'create_personal_repo_group')
         return value.app_settings_value if value else None or False
 
     def get_personal_group_name_pattern(self):
         value = SettingsModel().get_setting_by_name(
             'personal_repo_group_pattern')
         val = value.app_settings_value if value else None
         group_template = val or self.PERSONAL_GROUP_PATTERN
 
         group_template = group_template.lstrip('/')
         return group_template
 
     def get_personal_group_name(self, user):
         template = self.get_personal_group_name_pattern()
         return string.Template(template).safe_substitute(
             username=user.username,
             user_id=user.user_id,
             first_name=user.first_name,
             last_name=user.last_name,
         )
 
     def create_personal_repo_group(self, user, commit_early=True):
         desc = self.PERSONAL_GROUP_DESC % {'username': user.username}
         personal_repo_group_name = self.get_personal_group_name(user)
 
         # create a new one
         RepoGroupModel().create(
             group_name=personal_repo_group_name,
             group_description=desc,
             owner=user.username,
             personal=True,
             commit_early=commit_early)
 
     def _create_default_perms(self, new_group):
         # create default permission
         default_perm = 'group.read'
         def_user = User.get_default_user()
         for p in def_user.user_perms:
             if p.permission.permission_name.startswith('group.'):
                 default_perm = p.permission.permission_name
                 break
 
         repo_group_to_perm = UserRepoGroupToPerm()
         repo_group_to_perm.permission = Permission.get_by_key(default_perm)
 
         repo_group_to_perm.group = new_group
         repo_group_to_perm.user_id = def_user.user_id
         return repo_group_to_perm
 
     def _get_group_name_and_parent(self, group_name_full, repo_in_path=False,
                                    get_object=False):
         """
         Get's the group name and a parent group name from given group name.
         If repo_in_path is set to truth, we asume the full path also includes
         repo name, in such case we clean the last element.
 
         :param group_name_full:
         """
         split_paths = 1
         if repo_in_path:
             split_paths = 2
         _parts = group_name_full.rsplit(RepoGroup.url_sep(), split_paths)
 
         if repo_in_path and len(_parts) > 1:
             # such case last element is the repo_name
             _parts.pop(-1)
         group_name_cleaned = _parts[-1]  # just the group name
         parent_repo_group_name = None
 
         if len(_parts) > 1:
             parent_repo_group_name = _parts[0]
 
         parent_group = None
         if parent_repo_group_name:
             parent_group = RepoGroup.get_by_group_name(parent_repo_group_name)
 
         if get_object:
             return group_name_cleaned, parent_repo_group_name, parent_group
 
         return group_name_cleaned, parent_repo_group_name
 
     def check_exist_filesystem(self, group_name, exc_on_failure=True):
         create_path = os.path.join(self.repos_path, group_name)
         log.debug('creating new group in %s', create_path)
 
         if os.path.isdir(create_path):
             if exc_on_failure:
                 abs_create_path = os.path.abspath(create_path)
                 raise Exception('Directory `{}` already exists !'.format(abs_create_path))
             return False
         return True
 
     def _create_group(self, group_name):
         """
         makes repository group on filesystem
 
         :param repo_name:
         :param parent_id:
         """
 
         self.check_exist_filesystem(group_name)
         create_path = os.path.join(self.repos_path, group_name)
         log.debug('creating new group in %s', create_path)
         os.makedirs(create_path, mode=0o755)
         log.debug('created group in %s', create_path)
 
     def _rename_group(self, old, new):
         """
         Renames a group on filesystem
 
         :param group_name:
         """
 
         if old == new:
             log.debug('skipping group rename')
             return
 
         log.debug('renaming repository group from %s to %s', old, new)
 
         old_path = os.path.join(self.repos_path, old)
         new_path = os.path.join(self.repos_path, new)
 
         log.debug('renaming repos paths from %s to %s', old_path, new_path)
 
         if os.path.isdir(new_path):
             raise Exception('Was trying to rename to already '
                             'existing dir %s' % new_path)
         shutil.move(old_path, new_path)
 
     def _delete_filesystem_group(self, group, force_delete=False):
         """
         Deletes a group from a filesystem
 
         :param group: instance of group from database
         :param force_delete: use shutil rmtree to remove all objects
         """
         paths = group.full_path.split(RepoGroup.url_sep())
         paths = os.sep.join(paths)
 
         rm_path = os.path.join(self.repos_path, paths)
         log.info("Removing group %s", rm_path)
         # delete only if that path really exists
         if os.path.isdir(rm_path):
             if force_delete:
                 shutil.rmtree(rm_path)
             else:
                 # archive that group`
                 _now = datetime.datetime.now()
                 _ms = str(_now.microsecond).rjust(6, '0')
                 _d = 'rm__%s_GROUP_%s' % (
                     _now.strftime('%Y%m%d_%H%M%S_' + _ms), group.name)
                 shutil.move(rm_path, os.path.join(self.repos_path, _d))
 
     def create(self, group_name, group_description, owner, just_db=False,
                copy_permissions=False, personal=None, commit_early=True):
 
         (group_name_cleaned,
          parent_group_name) = RepoGroupModel()._get_group_name_and_parent(group_name)
 
         parent_group = None
         if parent_group_name:
             parent_group = self._get_repo_group(parent_group_name)
             if not parent_group:
                 # we tried to create a nested group, but the parent is not
                 # existing
                 raise ValueError(
                     'Parent group `%s` given in `%s` group name '
                     'is not yet existing.' % (parent_group_name, group_name))
 
         # because we are doing a cleanup, we need to check if such directory
         # already exists. If we don't do that we can accidentally delete
         # existing directory via cleanup that can cause data issues, since
         # delete does a folder rename to special syntax later cleanup
         # functions can delete this
         cleanup_group = self.check_exist_filesystem(group_name,
                                                     exc_on_failure=False)
         user = self._get_user(owner)
         if not user:
             raise ValueError('Owner %s not found as rhodecode user', owner)
 
         try:
             new_repo_group = RepoGroup()
             new_repo_group.user = user
             new_repo_group.group_description = group_description or group_name
             new_repo_group.parent_group = parent_group
             new_repo_group.group_name = group_name
             new_repo_group.personal = personal
 
             self.sa.add(new_repo_group)
 
             # create an ADMIN permission for owner except if we're super admin,
             # later owner should go into the owner field of groups
             if not user.is_admin:
                 self.grant_user_permission(repo_group=new_repo_group,
                                            user=owner, perm='group.admin')
 
             if parent_group and copy_permissions:
                 # copy permissions from parent
                 user_perms = UserRepoGroupToPerm.query() \
                     .filter(UserRepoGroupToPerm.group == parent_group).all()
 
                 group_perms = UserGroupRepoGroupToPerm.query() \
                     .filter(UserGroupRepoGroupToPerm.group == parent_group).all()
 
                 for perm in user_perms:
                     # don't copy over the permission for user who is creating
                     # this group, if he is not super admin he get's admin
                     # permission set above
                     if perm.user != user or user.is_admin:
                         UserRepoGroupToPerm.create(
                             perm.user, new_repo_group, perm.permission)
 
                 for perm in group_perms:
                     UserGroupRepoGroupToPerm.create(
                         perm.users_group, new_repo_group, perm.permission)
             else:
                 perm_obj = self._create_default_perms(new_repo_group)
                 self.sa.add(perm_obj)
 
             # now commit the changes, earlier so we are sure everything is in
             # the database.
             if commit_early:
                 self.sa.commit()
             if not just_db:
                 self._create_group(new_repo_group.group_name)
 
             # trigger the post hook
             from rhodecode.lib.hooks_base import log_create_repository_group
             repo_group = RepoGroup.get_by_group_name(group_name)
 
             # update repo group commit caches initially
             repo_group.update_commit_cache()
 
             log_create_repository_group(
                 created_by=user.username, **repo_group.get_dict())
 
             # Trigger create event.
             events.trigger(events.RepoGroupCreateEvent(repo_group))
 
             return new_repo_group
         except Exception:
             self.sa.rollback()
             log.exception('Exception occurred when creating repository group, '
                           'doing cleanup...')
             # rollback things manually !
             repo_group = RepoGroup.get_by_group_name(group_name)
             if repo_group:
                 RepoGroup.delete(repo_group.group_id)
                 self.sa.commit()
                 if cleanup_group:
                     RepoGroupModel()._delete_filesystem_group(repo_group)
             raise
 
     def update_permissions(
             self, repo_group, perm_additions=None, perm_updates=None,
             perm_deletions=None, recursive=None, check_perms=True,
             cur_user=None):
         from rhodecode.model.repo import RepoModel
         from rhodecode.lib.auth import HasUserGroupPermissionAny
 
         if not perm_additions:
             perm_additions = []
         if not perm_updates:
             perm_updates = []
         if not perm_deletions:
             perm_deletions = []
 
         req_perms = ('usergroup.read', 'usergroup.write', 'usergroup.admin')
 
         changes = {
             'added': [],
             'updated': [],
             'deleted': []
         }
 
         def _set_perm_user(obj, user, perm):
             if isinstance(obj, RepoGroup):
                 self.grant_user_permission(
                     repo_group=obj, user=user, perm=perm)
             elif isinstance(obj, Repository):
                 # private repos will not allow to change the default
                 # permissions using recursive mode
                 if obj.private and user == User.DEFAULT_USER:
                     return
 
                 # we set group permission but we have to switch to repo
                 # permission
                 perm = perm.replace('group.', 'repository.')
                 RepoModel().grant_user_permission(
                     repo=obj, user=user, perm=perm)
 
         def _set_perm_group(obj, users_group, perm):
             if isinstance(obj, RepoGroup):
                 self.grant_user_group_permission(
                     repo_group=obj, group_name=users_group, perm=perm)
             elif isinstance(obj, Repository):
                 # we set group permission but we have to switch to repo
                 # permission
                 perm = perm.replace('group.', 'repository.')
                 RepoModel().grant_user_group_permission(
                     repo=obj, group_name=users_group, perm=perm)
 
         def _revoke_perm_user(obj, user):
             if isinstance(obj, RepoGroup):
                 self.revoke_user_permission(repo_group=obj, user=user)
             elif isinstance(obj, Repository):
                 RepoModel().revoke_user_permission(repo=obj, user=user)
 
         def _revoke_perm_group(obj, user_group):
             if isinstance(obj, RepoGroup):
                 self.revoke_user_group_permission(
                     repo_group=obj, group_name=user_group)
             elif isinstance(obj, Repository):
                 RepoModel().revoke_user_group_permission(
                     repo=obj, group_name=user_group)
 
         # start updates
         log.debug('Now updating permissions for %s in recursive mode:%s',
                   repo_group, recursive)
 
         # initialize check function, we'll call that multiple times
         has_group_perm = HasUserGroupPermissionAny(*req_perms)
 
         for obj in repo_group.recursive_groups_and_repos():
             # iterated obj is an instance of a repos group or repository in
             # that group, recursive option can be: none, repos, groups, all
             if recursive == 'all':
                 obj = obj
             elif recursive == 'repos':
                 # skip groups, other than this one
                 if isinstance(obj, RepoGroup) and not obj == repo_group:
                     continue
             elif recursive == 'groups':
                 # skip repos
                 if isinstance(obj, Repository):
                     continue
             else:  # recursive == 'none':
                 #  DEFAULT option - don't apply to iterated objects
                 # also we do a break at the end of this loop. if we are not
                 # in recursive mode
                 obj = repo_group
 
             change_obj = obj.get_api_data()
 
             # update permissions
             for member_id, perm, member_type in perm_updates:
                 member_id = int(member_id)
                 if member_type == 'user':
                     member_name = User.get(member_id).username
                     # this updates also current one if found
                     _set_perm_user(obj, user=member_id, perm=perm)
                 elif member_type == 'user_group':
                     member_name = UserGroup.get(member_id).users_group_name
                     if not check_perms or has_group_perm(member_name,
                                                          user=cur_user):
                         _set_perm_group(obj, users_group=member_id, perm=perm)
                 else:
                     raise ValueError("member_type must be 'user' or 'user_group' "
                                      "got {} instead".format(member_type))
 
                 changes['updated'].append(
                     {'change_obj': change_obj, 'type': member_type,
                      'id': member_id, 'name': member_name, 'new_perm': perm})
 
             # set new permissions
             for member_id, perm, member_type in perm_additions:
                 member_id = int(member_id)
                 if member_type == 'user':
                     member_name = User.get(member_id).username
                     _set_perm_user(obj, user=member_id, perm=perm)
                 elif member_type == 'user_group':
                     # check if we have permissions to alter this usergroup
                     member_name = UserGroup.get(member_id).users_group_name
                     if not check_perms or has_group_perm(member_name,
                                                          user=cur_user):
                         _set_perm_group(obj, users_group=member_id, perm=perm)
                 else:
                     raise ValueError("member_type must be 'user' or 'user_group' "
                                      "got {} instead".format(member_type))
 
                 changes['added'].append(
                     {'change_obj': change_obj, 'type': member_type,
                      'id': member_id, 'name': member_name, 'new_perm': perm})
 
             # delete permissions
             for member_id, perm, member_type in perm_deletions:
                 member_id = int(member_id)
                 if member_type == 'user':
                     member_name = User.get(member_id).username
                     _revoke_perm_user(obj, user=member_id)
                 elif member_type == 'user_group':
                     # check if we have permissions to alter this usergroup
                     member_name = UserGroup.get(member_id).users_group_name
                     if not check_perms or has_group_perm(member_name,
                                                          user=cur_user):
                         _revoke_perm_group(obj, user_group=member_id)
                 else:
                     raise ValueError("member_type must be 'user' or 'user_group' "
                                      "got {} instead".format(member_type))
 
                 changes['deleted'].append(
                     {'change_obj': change_obj, 'type': member_type,
                      'id': member_id, 'name': member_name, 'new_perm': perm})
 
             # if it's not recursive call for all,repos,groups
             # break the loop and don't proceed with other changes
             if recursive not in ['all', 'repos', 'groups']:
                 break
 
         return changes
 
     def update(self, repo_group, form_data):
         try:
             repo_group = self._get_repo_group(repo_group)
             old_path = repo_group.full_path
 
             # change properties
             if 'group_description' in form_data:
                 repo_group.group_description = form_data['group_description']
 
             if 'enable_locking' in form_data:
                 repo_group.enable_locking = form_data['enable_locking']
 
             if 'group_parent_id' in form_data:
                 parent_group = (
                     self._get_repo_group(form_data['group_parent_id']))
                 repo_group.group_parent_id = (
                     parent_group.group_id if parent_group else None)
                 repo_group.parent_group = parent_group
 
             # mikhail: to update the full_path, we have to explicitly
             # update group_name
             group_name = form_data.get('group_name', repo_group.name)
             repo_group.group_name = repo_group.get_new_name(group_name)
 
             new_path = repo_group.full_path
 
             if 'user' in form_data:
                 repo_group.user = User.get_by_username(form_data['user'])
 
             self.sa.add(repo_group)
 
             # iterate over all members of this groups and do fixes
             # set locking if given
             # if obj is a repoGroup also fix the name of the group according
             # to the parent
             # if obj is a Repo fix it's name
             # this can be potentially heavy operation
             for obj in repo_group.recursive_groups_and_repos():
                 # set the value from it's parent
                 obj.enable_locking = repo_group.enable_locking
                 if isinstance(obj, RepoGroup):
                     new_name = obj.get_new_name(obj.name)
                     log.debug('Fixing group %s to new name %s',
                               obj.group_name, new_name)
                     obj.group_name = new_name
 
                 elif isinstance(obj, Repository):
                     # we need to get all repositories from this new group and
                     # rename them accordingly to new group path
                     new_name = obj.get_new_name(obj.just_name)
                     log.debug('Fixing repo %s to new name %s',
                               obj.repo_name, new_name)
                     obj.repo_name = new_name
 
                 self.sa.add(obj)
 
             self._rename_group(old_path, new_path)
 
             # Trigger update event.
             events.trigger(events.RepoGroupUpdateEvent(repo_group))
 
             return repo_group
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def delete(self, repo_group, force_delete=False, fs_remove=True):
         repo_group = self._get_repo_group(repo_group)
         if not repo_group:
             return False
         try:
             self.sa.delete(repo_group)
             if fs_remove:
                 self._delete_filesystem_group(repo_group, force_delete)
             else:
                 log.debug('skipping removal from filesystem')
 
             # Trigger delete event.
             events.trigger(events.RepoGroupDeleteEvent(repo_group))
             return True
 
         except Exception:
             log.error('Error removing repo_group %s', repo_group)
             raise
 
     def grant_user_permission(self, repo_group, user, perm):
         """
         Grant permission for user on given repository group, or update
         existing one if found
 
         :param repo_group: Instance of RepoGroup, repositories_group_id,
             or repositories_group name
         :param user: Instance of User, user_id or username
         :param perm: Instance of Permission, or permission_name
         """
 
         repo_group = self._get_repo_group(repo_group)
         user = self._get_user(user)
         permission = self._get_perm(perm)
 
         # check if we have that permission already
         obj = self.sa.query(UserRepoGroupToPerm)\
             .filter(UserRepoGroupToPerm.user == user)\
             .filter(UserRepoGroupToPerm.group == repo_group)\
             .scalar()
         if obj is None:
             # create new !
             obj = UserRepoGroupToPerm()
         obj.group = repo_group
         obj.user = user
         obj.permission = permission
         self.sa.add(obj)
         log.debug('Granted perm %s to %s on %s', perm, user, repo_group)
         action_logger_generic(
             'granted permission: {} to user: {} on repogroup: {}'.format(
                 perm, user, repo_group), namespace='security.repogroup')
         return obj
 
     def revoke_user_permission(self, repo_group, user):
         """
         Revoke permission for user on given repository group
 
         :param repo_group: Instance of RepoGroup, repositories_group_id,
             or repositories_group name
         :param user: Instance of User, user_id or username
         """
 
         repo_group = self._get_repo_group(repo_group)
         user = self._get_user(user)
 
         obj = self.sa.query(UserRepoGroupToPerm)\
             .filter(UserRepoGroupToPerm.user == user)\
             .filter(UserRepoGroupToPerm.group == repo_group)\
             .scalar()
         if obj:
             self.sa.delete(obj)
             log.debug('Revoked perm on %s on %s', repo_group, user)
             action_logger_generic(
                 'revoked permission from user: {} on repogroup: {}'.format(
                     user, repo_group), namespace='security.repogroup')
 
     def grant_user_group_permission(self, repo_group, group_name, perm):
         """
         Grant permission for user group on given repository group, or update
         existing one if found
 
         :param repo_group: Instance of RepoGroup, repositories_group_id,
             or repositories_group name
         :param group_name: Instance of UserGroup, users_group_id,
             or user group name
         :param perm: Instance of Permission, or permission_name
         """
         repo_group = self._get_repo_group(repo_group)
         group_name = self._get_user_group(group_name)
         permission = self._get_perm(perm)
 
         # check if we have that permission already
         obj = self.sa.query(UserGroupRepoGroupToPerm)\
             .filter(UserGroupRepoGroupToPerm.group == repo_group)\
             .filter(UserGroupRepoGroupToPerm.users_group == group_name)\
             .scalar()
 
         if obj is None:
             # create new
             obj = UserGroupRepoGroupToPerm()
 
         obj.group = repo_group
         obj.users_group = group_name
         obj.permission = permission
         self.sa.add(obj)
         log.debug('Granted perm %s to %s on %s', perm, group_name, repo_group)
         action_logger_generic(
             'granted permission: {} to usergroup: {} on repogroup: {}'.format(
                 perm, group_name, repo_group), namespace='security.repogroup')
         return obj
 
     def revoke_user_group_permission(self, repo_group, group_name):
         """
         Revoke permission for user group on given repository group
 
         :param repo_group: Instance of RepoGroup, repositories_group_id,
             or repositories_group name
         :param group_name: Instance of UserGroup, users_group_id,
             or user group name
         """
         repo_group = self._get_repo_group(repo_group)
         group_name = self._get_user_group(group_name)
 
         obj = self.sa.query(UserGroupRepoGroupToPerm)\
             .filter(UserGroupRepoGroupToPerm.group == repo_group)\
             .filter(UserGroupRepoGroupToPerm.users_group == group_name)\
             .scalar()
         if obj:
             self.sa.delete(obj)
             log.debug('Revoked perm to %s on %s', repo_group, group_name)
             action_logger_generic(
                 'revoked permission from usergroup: {} on repogroup: {}'.format(
                     group_name, repo_group), namespace='security.repogroup')
 
     @classmethod
     def update_commit_cache(cls, repo_groups=None):
         if not repo_groups:
             repo_groups = RepoGroup.getAll()
         for repo_group in repo_groups:
             repo_group.update_commit_cache()
 
 
 
     def get_repo_groups_as_dict(self, repo_group_list=None, admin=False,
                                 super_user_actions=False):
 
         from pyramid.threadlocal import get_current_request
         _render = get_current_request().get_partial_renderer(
             'rhodecode:templates/data_table/_dt_elements.mako')
         c = _render.get_call_context()
         h = _render.get_helpers()
 
         def quick_menu(repo_group_name):
             return _render('quick_repo_group_menu', repo_group_name)
 
         def repo_group_lnk(repo_group_name):
             return _render('repo_group_name', repo_group_name)
 
         def last_change(last_change):
             if admin and isinstance(last_change, datetime.datetime) and not last_change.tzinfo:
                 ts = time.time()
                 utc_offset = (datetime.datetime.fromtimestamp(ts)
                               - datetime.datetime.utcfromtimestamp(ts)).total_seconds()
                 last_change = last_change + datetime.timedelta(seconds=utc_offset)
             return _render("last_change", last_change)
 
         def desc(desc, personal):
             return _render(
                 'repo_group_desc', desc, personal, c.visual.stylify_metatags)
 
         def repo_group_actions(repo_group_id, repo_group_name, gr_count):
             return _render(
                 'repo_group_actions', repo_group_id, repo_group_name, gr_count)
 
         def repo_group_name(repo_group_name, children_groups):
             return _render("repo_group_name", repo_group_name, children_groups)
 
         def user_profile(username):
             return _render('user_profile', username)
 
         repo_group_data = []
         for group in repo_group_list:
             # NOTE(marcink): because we use only raw column we need to load it like that
             changeset_cache = RepoGroup._load_changeset_cache(
                 '', group._changeset_cache)
             last_commit_change = RepoGroup._load_commit_change(changeset_cache)
             row = {
                 "menu": quick_menu(group.group_name),
                 "name": repo_group_lnk(group.group_name),
                 "name_raw": group.group_name,
 
                 "last_change": last_change(last_commit_change),
-                "last_change_raw": datetime_to_time(last_commit_change),
 
                 "last_changeset": "",
                 "last_changeset_raw": "",
 
                 "desc": desc(group.group_description, group.personal),
                 "top_level_repos": 0,
                 "owner": user_profile(group.User.username)
             }
             if admin:
                 repo_count = group.repositories.count()
                 children_groups = map(
                     h.safe_unicode,
                     itertools.chain((g.name for g in group.parents),
                                     (x.name for x in [group])))
                 row.update({
                     "action": repo_group_actions(
                         group.group_id, group.group_name, repo_count),
                     "top_level_repos": repo_count,
                     "name": repo_group_name(group.group_name, children_groups),
 
                 })
             repo_group_data.append(row)
 
         return repo_group_data
 
     def get_repo_groups_data_table(
             self, draw, start, limit,
             search_q, order_by, order_dir,
             auth_user, repo_group_id):
         from rhodecode.model.scm import RepoGroupList
 
         _perms = ['group.read', 'group.write', 'group.admin']
         repo_groups = RepoGroup.query() \
             .filter(RepoGroup.group_parent_id == repo_group_id) \
             .all()
         auth_repo_group_list = RepoGroupList(
             repo_groups, perm_set=_perms,
             extra_kwargs=dict(user=auth_user))
 
         allowed_ids = [-1]
         for repo_group in auth_repo_group_list:
             allowed_ids.append(repo_group.group_id)
 
         repo_groups_data_total_count = RepoGroup.query() \
             .filter(RepoGroup.group_parent_id == repo_group_id) \
             .filter(or_(
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(RepoGroup.group_id, allowed_ids))
             ) \
             .count()
 
         base_q = Session.query(
             RepoGroup.group_name,
             RepoGroup.group_name_hash,
             RepoGroup.group_description,
             RepoGroup.group_id,
             RepoGroup.personal,
             RepoGroup.updated_on,
             RepoGroup._changeset_cache,
             User,
             ) \
             .filter(RepoGroup.group_parent_id == repo_group_id) \
             .filter(or_(
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(RepoGroup.group_id, allowed_ids))
             ) \
             .join(User, User.user_id == RepoGroup.user_id) \
             .group_by(RepoGroup, User)
 
         repo_groups_data_total_filtered_count = base_q.count()
 
         sort_defined = False
 
         if order_by == 'group_name':
             sort_col = func.lower(RepoGroup.group_name)
             sort_defined = True
         elif order_by == 'user_username':
             sort_col = User.username
         else:
             sort_col = getattr(RepoGroup, order_by, None)
 
         if sort_defined or sort_col:
             if order_dir == 'asc':
                 sort_col = sort_col.asc()
             else:
                 sort_col = sort_col.desc()
 
         base_q = base_q.order_by(sort_col)
         base_q = base_q.offset(start).limit(limit)
 
         repo_group_list = base_q.all()
 
         repo_groups_data = RepoGroupModel().get_repo_groups_as_dict(
             repo_group_list=repo_group_list, admin=False)
 
         data = ({
             'draw': draw,
             'data': repo_groups_data,
             'recordsTotal': repo_groups_data_total_count,
             'recordsFiltered': repo_groups_data_total_filtered_count,
         })
         return data
 
     def _get_defaults(self, repo_group_name):
         repo_group = RepoGroup.get_by_group_name(repo_group_name)
 
         if repo_group is None:
             return None
 
         defaults = repo_group.get_dict()
         defaults['repo_group_name'] = repo_group.name
         defaults['repo_group_description'] = repo_group.group_description
         defaults['repo_group_enable_locking'] = repo_group.enable_locking
 
         # we use -1 as this is how in HTML, we mark an empty group
         defaults['repo_group'] = defaults['group_parent_id'] or -1
 
         # fill owner
         if repo_group.user:
             defaults.update({'user': repo_group.user.username})
         else:
             replacement_user = User.get_first_super_admin().username
             defaults.update({'user': replacement_user})
 
         return defaults

rhodecode/templates/admin/repo_groups/repo_groups.mako

@@ -1,117 +1,117 b''
 ## -*- coding: utf-8 -*-
 <%inherit file="/base/base.mako"/>
 
 <%def name="title()">
     ${_('Repository groups administration')}
     %if c.rhodecode_name:
         &middot; ${h.branding(c.rhodecode_name)}
     %endif
 </%def>
 
 <%def name="breadcrumbs_links()"></%def>
 
 <%def name="menu_bar_nav()">
     ${self.menu_items(active='admin')}
 </%def>
 
 <%def name="menu_bar_subnav()">
     ${self.admin_menu(active='repository_groups')}
 </%def>
 
 <%def name="main()">
 <div class="box">
 
     <div class="title">
         <input class="q_filter_box" id="q_filter" size="15" type="text" name="filter" placeholder="${_('quick filter...')}" value=""/>
         <span id="repo_group_count"></span>
 
         <ul class="links">
         %if c.can_create_repo_group:
             <li>
               <a href="${h.route_path('repo_group_new')}" class="btn btn-small btn-success">${_(u'Add Repository Group')}</a>
             </li>
         %endif
         </ul>
     </div>
     <div id="repos_list_wrap">
         <table id="group_list_table" class="display"></table>
     </div>
 </div>
 
 <script>
 $(document).ready(function() {
     var $repoGroupsListTable = $('#group_list_table');
 
    // repo group list
    $repoGroupsListTable.DataTable({
       processing: true,
       serverSide: true,
       ajax: {
           "url": "${h.route_path('repo_groups_data')}",
           "dataSrc": function (json) {
               var filteredCount = json.recordsFiltered;
               var filteredInactiveCount = json.recordsFilteredInactive;
               var totalInactive = json.recordsTotalInactive;
               var total = json.recordsTotal;
 
               var _text = _gettext(
                       "{0} of {1} repository groups").format(
                       filteredCount, total);
 
               if (total === filteredCount) {
                   _text = _gettext("{0} repository groups").format(total);
               }
               $('#repo_group_count').text(_text);
               return json.data;
           },
       },
 
       dom: 'rtp',
       pageLength: ${c.visual.admin_grid_items},
       order: [[ 0, "asc" ]],
       columns: [
          { data: {"_": "name",
-                  "sort": "name_raw"}, title: "${_('Name')}", className: "td-componentname" },
+                  "sort": "name"}, title: "${_('Name')}", className: "td-componentname" },
          { data: 'menu', "bSortable": false, className: "quick_repo_menu" },
          { data: {"_": "desc",
                   "sort": "desc"}, title: "${_('Description')}", className: "td-description" },
          { data: {"_": "last_change",
-                  "sort": "last_change_raw",
+                  "sort": "last_change",
                   "type": Number}, title: "${_('Last Change')}", className: "td-time" },
          { data: {"_": "top_level_repos",
                   "sort": "top_level_repos"}, title: "${_('Number of top level repositories')}" },
          { data: {"_": "owner",
                   "sort": "owner"}, title: "${_('Owner')}", className: "td-user" },
          { data: {"_": "action",
                   "sort": "action"}, title: "${_('Action')}", className: "td-action", orderable: false }
       ],
       language: {
           paginate: DEFAULT_GRID_PAGINATION,
           sProcessing: _gettext('loading...'),
           emptyTable: _gettext("No repository groups available yet.")
       },
     });
 
     $repoGroupsListTable.on('xhr.dt', function(e, settings, json, xhr){
         $repoGroupsListTable.css('opacity', 1);
     });
 
     $repoGroupsListTable.on('preXhr.dt', function(e, settings, data){
         $repoGroupsListTable.css('opacity', 0.3);
     });
 
     // filter
     $('#q_filter').on('keyup',
         $.debounce(250, function() {
             $repoGroupsListTable.DataTable().search(
                 $('#q_filter').val()
             ).draw();
         })
     );
 
 });
 
 </script>
 
 </%def>
 

rhodecode/templates/admin/repos/repos.mako

@@ -1,149 +1,149 b''
 ## -*- coding: utf-8 -*-
 <%inherit file="/base/base.mako"/>
 
 <%def name="title()">
     ${_('Repositories administration')}
     %if c.rhodecode_name:
         &middot; ${h.branding(c.rhodecode_name)}
     %endif
 </%def>
 
 <%def name="breadcrumbs_links()"></%def>
 
 <%def name="menu_bar_nav()">
     ${self.menu_items(active='admin')}
 </%def>
 
 <%def name="menu_bar_subnav()">
     ${self.admin_menu(active='repositories')}
 </%def>
 
 <%def name="main()">
 <div class="box">
 
     <div class="title">
         <input class="q_filter_box" id="q_filter" size="15" type="text" name="filter" placeholder="${_('quick filter...')}" value=""/>
         <span id="repo_count"></span>
 
         <ul class="links">
         %if c.can_create_repo:
             <li>
               <a href="${h.route_path('repo_new')}" class="btn btn-small btn-success">${_(u'Add Repository')}</a>
             </li>
         %endif
         </ul>
     </div>
     <div id="repos_list_wrap">
         <table id="repo_list_table" class="display"></table>
     </div>
 
 </div>
 
 <script>
 $(document).ready(function() {
     var $repoListTable = $('#repo_list_table');
 
     // repo list
     $repoListTable.DataTable({
         processing: true,
         serverSide: true,
         ajax: {
             "url": "${h.route_path('repos_data')}",
             "dataSrc": function (json) {
                 var filteredCount = json.recordsFiltered;
                 var total = json.recordsTotal;
 
                 var _text = _gettext(
                         "{0} of {1} repositories").format(
                         filteredCount, total);
 
                 if (total === filteredCount) {
                     _text = _gettext("{0} repositories").format(total);
                 }
                 $('#repo_count').text(_text);
 
                 return json.data;
             },
         },
         dom: 'rtp',
         pageLength: ${c.visual.admin_grid_items},
         order: [[ 0, "asc" ]],
         columns: [
             {
                 data: {
                     "_": "name",
-                    "sort": "name_raw"
+                    "sort": "name"
                 }, title: "${_('Name')}", className: "td-componentname"
             },
             {
                 data: 'menu', "bSortable": false, className: "quick_repo_menu"},
             {
                 data: {
                     "_": "desc",
                     "sort": "desc"
                 }, title: "${_('Description')}", className: "td-description"
             },
             {
                 data: {
                     "_": "last_change",
-                    "sort": "last_change_raw",
+                    "sort": "last_change",
                     "type": Number
                 }, title: "${_('Last Change')}", className: "td-time"
             },
             {
                 data: {
                     "_": "last_changeset",
                     "sort": "last_changeset_raw",
                     "type": Number
                 }, title: "${_('Commit')}", className: "td-commit", orderable: false
             },
             {
                 data: {
                     "_": "owner",
                     "sort": "owner"
                 }, title: "${_('Owner')}", className: "td-user"
             },
             {
                 data: {
                     "_": "state",
                     "sort": "state"
                 }, title: "${_('State')}", className: "td-tags td-state"
             },
             {
                 data: {
                     "_": "action",
                     "sort": "action"
                 }, title: "${_('Action')}", className: "td-action", orderable: false
             }
         ],
         language: {
           paginate: DEFAULT_GRID_PAGINATION,
           sProcessing: _gettext('loading...'),
           emptyTable:_gettext("No repositories present.")
         },
         "initComplete": function( settings, json ) {
             quick_repo_menu();
         }
     });
 
     $repoListTable.on('xhr.dt', function(e, settings, json, xhr){
         $repoListTable.css('opacity', 1);
     });
 
     $repoListTable.on('preXhr.dt', function(e, settings, data){
         $repoListTable.css('opacity', 0.3);
     });
 
     $('#q_filter').on('keyup',
         $.debounce(250, function() {
             $repoListTable.DataTable().search(
                 $('#q_filter').val()
             ).draw();
         })
     );
 
   });
 
 </script>
 
 </%def>

rhodecode/templates/index_base.mako

@@ -1,221 +1,223 b''
 <%inherit file="/base/base.mako"/>
 
 
 <%def name="menu_bar_subnav()">
   % if c.repo_group:
     ${self.repo_group_menu(active='home')}
   % endif
 </%def>
 
 
 <%def name="main()">
    <div class="box">
         <!-- box / title -->
         <div class="title">
 
         </div>
         <!-- end box / title -->
         <div id="no_grid_data" class="table" style="display: none">
             <h2 class="no-object-border">
                  ${_('No repositories or repositories groups exists here.')}
             </h2>
         </div>
 
         <div class="table">
             <div id="groups_list_wrap" style="min-height: 200px;">
                 <table id="group_list_table" class="display" style="width: 100%;"></table>
             </div>
         </div>
 
         <div class="table">
             <div id="repos_list_wrap" style="min-height: 200px;">
                 <table id="repo_list_table" class="display" style="width: 100%;"></table>
             </div>
         </div>
 
     </div>
     <script>
     $(document).ready(function () {
 
         // repo group list
         var $groupListTable = $('#group_list_table');
 
         $groupListTable.DataTable({
             processing: true,
             serverSide: true,
             ajax: {
                 "url": "${h.route_path('main_page_repo_groups_data')}",
                 "data": function (d) {
                     % if c.repo_group:
                         d.repo_group_id = ${c.repo_group.group_id}
                     % endif
                 }
             },
             dom: 'rtp',
             pageLength: ${c.visual.dashboard_items},
             order: [[0, "asc"]],
             columns: [
                 {
                     data: {
                         "_": "name",
-                        "sort": "name_raw"
+                        "sort": "name"
                     }, title: "${_('Name')}", className: "truncate-wrap td-grid-name"
                 },
-                {data: 'menu', "bSortable": false, className: "quick_repo_menu"},
+                {
+                    data: 'menu', "bSortable": false, className: "quick_repo_menu"
+                },
                 {
                     data: {
                         "_": "desc",
                         "sort": "desc"
                     }, title: "${_('Description')}", className: "td-description"
                 },
                 {
                     data: {
                         "_": "last_change",
-                        "sort": "last_change_raw",
+                        "sort": "last_change",
                         "type": Number
                     }, title: "${_('Last Change')}", className: "td-time"
                 },
                 {
                     data: {
                         "_": "last_changeset",
                         "sort": "last_changeset_raw",
                         "type": Number
-                    }, title: "", className: "td-hash"
+                    }, title: "", className: "td-hash", orderable: false
                 },
                 {
                     data: {
                         "_": "owner",
                         "sort": "owner"
                     }, title: "${_('Owner')}", className: "td-user"
                 }
             ],
             language: {
                 paginate: DEFAULT_GRID_PAGINATION,
                 sProcessing: _gettext('loading...'),
                 emptyTable: _gettext("No repository groups present.")
             },
             "drawCallback": function (settings, json) {
                 // hide grid if it's empty
                 if (settings.fnRecordsDisplay() === 0) {
                     $('#groups_list_wrap').hide();
                     // both hidden, show no-data
                     if ($('#repos_list_wrap').is(':hidden')) {
                         $('#no_grid_data').show();
                     }
                 } else {
                     $('#groups_list_wrap').show();
                 }
 
                 timeagoActivate();
                 tooltipActivate();
                 quick_repo_menu();
                 // hide pagination for single page
                 if (settings._iDisplayLength >= settings.fnRecordsDisplay()) {
                     $(settings.nTableWrapper).find('.dataTables_paginate').hide();
                 }
 
             },
         });
 
         $groupListTable.on('xhr.dt', function (e, settings, json, xhr) {
             $groupListTable.css('opacity', 1);
         });
 
         $groupListTable.on('preXhr.dt', function (e, settings, data) {
             $groupListTable.css('opacity', 0.3);
         });
 
 
         ##  // repo list
         var $repoListTable = $('#repo_list_table');
 
         $repoListTable.DataTable({
             processing: true,
             serverSide: true,
             ajax: {
                 "url": "${h.route_path('main_page_repos_data')}",
                 "data": function (d) {
                     % if c.repo_group:
                         d.repo_group_id = ${c.repo_group.group_id}
                     % endif
                 }
             },
             order: [[0, "asc"]],
             dom: 'rtp',
             pageLength: ${c.visual.dashboard_items},
             columns: [
                 {
                     data: {
                         "_": "name",
-                        "sort": "name_raw"
+                        "sort": "name"
                     }, title: "${_('Name')}", className: "truncate-wrap td-grid-name"
                 },
                 {
                     data: 'menu', "bSortable": false, className: "quick_repo_menu"
                 },
                 {
                     data: {
                         "_": "desc",
                         "sort": "desc"
                     }, title: "${_('Description')}", className: "td-description"
                 },
                 {
                     data: {
                         "_": "last_change",
-                        "sort": "last_change_raw",
+                        "sort": "last_change",
                         "type": Number
-                    }, title: "${_('Last Change')}", className: "td-time", orderable: false
+                    }, title: "${_('Last Change')}", className: "td-time"
                 },
                 {
                     data: {
                         "_": "last_changeset",
                         "sort": "last_changeset_raw",
                         "type": Number
-                    }, title: "${_('Commit')}", className: "td-hash"
+                    }, title: "${_('Commit')}", className: "td-hash", orderable: false
                 },
                  {
                     data: {
                         "_": "owner",
                         "sort": "owner"
                     }, title: "${_('Owner')}", className: "td-user"
                 }
             ],
             language: {
                 paginate: DEFAULT_GRID_PAGINATION,
                 sProcessing: _gettext('loading...'),
                 emptyTable: _gettext("No repositories present.")
             },
             "drawCallback": function (settings, json) {
                 // hide grid if it's empty
                 if (settings.fnRecordsDisplay() == 0) {
                     $('#repos_list_wrap').hide()
                     // both hidden, show no-data
                     if ($('#groups_list_wrap').is(':hidden')) {
                         $('#no_grid_data').show()
                     }
                 } else {
                     $('#repos_list_wrap').show()
                 }
 
                 timeagoActivate();
                 tooltipActivate();
                 quick_repo_menu();
                 // hide pagination for single page
                 if (settings._iDisplayLength >= settings.fnRecordsDisplay()) {
                     $(settings.nTableWrapper).find('.dataTables_paginate').hide();
                 }
 
             },
         });
 
         $repoListTable.on('xhr.dt', function (e, settings, json, xhr) {
             $repoListTable.css('opacity', 1);
         });
 
         $repoListTable.on('preXhr.dt', function (e, settings, data) {
             $repoListTable.css('opacity', 0.3);
         });
 
     });
     </script>
 </%def>