cleanup: get rid of redundant assignments 'foo = foo'

Redundant assignments were searched with:

    grep '^ *\(.*\) = \1$' `hg files`

Some of these cases have resulted from earlier cleanup, like:

    foo = safe_unicode(foo)

and later batch removing safe_unicode usage. See e.g. commit
e35373106528b29461b7f5144780cb9ec79f765d .


Note: two false positives occur:

kallithea/tests/models/test_settings.py:39:
        setting.app_settings_value = setting.app_settings_value
kallithea/tests/models/test_settings.py:42:
        setting.app_settings_value = setting.app_settings_value

which is not a plain assignment but where 'app_settings_value' is a property
method with side effects.

kallithea/controllers/api/api.py

@@ -1,2443 +1,2442 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.controllers.api.api
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 API controller for Kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Aug 20, 2011
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 import logging
 import traceback
 from datetime import datetime
 
 from tg import request
 
 from kallithea.controllers.api import JSONRPCController, JSONRPCError
 from kallithea.lib.auth import (AuthUser, HasPermissionAny, HasPermissionAnyDecorator, HasRepoGroupPermissionLevel, HasRepoPermissionLevel,
                                 HasUserGroupPermissionLevel)
 from kallithea.lib.exceptions import DefaultUserException, UserGroupsAssignedException
 from kallithea.lib.utils import repo2db_mapper
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.lib.vcs.exceptions import EmptyRepositoryError
 from kallithea.model import db, meta, userlog
 from kallithea.model.changeset_status import ChangesetStatusModel
 from kallithea.model.comment import ChangesetCommentsModel
 from kallithea.model.gist import GistModel
 from kallithea.model.pull_request import PullRequestModel
 from kallithea.model.repo import RepoModel
 from kallithea.model.repo_group import RepoGroupModel
 from kallithea.model.scm import ScmModel, UserGroupList
 from kallithea.model.user import UserModel
 from kallithea.model.user_group import UserGroupModel
 
 
 log = logging.getLogger(__name__)
 
 
 def store_update(updates, attr, name):
     """
     Stores param in updates dict if it's not None (i.e. if user explicitly set
     a parameter). This allows easy updates of passed in params.
     """
     if attr is not None:
         updates[name] = attr
 
 
 def get_user_or_error(userid):
     """
     Get user by id or name or return JsonRPCError if not found
 
     :param userid:
     """
     user = UserModel().get_user(userid)
     if user is None:
         raise JSONRPCError("user `%s` does not exist" % (userid,))
     return user
 
 
 def get_repo_or_error(repoid):
     """
     Get repo by id or name or return JsonRPCError if not found
 
     :param repoid:
     """
     repo = RepoModel().get_repo(repoid)
     if repo is None:
         raise JSONRPCError('repository `%s` does not exist' % (repoid,))
     return repo
 
 
 def get_repo_group_or_error(repogroupid):
     """
     Get repo group by id or name or return JsonRPCError if not found
 
     :param repogroupid:
     """
     repo_group = db.RepoGroup.guess_instance(repogroupid)
     if repo_group is None:
         raise JSONRPCError(
             'repository group `%s` does not exist' % (repogroupid,))
     return repo_group
 
 
 def get_user_group_or_error(usergroupid):
     """
     Get user group by id or name or return JsonRPCError if not found
 
     :param usergroupid:
     """
     user_group = UserGroupModel().get_group(usergroupid)
     if user_group is None:
         raise JSONRPCError('user group `%s` does not exist' % (usergroupid,))
     return user_group
 
 
 def get_perm_or_error(permid, prefix=None):
     """
     Get permission by id or name or return JsonRPCError if not found
 
     :param permid:
     """
     perm = db.Permission.get_by_key(permid)
     if perm is None:
         raise JSONRPCError('permission `%s` does not exist' % (permid,))
     if prefix:
         if not perm.permission_name.startswith(prefix):
             raise JSONRPCError('permission `%s` is invalid, '
                                'should start with %s' % (permid, prefix))
     return perm
 
 
 def get_gist_or_error(gistid):
     """
     Get gist by id or gist_access_id or return JsonRPCError if not found
 
     :param gistid:
     """
     gist = GistModel().get_gist(gistid)
     if gist is None:
         raise JSONRPCError('gist `%s` does not exist' % (gistid,))
     return gist
 
 
 class ApiController(JSONRPCController):
     """
     API Controller
 
     The authenticated user can be found as request.authuser.
 
     Example function::
 
         def func(arg1, arg2,...):
             pass
 
     Each function should also **raise** JSONRPCError for any
     errors that happens.
     """
 
     @HasPermissionAnyDecorator('hg.admin')
     def test(self, args):
         return args
 
     @HasPermissionAnyDecorator('hg.admin')
     def pull(self, repoid, clone_uri=None):
         """
         Triggers a pull from remote location on given repo. Can be used to
         automatically keep remote repos up to date. This command can be executed
         only using api_key belonging to user with admin rights
 
         :param repoid: repository name or repository id
         :type repoid: str or int
         :param clone_uri: repository URI to pull from (optional)
         :type clone_uri: str
 
         OUTPUT::
 
           id : <id_given_in_input>
           result : {
             "msg": "Pulled from `<repository name>`"
             "repository": "<repository name>"
           }
           error :  null
 
         ERROR OUTPUT::
 
           id : <id_given_in_input>
           result : null
           error :  {
             "Unable to pull changes from `<reponame>`"
           }
 
         """
 
         repo = get_repo_or_error(repoid)
 
         try:
             ScmModel().pull_changes(repo.repo_name,
                                     request.authuser.username,
                                     request.ip_addr,
                                     clone_uri=clone_uri)
             return dict(
                 msg='Pulled from `%s`' % repo.repo_name,
                 repository=repo.repo_name
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'Unable to pull changes from `%s`' % repo.repo_name
             )
 
     @HasPermissionAnyDecorator('hg.admin')
     def rescan_repos(self, remove_obsolete=False):
         """
         Triggers rescan repositories action. If remove_obsolete is set
         than also delete repos that are in database but not in the filesystem.
         aka "clean zombies". This command can be executed only using api_key
         belonging to user with admin rights.
 
         :param remove_obsolete: deletes repositories from
             database that are not found on the filesystem
         :type remove_obsolete: Optional(bool)
 
         OUTPUT::
 
           id : <id_given_in_input>
           result : {
             'added': [<added repository name>,...]
             'removed': [<removed repository name>,...]
           }
           error :  null
 
         ERROR OUTPUT::
 
           id : <id_given_in_input>
           result : null
           error :  {
             'Error occurred during rescan repositories action'
           }
 
         """
 
         try:
             rm_obsolete = remove_obsolete
             added, removed = repo2db_mapper(ScmModel().repo_scan(),
                                             remove_obsolete=rm_obsolete)
             return {'added': added, 'removed': removed}
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'Error occurred during rescan repositories action'
             )
 
     def invalidate_cache(self, repoid):
         """
         Invalidate cache for repository.
         This command can be executed only using api_key belonging to user with admin
         rights or regular user that have write or admin or write access to repository.
 
         :param repoid: repository name or repository id
         :type repoid: str or int
 
         OUTPUT::
 
           id : <id_given_in_input>
           result : {
             'msg': Cache for repository `<repository name>` was invalidated,
             'repository': <repository name>
           }
           error :  null
 
         ERROR OUTPUT::
 
           id : <id_given_in_input>
           result : null
           error :  {
             'Error occurred during cache invalidation action'
           }
 
         """
         repo = get_repo_or_error(repoid)
         if not HasPermissionAny('hg.admin')():
             if not HasRepoPermissionLevel('write')(repo.repo_name):
                 raise JSONRPCError('repository `%s` does not exist' % (repoid,))
 
         try:
             ScmModel().mark_for_invalidation(repo.repo_name)
             return dict(
                 msg='Cache for repository `%s` was invalidated' % (repoid,),
                 repository=repo.repo_name
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'Error occurred during cache invalidation action'
             )
 
     @HasPermissionAnyDecorator('hg.admin')
     def get_ip(self, userid=None):
         """
         Shows IP address as seen from Kallithea server, together with all
         defined IP addresses for given user. If userid is not passed data is
         returned for user who's calling this function.
         This command can be executed only using api_key belonging to user with
         admin rights.
 
         :param userid: username to show ips for
         :type userid: Optional(str or int)
 
         OUTPUT::
 
             id : <id_given_in_input>
             result : {
                          "server_ip_addr": "<ip_from_clien>",
                          "user_ips": [
                                         {
                                            "ip_addr": "<ip_with_mask>",
                                            "ip_range": ["<start_ip>", "<end_ip>"],
                                         },
                                         ...
                                      ]
             }
 
         """
         if userid is None:
             userid = request.authuser.user_id
         user = get_user_or_error(userid)
         ips = db.UserIpMap.query().filter(db.UserIpMap.user == user).all()
         return dict(
             server_ip_addr=request.ip_addr,
             user_ips=ips
         )
 
     # alias for old
     show_ip = get_ip
 
     @HasPermissionAnyDecorator('hg.admin')
     def get_server_info(self):
         """
         return server info, including Kallithea version and installed packages
 
 
         OUTPUT::
 
           id : <id_given_in_input>
           result : {
             'modules': [<module name>,...]
             'py_version': <python version>,
             'platform': <platform type>,
             'kallithea_version': <kallithea version>
           }
           error :  null
         """
         return db.Setting.get_server_info()
 
     def get_user(self, userid=None):
         """
         Gets a user by username or user_id, Returns empty result if user is
         not found. If userid param is skipped it is set to id of user who is
         calling this method. This command can be executed only using api_key
         belonging to user with admin rights, or regular users that cannot
         specify different userid than theirs
 
         :param userid: user to get data for
         :type userid: Optional(str or int)
 
         OUTPUT::
 
             id : <id_given_in_input>
             result: None if user does not exist or
                     {
                         "user_id" :     "<user_id>",
                         "api_key" :     "<api_key>",
                         "api_keys":     "[<list of all API keys including additional ones>]"
                         "username" :    "<username>",
                         "firstname":    "<firstname>",
                         "lastname" :    "<lastname>",
                         "email" :       "<email>",
                         "emails":       "[<list of all emails including additional ones>]",
                         "ip_addresses": "[<ip_address_for_user>,...]",
                         "active" :      "<bool: user active>",
                         "admin" :       "<bool: user is admin>",
                         "extern_name" : "<extern_name>",
                         "extern_type" : "<extern type>
                         "last_login":   "<last_login>",
                         "permissions": {
                             "global": ["hg.create.repository",
                                        "repository.read",
                                        "hg.register.manual_activate"],
                             "repositories": {"repo1": "repository.none"},
                             "repositories_groups": {"Group1": "group.read"}
                          },
                     }
 
             error:  null
 
         """
         if not HasPermissionAny('hg.admin')():
             # make sure normal user does not pass someone else userid,
             # he is not allowed to do that
             if userid is not None and userid != request.authuser.user_id:
                 raise JSONRPCError(
                     'userid is not the same as your user'
                 )
 
         if userid is None:
             userid = request.authuser.user_id
 
         user = get_user_or_error(userid)
         data = user.get_api_data()
         data['permissions'] = AuthUser(user_id=user.user_id).permissions
         return data
 
     @HasPermissionAnyDecorator('hg.admin')
     def get_users(self):
         """
         Lists all existing users. This command can be executed only using api_key
         belonging to user with admin rights.
 
 
         OUTPUT::
 
             id : <id_given_in_input>
             result: [<user_object>, ...]
             error:  null
         """
 
         return [
             user.get_api_data()
             for user in db.User.query()
                 .order_by(db.User.username)
                 .filter_by(is_default_user=False)
         ]
 
     @HasPermissionAnyDecorator('hg.admin')
     def create_user(self, username, email, password='',
                     firstname='', lastname='',
                     active=True, admin=False,
                     extern_type=db.User.DEFAULT_AUTH_TYPE,
                     extern_name=''):
         """
         Creates new user. Returns new user object. This command can
         be executed only using api_key belonging to user with admin rights.
 
         :param username: new username
         :type username: str or int
         :param email: email
         :type email: str
         :param password: password
         :type password: Optional(str)
         :param firstname: firstname
         :type firstname: Optional(str)
         :param lastname: lastname
         :type lastname: Optional(str)
         :param active: active
         :type active: Optional(bool)
         :param admin: admin
         :type admin: Optional(bool)
         :param extern_name: name of extern
         :type extern_name: Optional(str)
         :param extern_type: extern_type
         :type extern_type: Optional(str)
 
 
         OUTPUT::
 
             id : <id_given_in_input>
             result: {
                       "msg" : "created new user `<username>`",
                       "user": <user_obj>
                     }
             error:  null
 
         ERROR OUTPUT::
 
           id : <id_given_in_input>
           result : null
           error :  {
             "user `<username>` already exist"
             or
             "email `<email>` already exist"
             or
             "failed to create user `<username>`"
           }
 
         """
 
         if db.User.get_by_username(username):
             raise JSONRPCError("user `%s` already exist" % (username,))
 
         if db.User.get_by_email(email):
             raise JSONRPCError("email `%s` already exist" % (email,))
 
         try:
             user = UserModel().create_or_update(
                 username=username,
                 password=password,
                 email=email,
                 firstname=firstname,
                 lastname=lastname,
                 active=active,
                 admin=admin,
                 extern_type=extern_type,
                 extern_name=extern_name
             )
             meta.Session().commit()
             return dict(
                 msg='created new user `%s`' % username,
                 user=user.get_api_data()
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError('failed to create user `%s`' % (username,))
 
     @HasPermissionAnyDecorator('hg.admin')
     def update_user(self, userid, username=None,
                     email=None, password=None,
                     firstname=None, lastname=None,
                     active=None, admin=None,
                     extern_type=None, extern_name=None):
         """
         updates given user if such user exists. This command can
         be executed only using api_key belonging to user with admin rights.
 
         :param userid: userid to update
         :type userid: str or int
         :param username: new username
         :type username: str or int
         :param email: email
         :type email: str
         :param password: password
         :type password: Optional(str)
         :param firstname: firstname
         :type firstname: Optional(str)
         :param lastname: lastname
         :type lastname: Optional(str)
         :param active: active
         :type active: Optional(bool)
         :param admin: admin
         :type admin: Optional(bool)
         :param extern_name:
         :type extern_name: Optional(str)
         :param extern_type:
         :type extern_type: Optional(str)
 
 
         OUTPUT::
 
             id : <id_given_in_input>
             result: {
                       "msg" : "updated user ID:<userid> <username>",
                       "user": <user_object>,
                     }
             error:  null
 
         ERROR OUTPUT::
 
           id : <id_given_in_input>
           result : null
           error :  {
             "failed to update user `<username>`"
           }
 
         """
 
         user = get_user_or_error(userid)
 
         # only non optional arguments will be stored in updates
         updates = {}
 
         try:
 
             store_update(updates, username, 'username')
             store_update(updates, password, 'password')
             store_update(updates, email, 'email')
             store_update(updates, firstname, 'name')
             store_update(updates, lastname, 'lastname')
             store_update(updates, active, 'active')
             store_update(updates, admin, 'admin')
             store_update(updates, extern_name, 'extern_name')
             store_update(updates, extern_type, 'extern_type')
 
             user = UserModel().update_user(user, **updates)
             meta.Session().commit()
             return dict(
                 msg='updated user ID:%s %s' % (user.user_id, user.username),
                 user=user.get_api_data()
             )
         except DefaultUserException:
             log.error(traceback.format_exc())
             raise JSONRPCError('editing default user is forbidden')
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError('failed to update user `%s`' % (userid,))
 
     @HasPermissionAnyDecorator('hg.admin')
     def delete_user(self, userid):
         """
         deletes given user if such user exists. This command can
         be executed only using api_key belonging to user with admin rights.
 
         :param userid: user to delete
         :type userid: str or int
 
         OUTPUT::
 
             id : <id_given_in_input>
             result: {
                       "msg" : "deleted user ID:<userid> <username>",
                       "user": null
                     }
             error:  null
 
         ERROR OUTPUT::
 
           id : <id_given_in_input>
           result : null
           error :  {
             "failed to delete user ID:<userid> <username>"
           }
 
         """
         user = get_user_or_error(userid)
 
         try:
             UserModel().delete(userid)
             meta.Session().commit()
             return dict(
                 msg='deleted user ID:%s %s' % (user.user_id, user.username),
                 user=None
             )
         except Exception:
 
             log.error(traceback.format_exc())
             raise JSONRPCError('failed to delete user ID:%s %s'
                                % (user.user_id, user.username))
 
     # permission check inside
     def get_user_group(self, usergroupid):
         """
         Gets an existing user group. This command can be executed only using api_key
         belonging to user with admin rights or user who has at least
         read access to user group.
 
         :param usergroupid: id of user_group to edit
         :type usergroupid: str or int
 
         OUTPUT::
 
             id : <id_given_in_input>
             result : None if group not exist
                      {
                        "users_group_id" : "<id>",
                        "group_name" :     "<groupname>",
                        "active":          "<bool>",
                        "members" :  [<user_obj>,...]
                      }
             error : null
 
         """
         user_group = get_user_group_or_error(usergroupid)
         if not HasPermissionAny('hg.admin')():
             if not HasUserGroupPermissionLevel('read')(user_group.users_group_name):
                 raise JSONRPCError('user group `%s` does not exist' % (usergroupid,))
 
         data = user_group.get_api_data()
         return data
 
     # permission check inside
     def get_user_groups(self):
         """
         Lists all existing user groups. This command can be executed only using
         api_key belonging to user with admin rights or user who has at least
         read access to user group.
 
 
         OUTPUT::
 
             id : <id_given_in_input>
             result : [<user_group_obj>,...]
             error : null
         """
 
         return [
             user_group.get_api_data()
             for user_group in UserGroupList(db.UserGroup.query().all(), perm_level='read')
         ]
 
     @HasPermissionAnyDecorator('hg.admin', 'hg.usergroup.create.true')
     def create_user_group(self, group_name, description='',
                           owner=None, active=True):
         """
         Creates new user group. This command can be executed only using api_key
         belonging to user with admin rights or an user who has create user group
         permission
 
         :param group_name: name of new user group
         :type group_name: str
         :param description: group description
         :type description: str
         :param owner: owner of group. If not passed apiuser is the owner
         :type owner: Optional(str or int)
         :param active: group is active
         :type active: Optional(bool)
 
         OUTPUT::
 
             id : <id_given_in_input>
             result: {
                       "msg": "created new user group `<groupname>`",
                       "user_group": <user_group_object>
                     }
             error:  null
 
         ERROR OUTPUT::
 
           id : <id_given_in_input>
           result : null
           error :  {
             "user group `<group name>` already exist"
             or
             "failed to create group `<group name>`"
           }
 
         """
 
         if UserGroupModel().get_by_name(group_name):
             raise JSONRPCError("user group `%s` already exist" % (group_name,))
 
         try:
             if owner is None:
                 owner = request.authuser.user_id
 
             owner = get_user_or_error(owner)
             ug = UserGroupModel().create(name=group_name, description=description,
                                          owner=owner, active=active)
             meta.Session().commit()
             return dict(
                 msg='created new user group `%s`' % group_name,
                 user_group=ug.get_api_data()
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError('failed to create group `%s`' % (group_name,))
 
     # permission check inside
     def update_user_group(self, usergroupid, group_name=None,
                           description=None, owner=None,
                           active=None):
         """
         Updates given usergroup.  This command can be executed only using api_key
         belonging to user with admin rights or an admin of given user group
 
         :param usergroupid: id of user group to update
         :type usergroupid: str or int
         :param group_name: name of new user group
         :type group_name: str
         :param description: group description
         :type description: str
         :param owner: owner of group.
         :type owner: Optional(str or int)
         :param active: group is active
         :type active: Optional(bool)
 
         OUTPUT::
 
           id : <id_given_in_input>
           result : {
             "msg": 'updated user group ID:<user group id> <user group name>',
             "user_group": <user_group_object>
           }
           error :  null
 
         ERROR OUTPUT::
 
           id : <id_given_in_input>
           result : null
           error :  {
             "failed to update user group `<user group name>`"
           }
 
         """
         user_group = get_user_group_or_error(usergroupid)
         if not HasPermissionAny('hg.admin')():
             if not HasUserGroupPermissionLevel('admin')(user_group.users_group_name):
                 raise JSONRPCError('user group `%s` does not exist' % (usergroupid,))
 
         if owner is not None:
             owner = get_user_or_error(owner)
 
         updates = {}
         store_update(updates, group_name, 'users_group_name')
         store_update(updates, description, 'user_group_description')
         store_update(updates, owner, 'owner')
         store_update(updates, active, 'users_group_active')
         try:
             UserGroupModel().update(user_group, updates)
             meta.Session().commit()
             return dict(
                 msg='updated user group ID:%s %s' % (user_group.users_group_id,
                                                      user_group.users_group_name),
                 user_group=user_group.get_api_data()
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError('failed to update user group `%s`' % (usergroupid,))
 
     # permission check inside
     def delete_user_group(self, usergroupid):
         """
         Delete given user group by user group id or name.
         This command can be executed only using api_key
         belonging to user with admin rights or an admin of given user group
 
         :param usergroupid:
         :type usergroupid: int
 
         OUTPUT::
 
           id : <id_given_in_input>
           result : {
             "msg": "deleted user group ID:<user_group_id> <user_group_name>"
           }
           error :  null
 
         ERROR OUTPUT::
 
           id : <id_given_in_input>
           result : null
           error :  {
             "failed to delete user group ID:<user_group_id> <user_group_name>"
             or
             "RepoGroup assigned to <repo_groups_list>"
           }
 
         """
         user_group = get_user_group_or_error(usergroupid)
         if not HasPermissionAny('hg.admin')():
             if not HasUserGroupPermissionLevel('admin')(user_group.users_group_name):
                 raise JSONRPCError('user group `%s` does not exist' % (usergroupid,))
 
         try:
             UserGroupModel().delete(user_group)
             meta.Session().commit()
             return dict(
                 msg='deleted user group ID:%s %s' %
                     (user_group.users_group_id, user_group.users_group_name),
                 user_group=None
             )
         except UserGroupsAssignedException as e:
             log.error(traceback.format_exc())
             raise JSONRPCError(str(e))
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError('failed to delete user group ID:%s %s' %
                                (user_group.users_group_id,
                                 user_group.users_group_name)
                                )
 
     # permission check inside
     def add_user_to_user_group(self, usergroupid, userid):
         """
         Adds a user to a user group. If user exists in that group success will be
         `false`. This command can be executed only using api_key
         belonging to user with admin rights  or an admin of given user group
 
         :param usergroupid:
         :type usergroupid: int
         :param userid:
         :type userid: int
 
         OUTPUT::
 
           id : <id_given_in_input>
           result : {
               "success": True|False # depends on if member is in group
               "msg": "added member `<username>` to user group `<groupname>` |
                       User is already in that group"
 
           }
           error :  null
 
         ERROR OUTPUT::
 
           id : <id_given_in_input>
           result : null
           error :  {
             "failed to add member to user group `<user_group_name>`"
           }
 
         """
         user = get_user_or_error(userid)
         user_group = get_user_group_or_error(usergroupid)
         if not HasPermissionAny('hg.admin')():
             if not HasUserGroupPermissionLevel('admin')(user_group.users_group_name):
                 raise JSONRPCError('user group `%s` does not exist' % (usergroupid,))
 
         try:
             ugm = UserGroupModel().add_user_to_group(user_group, user)
             success = True if ugm is not True else False
             msg = 'added member `%s` to user group `%s`' % (
                 user.username, user_group.users_group_name
             )
             msg = msg if success else 'User is already in that group'
             meta.Session().commit()
 
             return dict(
                 success=success,
                 msg=msg
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to add member to user group `%s`' % (
                     user_group.users_group_name,
                 )
             )
 
     # permission check inside
     def remove_user_from_user_group(self, usergroupid, userid):
         """
         Removes a user from a user group. If user is not in given group success will
         be `false`. This command can be executed only
         using api_key belonging to user with admin rights or an admin of given user group
 
         :param usergroupid:
         :param userid:
 
 
         OUTPUT::
 
             id : <id_given_in_input>
             result: {
                       "success":  True|False,  # depends on if member is in group
                       "msg": "removed member <username> from user group <groupname> |
                               User wasn't in group"
                     }
             error:  null
 
         """
         user = get_user_or_error(userid)
         user_group = get_user_group_or_error(usergroupid)
         if not HasPermissionAny('hg.admin')():
             if not HasUserGroupPermissionLevel('admin')(user_group.users_group_name):
                 raise JSONRPCError('user group `%s` does not exist' % (usergroupid,))
 
         try:
             success = UserGroupModel().remove_user_from_group(user_group, user)
             msg = 'removed member `%s` from user group `%s`' % (
                 user.username, user_group.users_group_name
             )
             msg = msg if success else "User wasn't in group"
             meta.Session().commit()
             return dict(success=success, msg=msg)
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to remove member from user group `%s`' % (
                     user_group.users_group_name,
                 )
             )
 
     # permission check inside
     def get_repo(self, repoid,
                  with_revision_names=False,
                  with_pullrequests=False):
         """
         Gets an existing repository by it's name or repository_id. Members will return
         either users_group or user associated to that repository. This command can be
         executed only using api_key belonging to user with admin
         rights or regular user that have at least read access to repository.
 
         :param repoid: repository name or repository id
         :type repoid: str or int
 
         OUTPUT::
 
           id : <id_given_in_input>
           result : {
             {
                 "repo_id" :          "<repo_id>",
                 "repo_name" :        "<reponame>"
                 "repo_type" :        "<repo_type>",
                 "clone_uri" :        "<clone_uri>",
                 "enable_downloads":  "<bool>",
                 "enable_statistics": "<bool>",
                 "private":           "<bool>",
                 "created_on" :       "<date_time_created>",
                 "description" :      "<description>",
                 "landing_rev":       "<landing_rev>",
                 "last_changeset":    {
                                        "author":   "<full_author>",
                                        "date":     "<date_time_of_commit>",
                                        "message":  "<commit_message>",
                                        "raw_id":   "<raw_id>",
                                        "revision": "<numeric_revision>",
                                        "short_id": "<short_id>"
                                      }
                 "owner":             "<repo_owner>",
                 "fork_of":           "<name_of_fork_parent>",
                 "members" :     [
                                   {
                                     "name":     "<username>",
                                     "type" :    "user",
                                     "permission" : "repository.(read|write|admin)"
                                   },
                                   …
                                   {
                                     "name":     "<usergroup name>",
                                     "type" :    "user_group",
                                     "permission" : "usergroup.(read|write|admin)"
                                   },
                                   …
                                 ]
                  "followers":   [<user_obj>, ...],
                  <if with_revision_names == True>
                  "tags": {
                             "<tagname>": "<raw_id>",
                             ...
                          },
                  "branches": {
                             "<branchname>": "<raw_id>",
                             ...
                          },
                  "bookmarks": {
                             "<bookmarkname>": "<raw_id>",
                             ...
                          },
             }
           }
           error :  null
 
         """
         repo = get_repo_or_error(repoid)
 
         if not HasPermissionAny('hg.admin')():
             if not HasRepoPermissionLevel('read')(repo.repo_name):
                 raise JSONRPCError('repository `%s` does not exist' % (repoid,))
 
         members = []
         for user in repo.repo_to_perm:
             perm = user.permission.permission_name
             user = user.user
             user_data = {
                 'name': user.username,
                 'type': "user",
                 'permission': perm
             }
             members.append(user_data)
 
         for user_group in repo.users_group_to_perm:
             perm = user_group.permission.permission_name
             user_group = user_group.users_group
             user_group_data = {
                 'name': user_group.users_group_name,
                 'type': "user_group",
                 'permission': perm
             }
             members.append(user_group_data)
 
         followers = [
             uf.user.get_api_data()
             for uf in repo.followers
         ]
 
         data = repo.get_api_data(with_revision_names=with_revision_names,
                                  with_pullrequests=with_pullrequests)
         data['members'] = members
         data['followers'] = followers
         return data
 
     # permission check inside
     def get_repos(self):
         """
         Lists all existing repositories. This command can be executed only using
         api_key belonging to user with admin rights or regular user that have
         admin, write or read access to repository.
 
 
         OUTPUT::
 
             id : <id_given_in_input>
             result: [
                       {
                         "repo_id" :          "<repo_id>",
                         "repo_name" :        "<reponame>"
                         "repo_type" :        "<repo_type>",
                         "clone_uri" :        "<clone_uri>",
                         "private": :         "<bool>",
                         "created_on" :       "<datetimecreated>",
                         "description" :      "<description>",
                         "landing_rev":       "<landing_rev>",
                         "owner":             "<repo_owner>",
                         "fork_of":           "<name_of_fork_parent>",
                         "enable_downloads":  "<bool>",
                         "enable_statistics": "<bool>",
                       },
                       …
                     ]
             error:  null
         """
         if not HasPermissionAny('hg.admin')():
             repos = request.authuser.get_all_user_repos()
         else:
             repos = db.Repository.query()
 
         return [
             repo.get_api_data()
             for repo in repos
         ]
 
     # permission check inside
     def get_repo_nodes(self, repoid, revision, root_path,
                        ret_type='all'):
         """
         returns a list of nodes and it's children in a flat list for a given path
         at given revision. It's possible to specify ret_type to show only `files` or
         `dirs`.  This command can be executed only using api_key belonging to
         user with admin rights or regular user that have at least read access to repository.
 
         :param repoid: repository name or repository id
         :type repoid: str or int
         :param revision: revision for which listing should be done
         :type revision: str
         :param root_path: path from which start displaying
         :type root_path: str
         :param ret_type: return type 'all|files|dirs' nodes
         :type ret_type: Optional(str)
 
 
         OUTPUT::
 
             id : <id_given_in_input>
             result: [
                       {
                         "name" :        "<name>"
                         "type" :        "<type>",
                       },
                       …
                     ]
             error:  null
         """
         repo = get_repo_or_error(repoid)
 
         if not HasPermissionAny('hg.admin')():
             if not HasRepoPermissionLevel('read')(repo.repo_name):
                 raise JSONRPCError('repository `%s` does not exist' % (repoid,))
 
-        ret_type = ret_type
         _map = {}
         try:
             _d, _f = ScmModel().get_nodes(repo, revision, root_path,
                                           flat=False)
             _map = {
                 'all': _d + _f,
                 'files': _f,
                 'dirs': _d,
             }
             return _map[ret_type]
         except KeyError:
             raise JSONRPCError('ret_type must be one of %s'
                                % (','.join(sorted(_map))))
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to get repo: `%s` nodes' % repo.repo_name
             )
 
     @HasPermissionAnyDecorator('hg.admin', 'hg.create.repository')
     def create_repo(self, repo_name, owner=None,
                     repo_type=None, description='',
                     private=False, clone_uri=None,
                     landing_rev='rev:tip',
                     enable_statistics=None,
                     enable_downloads=None,
                     copy_permissions=False):
         """
         Creates a repository. The repository name contains the full path, but the
         parent repository group must exist. For example "foo/bar/baz" require the groups
         "foo" and "bar" (with "foo" as parent), and create "baz" repository with
         "bar" as group. This command can be executed only using api_key
         belonging to user with admin rights or regular user that have create
         repository permission. Regular users cannot specify owner parameter
 
         :param repo_name: repository name
         :type repo_name: str
         :param owner: user_id or username
         :type owner: Optional(str)
         :param repo_type: 'hg' or 'git'
         :type repo_type: Optional(str)
         :param description: repository description
         :type description: Optional(str)
         :param private:
         :type private: bool
         :param clone_uri:
         :type clone_uri: str
         :param landing_rev: <rev_type>:<rev>
         :type landing_rev: str
         :param enable_downloads:
         :type enable_downloads: bool
         :param enable_statistics:
         :type enable_statistics: bool
         :param copy_permissions: Copy permission from group that repository is
             being created.
         :type copy_permissions: bool
 
         OUTPUT::
 
             id : <id_given_in_input>
             result: {
                       "msg": "Created new repository `<reponame>`",
                       "success": true,
                       "task": "<celery task id or None if done sync>"
                     }
             error:  null
 
         ERROR OUTPUT::
 
           id : <id_given_in_input>
           result : null
           error :  {
              'failed to create repository `<repo_name>`
           }
 
         """
         if not HasPermissionAny('hg.admin')():
             if owner is not None:
                 # forbid setting owner for non-admins
                 raise JSONRPCError(
                     'Only Kallithea admin can specify `owner` param'
                 )
         if owner is None:
             owner = request.authuser.user_id
 
         owner = get_user_or_error(owner)
 
         if RepoModel().get_by_repo_name(repo_name):
             raise JSONRPCError("repo `%s` already exist" % repo_name)
 
         defs = db.Setting.get_default_repo_settings(strip_prefix=True)
         if private is None:
             private = defs.get('repo_private') or False
         if repo_type is None:
             repo_type = defs.get('repo_type')
         if enable_statistics is None:
             enable_statistics = defs.get('repo_enable_statistics')
         if enable_downloads is None:
             enable_downloads = defs.get('repo_enable_downloads')
 
         try:
             repo_name_parts = repo_name.split('/')
             repo_group = None
             if len(repo_name_parts) > 1:
                 group_name = '/'.join(repo_name_parts[:-1])
                 repo_group = db.RepoGroup.get_by_group_name(group_name)
                 if repo_group is None:
                     raise JSONRPCError("repo group `%s` not found" % group_name)
             data = dict(
                 repo_name=repo_name_parts[-1],
                 repo_name_full=repo_name,
                 repo_type=repo_type,
                 repo_description=description,
                 owner=owner,
                 repo_private=private,
                 clone_uri=clone_uri,
                 repo_group=repo_group,
                 repo_landing_rev=landing_rev,
                 enable_statistics=enable_statistics,
                 enable_downloads=enable_downloads,
                 repo_copy_permissions=copy_permissions,
             )
 
             task = RepoModel().create(form_data=data, cur_user=owner)
             task_id = task.task_id
             # no commit, it's done in RepoModel, or async via celery
             return dict(
                 msg="Created new repository `%s`" % (repo_name,),
                 success=True,  # cannot return the repo data here since fork
                                # can be done async
                 task=task_id
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to create repository `%s`' % (repo_name,))
 
     # permission check inside
     def update_repo(self, repoid, name=None,
                     owner=None,
                     group=None,
                     description=None, private=None,
                     clone_uri=None, landing_rev=None,
                     enable_statistics=None,
                     enable_downloads=None):
 
         """
         Updates repo
 
         :param repoid: repository name or repository id
         :type repoid: str or int
         :param name:
         :param owner:
         :param group:
         :param description:
         :param private:
         :param clone_uri:
         :param landing_rev:
         :param enable_statistics:
         :param enable_downloads:
         """
         repo = get_repo_or_error(repoid)
         if not HasPermissionAny('hg.admin')():
             if not HasRepoPermissionLevel('admin')(repo.repo_name):
                 raise JSONRPCError('repository `%s` does not exist' % (repoid,))
 
             if (name != repo.repo_name and
                 not HasPermissionAny('hg.create.repository')()
             ):
                 raise JSONRPCError('no permission to create (or move) repositories')
 
             if owner is not None:
                 # forbid setting owner for non-admins
                 raise JSONRPCError(
                     'Only Kallithea admin can specify `owner` param'
                 )
 
         updates = {}
         repo_group = group
         if repo_group is not None:
             repo_group = get_repo_group_or_error(repo_group)
             repo_group = repo_group.group_id
         try:
             store_update(updates, name, 'repo_name')
             store_update(updates, repo_group, 'repo_group')
             store_update(updates, owner, 'owner')
             store_update(updates, description, 'repo_description')
             store_update(updates, private, 'repo_private')
             store_update(updates, clone_uri, 'clone_uri')
             store_update(updates, landing_rev, 'repo_landing_rev')
             store_update(updates, enable_statistics, 'repo_enable_statistics')
             store_update(updates, enable_downloads, 'repo_enable_downloads')
 
             RepoModel().update(repo, **updates)
             meta.Session().commit()
             return dict(
                 msg='updated repo ID:%s %s' % (repo.repo_id, repo.repo_name),
                 repository=repo.get_api_data()
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError('failed to update repo `%s`' % repoid)
 
     @HasPermissionAnyDecorator('hg.admin', 'hg.fork.repository')
     def fork_repo(self, repoid, fork_name,
                   owner=None,
                   description='', copy_permissions=False,
                   private=False, landing_rev='rev:tip'):
         """
         Creates a fork of given repo. In case of using celery this will
         immediately return success message, while fork is going to be created
         asynchronous. This command can be executed only using api_key belonging to
         user with admin rights or regular user that have fork permission, and at least
         read access to forking repository. Regular users cannot specify owner parameter.
 
         :param repoid: repository name or repository id
         :type repoid: str or int
         :param fork_name:
         :param owner:
         :param description:
         :param copy_permissions:
         :param private:
         :param landing_rev:
 
         INPUT::
 
             id : <id_for_response>
             api_key : "<api_key>"
             args:     {
                         "repoid" :          "<reponame or repo_id>",
                         "fork_name":        "<forkname>",
                         "owner":            "<username or user_id = Optional(=apiuser)>",
                         "description":      "<description>",
                         "copy_permissions": "<bool>",
                         "private":          "<bool>",
                         "landing_rev":      "<landing_rev>"
                       }
 
         OUTPUT::
 
             id : <id_given_in_input>
             result: {
                       "msg": "Created fork of `<reponame>` as `<forkname>`",
                       "success": true,
                       "task": "<celery task id or None if done sync>"
                     }
             error:  null
 
         """
         repo = get_repo_or_error(repoid)
         repo_name = repo.repo_name
 
         _repo = RepoModel().get_by_repo_name(fork_name)
         if _repo:
             type_ = 'fork' if _repo.fork else 'repo'
             raise JSONRPCError("%s `%s` already exist" % (type_, fork_name))
 
         if HasPermissionAny('hg.admin')():
             pass
         elif HasRepoPermissionLevel('read')(repo.repo_name):
             if owner is not None:
                 # forbid setting owner for non-admins
                 raise JSONRPCError(
                     'Only Kallithea admin can specify `owner` param'
                 )
 
             if not HasPermissionAny('hg.create.repository')():
                 raise JSONRPCError('no permission to create repositories')
         else:
             raise JSONRPCError('repository `%s` does not exist' % (repoid,))
 
         if owner is None:
             owner = request.authuser.user_id
 
         owner = get_user_or_error(owner)
 
         try:
             fork_name_parts = fork_name.split('/')
             repo_group = None
             if len(fork_name_parts) > 1:
                 group_name = '/'.join(fork_name_parts[:-1])
                 repo_group = db.RepoGroup.get_by_group_name(group_name)
                 if repo_group is None:
                     raise JSONRPCError("repo group `%s` not found" % group_name)
 
             form_data = dict(
                 repo_name=fork_name_parts[-1],
                 repo_name_full=fork_name,
                 repo_group=repo_group,
                 repo_type=repo.repo_type,
                 description=description,
                 private=private,
                 copy_permissions=copy_permissions,
                 landing_rev=landing_rev,
                 update_after_clone=False,
                 fork_parent_id=repo.repo_id,
             )
             task = RepoModel().create_fork(form_data, cur_user=owner)
             # no commit, it's done in RepoModel, or async via celery
             task_id = task.task_id
             return dict(
                 msg='Created fork of `%s` as `%s`' % (repo.repo_name,
                                                       fork_name),
                 success=True,  # cannot return the repo data here since fork
                                # can be done async
                 task=task_id
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to fork repository `%s` as `%s`' % (repo_name,
                                                             fork_name)
             )
 
     # permission check inside
     def delete_repo(self, repoid, forks=''):
         """
         Deletes a repository. This command can be executed only using api_key belonging
         to user with admin rights or regular user that have admin access to repository.
         When `forks` param is set it's possible to detach or delete forks of deleting
         repository
 
         :param repoid: repository name or repository id
         :type repoid: str or int
         :param forks: `detach` or `delete`, what do do with attached forks for repo
         :type forks: Optional(str)
 
         OUTPUT::
 
             id : <id_given_in_input>
             result: {
                       "msg": "Deleted repository `<reponame>`",
                       "success": true
                     }
             error:  null
 
         """
         repo = get_repo_or_error(repoid)
 
         if not HasPermissionAny('hg.admin')():
             if not HasRepoPermissionLevel('admin')(repo.repo_name):
                 raise JSONRPCError('repository `%s` does not exist' % (repoid,))
 
         try:
             handle_forks = forks
             _forks_msg = ''
             _forks = [f for f in repo.forks]
             if handle_forks == 'detach':
                 _forks_msg = ' ' + 'Detached %s forks' % len(_forks)
             elif handle_forks == 'delete':
                 _forks_msg = ' ' + 'Deleted %s forks' % len(_forks)
             elif _forks:
                 raise JSONRPCError(
                     'Cannot delete `%s` it still contains attached forks' %
                     (repo.repo_name,)
                 )
 
             RepoModel().delete(repo, forks=forks)
             meta.Session().commit()
             return dict(
                 msg='Deleted repository `%s`%s' % (repo.repo_name, _forks_msg),
                 success=True
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to delete repository `%s`' % (repo.repo_name,)
             )
 
     @HasPermissionAnyDecorator('hg.admin')
     def grant_user_permission(self, repoid, userid, perm):
         """
         Grant permission for user on given repository, or update existing one
         if found. This command can be executed only using api_key belonging to user
         with admin rights.
 
         :param repoid: repository name or repository id
         :type repoid: str or int
         :param userid:
         :param perm: (repository.(none|read|write|admin))
         :type perm: str
 
         OUTPUT::
 
             id : <id_given_in_input>
             result: {
                       "msg" : "Granted perm: `<perm>` for user: `<username>` in repo: `<reponame>`",
                       "success": true
                     }
             error:  null
         """
         repo = get_repo_or_error(repoid)
         user = get_user_or_error(userid)
         perm = get_perm_or_error(perm)
 
         try:
 
             RepoModel().grant_user_permission(repo=repo, user=user, perm=perm)
 
             meta.Session().commit()
             return dict(
                 msg='Granted perm: `%s` for user: `%s` in repo: `%s`' % (
                     perm.permission_name, user.username, repo.repo_name
                 ),
                 success=True
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to edit permission for user: `%s` in repo: `%s`' % (
                     userid, repoid
                 )
             )
 
     @HasPermissionAnyDecorator('hg.admin')
     def revoke_user_permission(self, repoid, userid):
         """
         Revoke permission for user on given repository. This command can be executed
         only using api_key belonging to user with admin rights.
 
         :param repoid: repository name or repository id
         :type repoid: str or int
         :param userid:
 
         OUTPUT::
 
             id : <id_given_in_input>
             result: {
                       "msg" : "Revoked perm for user: `<username>` in repo: `<reponame>`",
                       "success": true
                     }
             error:  null
 
         """
 
         repo = get_repo_or_error(repoid)
         user = get_user_or_error(userid)
         try:
             RepoModel().revoke_user_permission(repo=repo, user=user)
             meta.Session().commit()
             return dict(
                 msg='Revoked perm for user: `%s` in repo: `%s`' % (
                     user.username, repo.repo_name
                 ),
                 success=True
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to edit permission for user: `%s` in repo: `%s`' % (
                     userid, repoid
                 )
             )
 
     # permission check inside
     def grant_user_group_permission(self, repoid, usergroupid, perm):
         """
         Grant permission for user group on given repository, or update
         existing one if found. This command can be executed only using
         api_key belonging to user with admin rights.
 
         :param repoid: repository name or repository id
         :type repoid: str or int
         :param usergroupid: id of usergroup
         :type usergroupid: str or int
         :param perm: (repository.(none|read|write|admin))
         :type perm: str
 
         OUTPUT::
 
           id : <id_given_in_input>
           result : {
             "msg" : "Granted perm: `<perm>` for group: `<usersgroupname>` in repo: `<reponame>`",
             "success": true
 
           }
           error :  null
 
         ERROR OUTPUT::
 
           id : <id_given_in_input>
           result : null
           error :  {
             "failed to edit permission for user group: `<usergroup>` in repo `<repo>`'
           }
 
         """
         repo = get_repo_or_error(repoid)
         perm = get_perm_or_error(perm)
         user_group = get_user_group_or_error(usergroupid)
         if not HasPermissionAny('hg.admin')():
             if not HasRepoPermissionLevel('admin')(repo.repo_name):
                 raise JSONRPCError('repository `%s` does not exist' % (repoid,))
 
             if not HasUserGroupPermissionLevel('read')(user_group.users_group_name):
                 raise JSONRPCError('user group `%s` does not exist' % (usergroupid,))
 
         try:
             RepoModel().grant_user_group_permission(
                 repo=repo, group_name=user_group, perm=perm)
 
             meta.Session().commit()
             return dict(
                 msg='Granted perm: `%s` for user group: `%s` in '
                     'repo: `%s`' % (
                         perm.permission_name, user_group.users_group_name,
                         repo.repo_name
                     ),
                 success=True
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to edit permission for user group: `%s` in '
                 'repo: `%s`' % (
                     usergroupid, repo.repo_name
                 )
             )
 
     # permission check inside
     def revoke_user_group_permission(self, repoid, usergroupid):
         """
         Revoke permission for user group on given repository. This command can be
         executed only using api_key belonging to user with admin rights.
 
         :param repoid: repository name or repository id
         :type repoid: str or int
         :param usergroupid:
 
         OUTPUT::
 
             id : <id_given_in_input>
             result: {
                       "msg" : "Revoked perm for group: `<usersgroupname>` in repo: `<reponame>`",
                       "success": true
                     }
             error:  null
         """
         repo = get_repo_or_error(repoid)
         user_group = get_user_group_or_error(usergroupid)
         if not HasPermissionAny('hg.admin')():
             if not HasRepoPermissionLevel('admin')(repo.repo_name):
                 raise JSONRPCError('repository `%s` does not exist' % (repoid,))
 
             if not HasUserGroupPermissionLevel('read')(user_group.users_group_name):
                 raise JSONRPCError('user group `%s` does not exist' % (usergroupid,))
 
         try:
             RepoModel().revoke_user_group_permission(
                 repo=repo, group_name=user_group)
 
             meta.Session().commit()
             return dict(
                 msg='Revoked perm for user group: `%s` in repo: `%s`' % (
                     user_group.users_group_name, repo.repo_name
                 ),
                 success=True
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to edit permission for user group: `%s` in '
                 'repo: `%s`' % (
                     user_group.users_group_name, repo.repo_name
                 )
             )
 
     @HasPermissionAnyDecorator('hg.admin')
     def get_repo_group(self, repogroupid):
         """
         Returns given repo group together with permissions, and repositories
         inside the group
 
         :param repogroupid: id/name of repository group
         :type repogroupid: str or int
         """
         repo_group = get_repo_group_or_error(repogroupid)
 
         members = []
         for user in repo_group.repo_group_to_perm:
             perm = user.permission.permission_name
             user = user.user
             user_data = {
                 'name': user.username,
                 'type': "user",
                 'permission': perm
             }
             members.append(user_data)
 
         for user_group in repo_group.users_group_to_perm:
             perm = user_group.permission.permission_name
             user_group = user_group.users_group
             user_group_data = {
                 'name': user_group.users_group_name,
                 'type': "user_group",
                 'permission': perm
             }
             members.append(user_group_data)
 
         data = repo_group.get_api_data()
         data["members"] = members
         return data
 
     @HasPermissionAnyDecorator('hg.admin')
     def get_repo_groups(self):
         """
         Returns all repository groups
 
         """
         return [
             repo_group.get_api_data()
             for repo_group in db.RepoGroup.query()
         ]
 
     @HasPermissionAnyDecorator('hg.admin')
     def create_repo_group(self, group_name, description='',
                           owner=None,
                           parent=None,
                           copy_permissions=False):
         """
         Creates a repository group. This command can be executed only using
         api_key belonging to user with admin rights.
 
         :param group_name:
         :type group_name:
         :param description:
         :type description:
         :param owner:
         :type owner:
         :param parent:
         :type parent:
         :param copy_permissions:
         :type copy_permissions:
 
         OUTPUT::
 
           id : <id_given_in_input>
           result : {
               "msg": "created new repo group `<repo_group_name>`"
               "repo_group": <repogroup_object>
           }
           error :  null
 
         ERROR OUTPUT::
 
           id : <id_given_in_input>
           result : null
           error :  {
             failed to create repo group `<repogroupid>`
           }
 
         """
         if db.RepoGroup.get_by_group_name(group_name):
             raise JSONRPCError("repo group `%s` already exist" % (group_name,))
 
         if owner is None:
             owner = request.authuser.user_id
         group_description = description
         parent_group = None
         if parent is not None:
             parent_group = get_repo_group_or_error(parent)
 
         try:
             repo_group = RepoGroupModel().create(
                 group_name=group_name,
                 group_description=group_description,
                 owner=owner,
                 parent=parent_group,
                 copy_permissions=copy_permissions
             )
             meta.Session().commit()
             return dict(
                 msg='created new repo group `%s`' % group_name,
                 repo_group=repo_group.get_api_data()
             )
         except Exception:
 
             log.error(traceback.format_exc())
             raise JSONRPCError('failed to create repo group `%s`' % (group_name,))
 
     @HasPermissionAnyDecorator('hg.admin')
     def update_repo_group(self, repogroupid, group_name=None,
                           description=None,
                           owner=None,
                           parent=None):
         repo_group = get_repo_group_or_error(repogroupid)
 
         updates = {}
         try:
             store_update(updates, group_name, 'group_name')
             store_update(updates, description, 'group_description')
             store_update(updates, owner, 'owner')
             store_update(updates, parent, 'parent_group')
             repo_group = RepoGroupModel().update(repo_group, updates)
             meta.Session().commit()
             return dict(
                 msg='updated repository group ID:%s %s' % (repo_group.group_id,
                                                            repo_group.group_name),
                 repo_group=repo_group.get_api_data()
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError('failed to update repository group `%s`'
                                % (repogroupid,))
 
     @HasPermissionAnyDecorator('hg.admin')
     def delete_repo_group(self, repogroupid):
         """
 
         :param repogroupid: name or id of repository group
         :type repogroupid: str or int
 
         OUTPUT::
 
           id : <id_given_in_input>
           result : {
             'msg': 'deleted repo group ID:<repogroupid> <repogroupname>
             'repo_group': null
           }
           error :  null
 
         ERROR OUTPUT::
 
           id : <id_given_in_input>
           result : null
           error :  {
             "failed to delete repo group ID:<repogroupid> <repogroupname>"
           }
 
         """
         repo_group = get_repo_group_or_error(repogroupid)
 
         try:
             RepoGroupModel().delete(repo_group)
             meta.Session().commit()
             return dict(
                 msg='deleted repo group ID:%s %s' %
                     (repo_group.group_id, repo_group.group_name),
                 repo_group=None
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError('failed to delete repo group ID:%s %s' %
                                (repo_group.group_id, repo_group.group_name)
                                )
 
     # permission check inside
     def grant_user_permission_to_repo_group(self, repogroupid, userid,
                                             perm, apply_to_children='none'):
         """
         Grant permission for user on given repository group, or update existing
         one if found. This command can be executed only using api_key belonging
         to user with admin rights, or user who has admin right to given repository
         group.
 
         :param repogroupid: name or id of repository group
         :type repogroupid: str or int
         :param userid:
         :param perm: (group.(none|read|write|admin))
         :type perm: str
         :param apply_to_children: 'none', 'repos', 'groups', 'all'
         :type apply_to_children: str
 
         OUTPUT::
 
             id : <id_given_in_input>
             result: {
                       "msg" : "Granted perm: `<perm>` (recursive:<apply_to_children>) for user: `<username>` in repo group: `<repo_group_name>`",
                       "success": true
                     }
             error:  null
 
         ERROR OUTPUT::
 
           id : <id_given_in_input>
           result : null
           error :  {
             "failed to edit permission for user: `<userid>` in repo group: `<repo_group_name>`"
           }
 
         """
 
         repo_group = get_repo_group_or_error(repogroupid)
 
         if not HasPermissionAny('hg.admin')():
             if not HasRepoGroupPermissionLevel('admin')(repo_group.group_name):
                 raise JSONRPCError('repository group `%s` does not exist' % (repogroupid,))
 
         user = get_user_or_error(userid)
         perm = get_perm_or_error(perm, prefix='group.')
 
         try:
             RepoGroupModel().add_permission(repo_group=repo_group,
                                             obj=user,
                                             obj_type="user",
                                             perm=perm,
                                             recursive=apply_to_children)
             meta.Session().commit()
             return dict(
                 msg='Granted perm: `%s` (recursive:%s) for user: `%s` in repo group: `%s`' % (
                     perm.permission_name, apply_to_children, user.username, repo_group.name
                 ),
                 success=True
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to edit permission for user: `%s` in repo group: `%s`' % (
                     userid, repo_group.name))
 
     # permission check inside
     def revoke_user_permission_from_repo_group(self, repogroupid, userid,
                                                apply_to_children='none'):
         """
         Revoke permission for user on given repository group. This command can
         be executed only using api_key belonging to user with admin rights, or
         user who has admin right to given repository group.
 
         :param repogroupid: name or id of repository group
         :type repogroupid: str or int
         :param userid:
         :type userid:
         :param apply_to_children: 'none', 'repos', 'groups', 'all'
         :type apply_to_children: str
 
         OUTPUT::
 
             id : <id_given_in_input>
             result: {
                       "msg" : "Revoked perm (recursive:<apply_to_children>) for user: `<username>` in repo group: `<repo_group_name>`",
                       "success": true
                     }
             error:  null
 
         ERROR OUTPUT::
 
           id : <id_given_in_input>
           result : null
           error :  {
             "failed to edit permission for user: `<userid>` in repo group: `<repo_group_name>`"
           }
 
         """
 
         repo_group = get_repo_group_or_error(repogroupid)
 
         if not HasPermissionAny('hg.admin')():
             if not HasRepoGroupPermissionLevel('admin')(repo_group.group_name):
                 raise JSONRPCError('repository group `%s` does not exist' % (repogroupid,))
 
         user = get_user_or_error(userid)
 
         try:
             RepoGroupModel().delete_permission(repo_group=repo_group,
                                                obj=user,
                                                obj_type="user",
                                                recursive=apply_to_children)
 
             meta.Session().commit()
             return dict(
                 msg='Revoked perm (recursive:%s) for user: `%s` in repo group: `%s`' % (
                     apply_to_children, user.username, repo_group.name
                 ),
                 success=True
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to edit permission for user: `%s` in repo group: `%s`' % (
                     userid, repo_group.name))
 
     # permission check inside
     def grant_user_group_permission_to_repo_group(
             self, repogroupid, usergroupid, perm,
             apply_to_children='none'):
         """
         Grant permission for user group on given repository group, or update
         existing one if found. This command can be executed only using
         api_key belonging to user with admin rights, or user who has admin
         right to given repository group.
 
         :param repogroupid: name or id of repository group
         :type repogroupid: str or int
         :param usergroupid: id of usergroup
         :type usergroupid: str or int
         :param perm: (group.(none|read|write|admin))
         :type perm: str
         :param apply_to_children: 'none', 'repos', 'groups', 'all'
         :type apply_to_children: str
 
         OUTPUT::
 
           id : <id_given_in_input>
           result : {
             "msg" : "Granted perm: `<perm>` (recursive:<apply_to_children>) for user group: `<usersgroupname>` in repo group: `<repo_group_name>`",
             "success": true
 
           }
           error :  null
 
         ERROR OUTPUT::
 
           id : <id_given_in_input>
           result : null
           error :  {
             "failed to edit permission for user group: `<usergroup>` in repo group: `<repo_group_name>`"
           }
 
         """
         repo_group = get_repo_group_or_error(repogroupid)
         perm = get_perm_or_error(perm, prefix='group.')
         user_group = get_user_group_or_error(usergroupid)
         if not HasPermissionAny('hg.admin')():
             if not HasRepoGroupPermissionLevel('admin')(repo_group.group_name):
                 raise JSONRPCError(
                     'repository group `%s` does not exist' % (repogroupid,))
 
             if not HasUserGroupPermissionLevel('read')(user_group.users_group_name):
                 raise JSONRPCError(
                     'user group `%s` does not exist' % (usergroupid,))
 
         try:
             RepoGroupModel().add_permission(repo_group=repo_group,
                                             obj=user_group,
                                             obj_type="user_group",
                                             perm=perm,
                                             recursive=apply_to_children)
             meta.Session().commit()
             return dict(
                 msg='Granted perm: `%s` (recursive:%s) for user group: `%s` in repo group: `%s`' % (
                     perm.permission_name, apply_to_children,
                     user_group.users_group_name, repo_group.name
                 ),
                 success=True
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to edit permission for user group: `%s` in '
                 'repo group: `%s`' % (
                     usergroupid, repo_group.name
                 )
             )
 
     # permission check inside
     def revoke_user_group_permission_from_repo_group(
             self, repogroupid, usergroupid,
             apply_to_children='none'):
         """
         Revoke permission for user group on given repository. This command can be
         executed only using api_key belonging to user with admin rights, or
         user who has admin right to given repository group.
 
         :param repogroupid: name or id of repository group
         :type repogroupid: str or int
         :param usergroupid:
         :param apply_to_children: 'none', 'repos', 'groups', 'all'
         :type apply_to_children: str
 
         OUTPUT::
 
             id : <id_given_in_input>
             result: {
                       "msg" : "Revoked perm (recursive:<apply_to_children>) for user group: `<usersgroupname>` in repo group: `<repo_group_name>`",
                       "success": true
                     }
             error:  null
 
         ERROR OUTPUT::
 
           id : <id_given_in_input>
           result : null
           error :  {
             "failed to edit permission for user group: `<usergroup>` in repo group: `<repo_group_name>`"
           }
 
 
         """
         repo_group = get_repo_group_or_error(repogroupid)
         user_group = get_user_group_or_error(usergroupid)
         if not HasPermissionAny('hg.admin')():
             if not HasRepoGroupPermissionLevel('admin')(repo_group.group_name):
                 raise JSONRPCError(
                     'repository group `%s` does not exist' % (repogroupid,))
 
             if not HasUserGroupPermissionLevel('read')(user_group.users_group_name):
                 raise JSONRPCError(
                     'user group `%s` does not exist' % (usergroupid,))
 
         try:
             RepoGroupModel().delete_permission(repo_group=repo_group,
                                                obj=user_group,
                                                obj_type="user_group",
                                                recursive=apply_to_children)
             meta.Session().commit()
             return dict(
                 msg='Revoked perm (recursive:%s) for user group: `%s` in repo group: `%s`' % (
                     apply_to_children, user_group.users_group_name, repo_group.name
                 ),
                 success=True
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to edit permission for user group: `%s` in repo group: `%s`' % (
                     user_group.users_group_name, repo_group.name
                 )
             )
 
     def get_gist(self, gistid):
         """
         Get given gist by id
 
         :param gistid: id of private or public gist
         :type gistid: str
         """
         gist = get_gist_or_error(gistid)
         if not HasPermissionAny('hg.admin')():
             if gist.owner_id != request.authuser.user_id:
                 raise JSONRPCError('gist `%s` does not exist' % (gistid,))
         return gist.get_api_data()
 
     def get_gists(self, userid=None):
         """
         Get all gists for given user. If userid is empty returned gists
         are for user who called the api
 
         :param userid: user to get gists for
         :type userid: Optional(str or int)
         """
         if not HasPermissionAny('hg.admin')():
             # make sure normal user does not pass someone else userid,
             # he is not allowed to do that
             if userid is not None and userid != request.authuser.user_id:
                 raise JSONRPCError(
                     'userid is not the same as your user'
                 )
 
         if userid is None:
             user_id = request.authuser.user_id
         else:
             user_id = get_user_or_error(userid).user_id
 
         return [
             gist.get_api_data()
             for gist in db.Gist().query()
                 .filter_by(is_expired=False)
                 .filter(db.Gist.owner_id == user_id)
                 .order_by(db.Gist.created_on.desc())
         ]
 
     def create_gist(self, files, owner=None,
                     gist_type=db.Gist.GIST_PUBLIC, lifetime=-1,
                     description=''):
 
         """
         Creates new Gist
 
         :param files: files to be added to gist
             {'filename': {'content':'...', 'lexer': null},
              'filename2': {'content':'...', 'lexer': null}}
         :type files: dict
         :param owner: gist owner, defaults to api method caller
         :type owner: Optional(str or int)
         :param gist_type: type of gist 'public' or 'private'
         :type gist_type: Optional(str)
         :param lifetime: time in minutes of gist lifetime
         :type lifetime: Optional(int)
         :param description: gist description
         :type description: Optional(str)
 
         OUTPUT::
 
           id : <id_given_in_input>
           result : {
             "msg": "created new gist",
             "gist": {}
           }
           error :  null
 
         ERROR OUTPUT::
 
           id : <id_given_in_input>
           result : null
           error :  {
             "failed to create gist"
           }
 
         """
         try:
             if owner is None:
                 owner = request.authuser.user_id
 
             owner = get_user_or_error(owner)
 
             gist = GistModel().create(description=description,
                                       owner=owner,
                                       ip_addr=request.ip_addr,
                                       gist_mapping=files,
                                       gist_type=gist_type,
                                       lifetime=lifetime)
             meta.Session().commit()
             return dict(
                 msg='created new gist',
                 gist=gist.get_api_data()
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError('failed to create gist')
 
     # permission check inside
     def delete_gist(self, gistid):
         """
         Deletes existing gist
 
         :param gistid: id of gist to delete
         :type gistid: str
 
         OUTPUT::
 
           id : <id_given_in_input>
           result : {
             "deleted gist ID: <gist_id>",
             "gist": null
           }
           error :  null
 
         ERROR OUTPUT::
 
           id : <id_given_in_input>
           result : null
           error :  {
             "failed to delete gist ID:<gist_id>"
           }
 
         """
         gist = get_gist_or_error(gistid)
         if not HasPermissionAny('hg.admin')():
             if gist.owner_id != request.authuser.user_id:
                 raise JSONRPCError('gist `%s` does not exist' % (gistid,))
 
         try:
             GistModel().delete(gist)
             meta.Session().commit()
             return dict(
                 msg='deleted gist ID:%s' % (gist.gist_access_id,),
                 gist=None
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError('failed to delete gist ID:%s'
                                % (gist.gist_access_id,))
 
     # permission check inside
     def get_changesets(self, repoid, start=None, end=None, start_date=None,
                        end_date=None, branch_name=None, reverse=False, with_file_list=False, max_revisions=None):
         repo = get_repo_or_error(repoid)
         if not HasRepoPermissionLevel('read')(repo.repo_name):
             raise JSONRPCError('Access denied to repo %s' % repo.repo_name)
 
         format = "%Y-%m-%dT%H:%M:%S"
         try:
             return [e.__json__(with_file_list) for e in
                 repo.scm_instance.get_changesets(start,
                                                  end,
                                                  datetime.strptime(start_date, format) if start_date else None,
                                                  datetime.strptime(end_date, format) if end_date else None,
                                                  branch_name,
                                                  reverse, max_revisions)]
         except EmptyRepositoryError as e:
             raise JSONRPCError('Repository is empty')
 
     # permission check inside
     def get_changeset(self, repoid, raw_id, with_reviews=False):
         repo = get_repo_or_error(repoid)
         if not HasRepoPermissionLevel('read')(repo.repo_name):
             raise JSONRPCError('Access denied to repo %s' % repo.repo_name)
         changeset = repo.get_changeset(raw_id)
         if isinstance(changeset, EmptyChangeset):
             raise JSONRPCError('Changeset %s does not exist' % raw_id)
 
         info = dict(changeset.as_dict())
 
         if with_reviews:
             reviews = ChangesetStatusModel().get_statuses(
                                 repo.repo_name, raw_id)
             info["reviews"] = reviews
 
         return info
 
     # permission check inside
     def get_pullrequest(self, pullrequest_id):
         """
         Get given pull request by id
         """
         pull_request = db.PullRequest.get(pullrequest_id)
         if pull_request is None:
             raise JSONRPCError('pull request `%s` does not exist' % (pullrequest_id,))
         if not HasRepoPermissionLevel('read')(pull_request.org_repo.repo_name):
             raise JSONRPCError('not allowed')
         return pull_request.get_api_data()
 
     # permission check inside
     def comment_pullrequest(self, pull_request_id, comment_msg='', status=None, close_pr=False):
         """
         Add comment, close and change status of pull request.
         """
         apiuser = get_user_or_error(request.authuser.user_id)
         pull_request = db.PullRequest.get(pull_request_id)
         if pull_request is None:
             raise JSONRPCError('pull request `%s` does not exist' % (pull_request_id,))
         if (not HasRepoPermissionLevel('read')(pull_request.org_repo.repo_name)):
             raise JSONRPCError('No permission to add comment. User needs at least reading permissions'
                                ' to the source repository.')
         owner = apiuser.user_id == pull_request.owner_id
         reviewer = apiuser.user_id in [reviewer.user_id for reviewer in pull_request.reviewers]
         if close_pr and not (apiuser.admin or owner):
             raise JSONRPCError('No permission to close pull request. User needs to be admin or owner.')
         if status and not (apiuser.admin or owner or reviewer):
             raise JSONRPCError('No permission to change pull request status. User needs to be admin, owner or reviewer.')
         if pull_request.is_closed():
             raise JSONRPCError('pull request is already closed')
 
         comment = ChangesetCommentsModel().create(
             text=comment_msg,
             repo=pull_request.org_repo.repo_id,
             author=apiuser.user_id,
             pull_request=pull_request.pull_request_id,
             f_path=None,
             line_no=None,
             status_change=db.ChangesetStatus.get_status_lbl(status),
             closing_pr=close_pr
         )
         userlog.action_logger(apiuser,
                       'user_commented_pull_request:%s' % pull_request_id,
                       pull_request.org_repo, request.ip_addr)
         if status:
             ChangesetStatusModel().set_status(
                 pull_request.org_repo_id,
                 status,
                 apiuser.user_id,
                 comment,
                 pull_request=pull_request_id
             )
         if close_pr:
             PullRequestModel().close_pull_request(pull_request_id)
             userlog.action_logger(apiuser,
                           'user_closed_pull_request:%s' % pull_request_id,
                           pull_request.org_repo, request.ip_addr)
         meta.Session().commit()
         return True
 
     # permission check inside
     def edit_reviewers(self, pull_request_id, add=None, remove=None):
         """
         Add and/or remove one or more reviewers to a pull request, by username
         or user ID. Reviewers are specified either as a single-user string or
         as a JSON list of one or more strings.
         """
         if add is None and remove is None:
             raise JSONRPCError('''Invalid request. Neither 'add' nor 'remove' is specified.''')
 
         pull_request = db.PullRequest.get(pull_request_id)
         if pull_request is None:
             raise JSONRPCError('pull request `%s` does not exist' % (pull_request_id,))
 
         apiuser = get_user_or_error(request.authuser.user_id)
         is_owner = apiuser.user_id == pull_request.owner_id
         is_repo_admin = HasRepoPermissionLevel('admin')(pull_request.other_repo.repo_name)
         if not (apiuser.admin or is_repo_admin or is_owner):
             raise JSONRPCError('No permission to edit reviewers of this pull request. User needs to be admin or pull request owner.')
         if pull_request.is_closed():
             raise JSONRPCError('Cannot edit reviewers of a closed pull request.')
 
         if not isinstance(add, list):
             add = [add]
         if not isinstance(remove, list):
             remove = [remove]
 
         # look up actual user objects from given name or id. Bail out if unknown.
         add_objs = set(get_user_or_error(user) for user in add if user is not None)
         remove_objs = set(get_user_or_error(user) for user in remove if user is not None)
 
         new_reviewers = redundant_reviewers = set()
         if add_objs:
             new_reviewers, redundant_reviewers = PullRequestModel().add_reviewers(apiuser, pull_request, add_objs)
         if remove_objs:
             PullRequestModel().remove_reviewers(apiuser, pull_request, remove_objs)
 
         meta.Session().commit()
 
         return {
             'added': [x.username for x in new_reviewers],
             'already_present': [x.username for x in redundant_reviewers],
             # NOTE: no explicit check that removed reviewers were actually present.
             'removed': [x.username for x in remove_objs],
         }

kallithea/model/repo.py

@@ -1,700 +1,698 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.model.repo
 ~~~~~~~~~~~~~~~~~~~~
 
 Repository model for kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Jun 5, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 
 """
 
 import logging
 import os
 import shutil
 import traceback
 from datetime import datetime
 
 import kallithea.lib.utils2
 from kallithea.lib import hooks
 from kallithea.lib.auth import HasRepoPermissionLevel, HasUserGroupPermissionLevel
 from kallithea.lib.exceptions import AttachedForksError
 from kallithea.lib.utils import is_valid_repo_uri, make_ui
 from kallithea.lib.utils2 import LazyProperty, get_current_authuser, obfuscate_url_pw, remove_prefix
 from kallithea.lib.vcs.backends import get_backend
 from kallithea.model import db, meta, scm
 
 
 log = logging.getLogger(__name__)
 
 
 class RepoModel(object):
 
     def _create_default_perms(self, repository, private):
         # create default permission
         default = 'repository.read'
         def_user = db.User.get_default_user()
         for p in def_user.user_perms:
             if p.permission.permission_name.startswith('repository.'):
                 default = p.permission.permission_name
                 break
 
         default_perm = 'repository.none' if private else default
 
         repo_to_perm = db.UserRepoToPerm()
         repo_to_perm.permission = db.Permission.get_by_key(default_perm)
 
         repo_to_perm.repository = repository
         repo_to_perm.user_id = def_user.user_id
         meta.Session().add(repo_to_perm)
 
         return repo_to_perm
 
     @LazyProperty
     def repos_path(self):
         """
         Gets the repositories root path from database
         """
 
         q = db.Ui.query().filter(db.Ui.ui_key == '/').one()
         return q.ui_value
 
     def get(self, repo_id):
         repo = db.Repository.query() \
             .filter(db.Repository.repo_id == repo_id)
         return repo.scalar()
 
     def get_repo(self, repository):
         return db.Repository.guess_instance(repository)
 
     def get_by_repo_name(self, repo_name):
         repo = db.Repository.query() \
             .filter(db.Repository.repo_name == repo_name)
         return repo.scalar()
 
     @classmethod
     def _render_datatable(cls, tmpl, *args, **kwargs):
         from tg import app_globals, request
         from tg import tmpl_context as c
         from tg.i18n import ugettext as _
 
         import kallithea.lib.helpers as h
 
         _tmpl_lookup = app_globals.mako_lookup
         template = _tmpl_lookup.get_template('data_table/_dt_elements.html')
 
         tmpl = template.get_def(tmpl)
         kwargs.update(dict(_=_, h=h, c=c, request=request))
         return tmpl.render_unicode(*args, **kwargs)
 
     def get_repos_as_dict(self, repos_list, repo_groups_list=None,
                           admin=False,
                           short_name=False):
         """Return repository list for use by DataTable.
         repos_list: list of repositories - but will be filtered for read permission.
         repo_groups_list: added at top of list without permission check.
         admin: return data for action column.
         """
         _render = self._render_datatable
         from tg import request
         from tg import tmpl_context as c
 
         import kallithea.lib.helpers as h
 
         def repo_lnk(name, rtype, rstate, private, fork_of):
             return _render('repo_name', name, rtype, rstate, private, fork_of,
                            short_name=short_name)
 
         def following(repo_id, is_following):
             return _render('following', repo_id, is_following)
 
         def last_change(last_change):
             return _render("last_change", last_change)
 
         def rss_lnk(repo_name):
             return _render("rss", repo_name)
 
         def atom_lnk(repo_name):
             return _render("atom", repo_name)
 
         def last_rev(repo_name, cs_cache):
             return _render('revision', repo_name, cs_cache.get('revision'),
                            cs_cache.get('raw_id'), cs_cache.get('author'),
                            cs_cache.get('message'))
 
         def desc(desc):
             import kallithea.lib.helpers as h
             return h.urlify_text(desc, truncate=80, stylize=c.visual.stylify_metalabels)
 
         def state(repo_state):
             return _render("repo_state", repo_state)
 
         def repo_actions(repo_name):
             return _render('repo_actions', repo_name)
 
         def owner_actions(owner_id, username):
             return _render('user_name', owner_id, username)
 
         repos_data = []
 
         for gr in repo_groups_list or []:
             repos_data.append(dict(
                 raw_name='\0' + gr.name, # sort before repositories
                 just_name=gr.name,
                 name=_render('group_name_html', group_name=gr.group_name, name=gr.name),
                 desc=gr.group_description))
 
         for repo in repos_list:
             if not HasRepoPermissionLevel('read')(repo.repo_name, 'get_repos_as_dict check'):
                 continue
             cs_cache = repo.changeset_cache
             row = {
                 "raw_name": repo.repo_name,
                 "just_name": repo.just_name,
                 "name": repo_lnk(repo.repo_name, repo.repo_type,
                                  repo.repo_state, repo.private, repo.fork),
                 "following": following(
                     repo.repo_id,
                     scm.ScmModel().is_following_repo(repo.repo_name, request.authuser.user_id),
                 ),
                 "last_change_iso": repo.last_db_change.isoformat(),
                 "last_change": last_change(repo.last_db_change),
                 "last_changeset": last_rev(repo.repo_name, cs_cache),
                 "last_rev_raw": cs_cache.get('revision'),
                 "desc": desc(repo.description),
                 "owner": h.person(repo.owner),
                 "state": state(repo.repo_state),
                 "rss": rss_lnk(repo.repo_name),
                 "atom": atom_lnk(repo.repo_name),
             }
             if admin:
                 row.update({
                     "action": repo_actions(repo.repo_name),
                     "owner": owner_actions(repo.owner_id,
                                            h.person(repo.owner))
                 })
             repos_data.append(row)
 
         return {
             "sort": "name",
             "dir": "asc",
             "records": repos_data
         }
 
     def _get_defaults(self, repo_name):
         """
         Gets information about repository, and returns a dict for
         usage in forms
 
         :param repo_name:
         """
 
         repo_info = db.Repository.get_by_repo_name(repo_name)
 
         if repo_info is None:
             return None
 
         defaults = repo_info.get_dict()
         defaults['repo_name'] = repo_info.just_name
         defaults['repo_group'] = repo_info.group_id
 
         for strip, k in [(0, 'repo_type'), (1, 'repo_enable_downloads'),
                          (1, 'repo_description'),
                          (1, 'repo_landing_rev'), (0, 'clone_uri'),
                          (1, 'repo_private'), (1, 'repo_enable_statistics')]:
             attr = k
             if strip:
                 attr = remove_prefix(k, 'repo_')
 
             val = defaults[attr]
             if k == 'repo_landing_rev':
                 val = ':'.join(defaults[attr])
             defaults[k] = val
             if k == 'clone_uri':
                 defaults['clone_uri_hidden'] = repo_info.clone_uri_hidden
 
         # fill owner
         if repo_info.owner:
             defaults.update({'owner': repo_info.owner.username})
         else:
             replacement_user = db.User.query().filter(db.User.admin ==
                                                    True).first().username
             defaults.update({'owner': replacement_user})
 
         # fill repository users
         for p in repo_info.repo_to_perm:
             defaults.update({'u_perm_%s' % p.user.username:
                                  p.permission.permission_name})
 
         # fill repository groups
         for p in repo_info.users_group_to_perm:
             defaults.update({'g_perm_%s' % p.users_group.users_group_name:
                                  p.permission.permission_name})
 
         return defaults
 
     def update(self, repo, **kwargs):
         try:
             cur_repo = db.Repository.guess_instance(repo)
             org_repo_name = cur_repo.repo_name
             if 'owner' in kwargs:
                 cur_repo.owner = db.User.get_by_username(kwargs['owner'])
 
             if 'repo_group' in kwargs:
                 assert kwargs['repo_group'] != '-1', kwargs # RepoForm should have converted to None
                 cur_repo.group = db.RepoGroup.get(kwargs['repo_group'])
                 cur_repo.repo_name = cur_repo.get_new_name(cur_repo.just_name)
             log.debug('Updating repo %s with params:%s', cur_repo, kwargs)
             for k in ['repo_enable_downloads',
                       'repo_description',
                       'repo_landing_rev',
                       'repo_private',
                       'repo_enable_statistics',
                       ]:
                 if k in kwargs:
                     setattr(cur_repo, remove_prefix(k, 'repo_'), kwargs[k])
             clone_uri = kwargs.get('clone_uri')
             if clone_uri is not None and clone_uri != cur_repo.clone_uri_hidden:
                 # clone_uri is modified - if given a value, check it is valid
                 if clone_uri != '':
                     # will raise exception on error
                     is_valid_repo_uri(cur_repo.repo_type, clone_uri, make_ui())
                 cur_repo.clone_uri = clone_uri
 
             if 'repo_name' in kwargs:
                 repo_name = kwargs['repo_name']
                 if kallithea.lib.utils2.repo_name_slug(repo_name) != repo_name:
                     raise Exception('invalid repo name %s' % repo_name)
                 cur_repo.repo_name = cur_repo.get_new_name(repo_name)
 
             # if private flag is set, reset default permission to NONE
             if kwargs.get('repo_private'):
                 EMPTY_PERM = 'repository.none'
                 RepoModel().grant_user_permission(
                     repo=cur_repo, user='default', perm=EMPTY_PERM
                 )
                 # handle extra fields
             for field in [k for k in kwargs if k.startswith(db.RepositoryField.PREFIX)]:
                 k = db.RepositoryField.un_prefix_key(field)
                 ex_field = db.RepositoryField.get_by_key_name(key=k, repo=cur_repo)
                 if ex_field:
                     ex_field.field_value = kwargs[field]
 
             if org_repo_name != cur_repo.repo_name:
                 # rename repository
                 self._rename_filesystem_repo(old=org_repo_name, new=cur_repo.repo_name)
 
             return cur_repo
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def _create_repo(self, repo_name, repo_type, description, owner,
                      private=False, clone_uri=None, repo_group=None,
                      landing_rev='rev:tip', fork_of=None,
                      copy_fork_permissions=False, enable_statistics=False,
                      enable_downloads=False,
                      copy_group_permissions=False, state=db.Repository.STATE_PENDING):
         """
         Create repository inside database with PENDING state. This should only be
         executed by create() repo, with exception of importing existing repos.
 
         """
         owner = db.User.guess_instance(owner)
         fork_of = db.Repository.guess_instance(fork_of)
         repo_group = db.RepoGroup.guess_instance(repo_group)
         try:
-            repo_name = repo_name
-            description = description
             # repo name is just a name of repository
             # while repo_name_full is a full qualified name that is combined
             # with name and path of group
             repo_name_full = repo_name
             repo_name = repo_name.split(kallithea.URL_SEP)[-1]
             if kallithea.lib.utils2.repo_name_slug(repo_name) != repo_name:
                 raise Exception('invalid repo name %s' % repo_name)
 
             new_repo = db.Repository()
             new_repo.repo_state = state
             new_repo.enable_statistics = False
             new_repo.repo_name = repo_name_full
             new_repo.repo_type = repo_type
             new_repo.owner = owner
             new_repo.group = repo_group
             new_repo.description = description or repo_name
             new_repo.private = private
             if clone_uri:
                 # will raise exception on error
                 is_valid_repo_uri(repo_type, clone_uri, make_ui())
             new_repo.clone_uri = clone_uri
             new_repo.landing_rev = landing_rev
 
             new_repo.enable_statistics = enable_statistics
             new_repo.enable_downloads = enable_downloads
 
             if fork_of:
                 parent_repo = fork_of
                 new_repo.fork = parent_repo
 
             meta.Session().add(new_repo)
 
             if fork_of and copy_fork_permissions:
                 repo = fork_of
                 user_perms = db.UserRepoToPerm.query() \
                     .filter(db.UserRepoToPerm.repository == repo).all()
                 group_perms = db.UserGroupRepoToPerm.query() \
                     .filter(db.UserGroupRepoToPerm.repository == repo).all()
 
                 for perm in user_perms:
                     db.UserRepoToPerm.create(perm.user, new_repo, perm.permission)
 
                 for perm in group_perms:
                     db.UserGroupRepoToPerm.create(perm.users_group, new_repo,
                                                perm.permission)
 
             elif repo_group and copy_group_permissions:
 
                 user_perms = db.UserRepoGroupToPerm.query() \
                     .filter(db.UserRepoGroupToPerm.group == repo_group).all()
 
                 group_perms = db.UserGroupRepoGroupToPerm.query() \
                     .filter(db.UserGroupRepoGroupToPerm.group == repo_group).all()
 
                 for perm in user_perms:
                     perm_name = perm.permission.permission_name.replace('group.', 'repository.')
                     perm_obj = db.Permission.get_by_key(perm_name)
                     db.UserRepoToPerm.create(perm.user, new_repo, perm_obj)
 
                 for perm in group_perms:
                     perm_name = perm.permission.permission_name.replace('group.', 'repository.')
                     perm_obj = db.Permission.get_by_key(perm_name)
                     db.UserGroupRepoToPerm.create(perm.users_group, new_repo, perm_obj)
 
             else:
                 self._create_default_perms(new_repo, private)
 
             # now automatically start following this repository as owner
             scm.ScmModel().toggle_following_repo(new_repo.repo_id, owner.user_id)
             # we need to flush here, in order to check if database won't
             # throw any exceptions, create filesystem dirs at the very end
             meta.Session().flush()
             return new_repo
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def create(self, form_data, cur_user):
         """
         Create repository using celery tasks
 
         :param form_data:
         :param cur_user:
         """
         from kallithea.lib.celerylib import tasks
         return tasks.create_repo(form_data, cur_user)
 
     def _update_permissions(self, repo, perms_new=None, perms_updates=None,
                             check_perms=True):
         if not perms_new:
             perms_new = []
         if not perms_updates:
             perms_updates = []
 
         # update permissions
         for member, perm, member_type in perms_updates:
             if member_type == 'user':
                 # this updates existing one
                 self.grant_user_permission(
                     repo=repo, user=member, perm=perm
                 )
             else:
                 # check if we have permissions to alter this usergroup's access
                 if not check_perms or HasUserGroupPermissionLevel('read')(member):
                     self.grant_user_group_permission(
                         repo=repo, group_name=member, perm=perm
                     )
             # set new permissions
         for member, perm, member_type in perms_new:
             if member_type == 'user':
                 self.grant_user_permission(
                     repo=repo, user=member, perm=perm
                 )
             else:
                 # check if we have permissions to alter this usergroup's access
                 if not check_perms or HasUserGroupPermissionLevel('read')(member):
                     self.grant_user_group_permission(
                         repo=repo, group_name=member, perm=perm
                     )
 
     def create_fork(self, form_data, cur_user):
         """
         Simple wrapper into executing celery task for fork creation
 
         :param form_data:
         :param cur_user:
         """
         from kallithea.lib.celerylib import tasks
         return tasks.create_repo_fork(form_data, cur_user)
 
     def delete(self, repo, forks=None, fs_remove=True, cur_user=None):
         """
         Delete given repository, forks parameter defines what do do with
         attached forks. Throws AttachedForksError if deleted repo has attached
         forks
 
         :param repo:
         :param forks: str 'delete' or 'detach'
         :param fs_remove: remove(archive) repo from filesystem
         """
         if not cur_user:
             cur_user = getattr(get_current_authuser(), 'username', None)
         repo = db.Repository.guess_instance(repo)
         if repo is not None:
             if forks == 'detach':
                 for r in repo.forks:
                     r.fork = None
             elif forks == 'delete':
                 for r in repo.forks:
                     self.delete(r, forks='delete')
             elif [f for f in repo.forks]:
                 raise AttachedForksError()
 
             old_repo_dict = repo.get_dict()
             try:
                 meta.Session().delete(repo)
                 if fs_remove:
                     self._delete_filesystem_repo(repo)
                 else:
                     log.debug('skipping removal from filesystem')
                 hooks.log_delete_repository(old_repo_dict,
                                       deleted_by=cur_user)
             except Exception:
                 log.error(traceback.format_exc())
                 raise
 
     def grant_user_permission(self, repo, user, perm):
         """
         Grant permission for user on given repository, or update existing one
         if found
 
         :param repo: Instance of Repository, repository_id, or repository name
         :param user: Instance of User, user_id or username
         :param perm: Instance of Permission, or permission_name
         """
         user = db.User.guess_instance(user)
         repo = db.Repository.guess_instance(repo)
         permission = db.Permission.guess_instance(perm)
 
         # check if we have that permission already
         obj = db.UserRepoToPerm.query() \
             .filter(db.UserRepoToPerm.user == user) \
             .filter(db.UserRepoToPerm.repository == repo) \
             .scalar()
         if obj is None:
             # create new !
             obj = db.UserRepoToPerm()
             meta.Session().add(obj)
         obj.repository = repo
         obj.user = user
         obj.permission = permission
         log.debug('Granted perm %s to %s on %s', perm, user, repo)
         return obj
 
     def revoke_user_permission(self, repo, user):
         """
         Revoke permission for user on given repository
 
         :param repo: Instance of Repository, repository_id, or repository name
         :param user: Instance of User, user_id or username
         """
 
         user = db.User.guess_instance(user)
         repo = db.Repository.guess_instance(repo)
 
         obj = db.UserRepoToPerm.query() \
             .filter(db.UserRepoToPerm.repository == repo) \
             .filter(db.UserRepoToPerm.user == user) \
             .scalar()
         if obj is not None:
             meta.Session().delete(obj)
             log.debug('Revoked perm on %s on %s', repo, user)
 
     def grant_user_group_permission(self, repo, group_name, perm):
         """
         Grant permission for user group on given repository, or update
         existing one if found
 
         :param repo: Instance of Repository, repository_id, or repository name
         :param group_name: Instance of UserGroup, users_group_id,
             or user group name
         :param perm: Instance of Permission, or permission_name
         """
         repo = db.Repository.guess_instance(repo)
         group_name = db.UserGroup.guess_instance(group_name)
         permission = db.Permission.guess_instance(perm)
 
         # check if we have that permission already
         obj = db.UserGroupRepoToPerm.query() \
             .filter(db.UserGroupRepoToPerm.users_group == group_name) \
             .filter(db.UserGroupRepoToPerm.repository == repo) \
             .scalar()
 
         if obj is None:
             # create new
             obj = db.UserGroupRepoToPerm()
             meta.Session().add(obj)
 
         obj.repository = repo
         obj.users_group = group_name
         obj.permission = permission
         log.debug('Granted perm %s to %s on %s', perm, group_name, repo)
         return obj
 
     def revoke_user_group_permission(self, repo, group_name):
         """
         Revoke permission for user group on given repository
 
         :param repo: Instance of Repository, repository_id, or repository name
         :param group_name: Instance of UserGroup, users_group_id,
             or user group name
         """
         repo = db.Repository.guess_instance(repo)
         group_name = db.UserGroup.guess_instance(group_name)
 
         obj = db.UserGroupRepoToPerm.query() \
             .filter(db.UserGroupRepoToPerm.repository == repo) \
             .filter(db.UserGroupRepoToPerm.users_group == group_name) \
             .scalar()
         if obj is not None:
             meta.Session().delete(obj)
             log.debug('Revoked perm to %s on %s', repo, group_name)
 
     def delete_stats(self, repo_name):
         """
         removes stats for given repo
 
         :param repo_name:
         """
         repo = db.Repository.guess_instance(repo_name)
         try:
             obj = db.Statistics.query() \
                 .filter(db.Statistics.repository == repo).scalar()
             if obj is not None:
                 meta.Session().delete(obj)
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def _create_filesystem_repo(self, repo_name, repo_type, repo_group,
                                 clone_uri=None, repo_store_location=None):
         """
         Makes repository on filesystem. Operation is group aware, meaning that it will create
         a repository within a group, and alter the paths accordingly to the group location.
 
         Note: clone_uri is low level and not validated - it might be a file system path used for validated cloning
         """
         from kallithea.lib.utils import is_valid_repo, is_valid_repo_group
 
         if '/' in repo_name:
             raise ValueError('repo_name must not contain groups got `%s`' % repo_name)
 
         if isinstance(repo_group, db.RepoGroup):
             new_parent_path = os.sep.join(repo_group.full_path_splitted)
         else:
             new_parent_path = repo_group or ''
 
         if repo_store_location:
             _paths = [repo_store_location]
         else:
             _paths = [self.repos_path, new_parent_path, repo_name]
         repo_path = os.path.join(*_paths)
 
         # check if this path is not a repository
         if is_valid_repo(repo_path, self.repos_path):
             raise Exception('This path %s is a valid repository' % repo_path)
 
         # check if this path is a group
         if is_valid_repo_group(repo_path, self.repos_path):
             raise Exception('This path %s is a valid group' % repo_path)
 
         log.info('creating repo %s in %s from url: `%s`',
             repo_name, repo_path,
             obfuscate_url_pw(clone_uri))
 
         backend = get_backend(repo_type)
 
         if repo_type == 'hg':
             baseui = make_ui()
             # patch and reset hooks section of UI config to not run any
             # hooks on creating remote repo
             for k, v in baseui.configitems('hooks'):
                 baseui.setconfig('hooks', k, None)
 
             repo = backend(repo_path, create=True, src_url=clone_uri, baseui=baseui)
         elif repo_type == 'git':
             repo = backend(repo_path, create=True, src_url=clone_uri, bare=True)
             # add kallithea hook into this repo
             scm.ScmModel().install_git_hooks(repo)
         else:
             raise Exception('Not supported repo_type %s expected hg/git' % repo_type)
 
         log.debug('Created repo %s with %s backend',
                   repo_name, repo_type)
         return repo
 
     def _rename_filesystem_repo(self, old, new):
         """
         renames repository on filesystem
 
         :param old: old name
         :param new: new name
         """
         log.info('renaming repo from %s to %s', old, new)
 
         old_path = os.path.join(self.repos_path, old)
         new_path = os.path.join(self.repos_path, new)
         if os.path.isdir(new_path):
             raise Exception(
                 'Was trying to rename to already existing dir %s' % new_path
             )
         shutil.move(old_path, new_path)
 
     def _delete_filesystem_repo(self, repo):
         """
         removes repo from filesystem, the removal is actually done by
         renaming dir to a 'rm__*' prefix which Kallithea will skip.
         It can be undeleted later by reverting the rename.
 
         :param repo: repo object
         """
         rm_path = os.path.join(self.repos_path, repo.repo_name)
         log.info("Removing %s", rm_path)
 
         _now = datetime.now()
         _ms = str(_now.microsecond).rjust(6, '0')
         _d = 'rm__%s__%s' % (_now.strftime('%Y%m%d_%H%M%S_' + _ms),
                              repo.just_name)
         if repo.group:
             args = repo.group.full_path_splitted + [_d]
             _d = os.path.join(*args)
         if os.path.exists(rm_path):
             shutil.move(rm_path, os.path.join(self.repos_path, _d))
         else:
             log.error("Can't find repo to delete in %r", rm_path)

kallithea/model/scm.py

@@ -1,736 +1,733 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.model.scm
 ~~~~~~~~~~~~~~~~~~~
 
 Scm model for Kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 9, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 import logging
 import os
 import posixpath
 import re
 import sys
 import tempfile
 import traceback
 
 import pkg_resources
 from tg.i18n import ugettext as _
 
 import kallithea
 from kallithea.lib import hooks
 from kallithea.lib.auth import HasPermissionAny, HasRepoGroupPermissionLevel, HasRepoPermissionLevel, HasUserGroupPermissionLevel
 from kallithea.lib.exceptions import IMCCommitError, NonRelativePathError
 from kallithea.lib.utils import get_filesystem_repos, make_ui
 from kallithea.lib.utils2 import safe_bytes, safe_str, set_hook_environment, umask
 from kallithea.lib.vcs import get_repo
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.lib.vcs.exceptions import RepositoryError, VCSError
 from kallithea.lib.vcs.nodes import FileNode
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 from kallithea.model import db, meta, userlog
 
 
 log = logging.getLogger(__name__)
 
 
 class UserTemp(object):
     def __init__(self, user_id):
         self.user_id = user_id
 
     def __repr__(self):
         return "<%s('id:%s')>" % (self.__class__.__name__, self.user_id)
 
 
 class RepoTemp(object):
     def __init__(self, repo_id):
         self.repo_id = repo_id
 
     def __repr__(self):
         return "<%s('id:%s')>" % (self.__class__.__name__, self.repo_id)
 
 
 class _PermCheckIterator(object):
     def __init__(self, obj_list, obj_attr, perm_set, perm_checker, extra_kwargs=None):
         """
         Creates iterator from given list of objects, additionally
         checking permission for them from perm_set var
 
         :param obj_list: list of db objects
         :param obj_attr: attribute of object to pass into perm_checker
         :param perm_set: list of permissions to check
         :param perm_checker: callable to check permissions against
         """
         self.obj_list = obj_list
         self.obj_attr = obj_attr
         self.perm_set = perm_set
         self.perm_checker = perm_checker
         self.extra_kwargs = extra_kwargs or {}
 
     def __len__(self):
         return len(self.obj_list)
 
     def __repr__(self):
         return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
 
     def __iter__(self):
         for db_obj in self.obj_list:
             # check permission at this level
             name = getattr(db_obj, self.obj_attr, None)
             if not self.perm_checker(*self.perm_set)(
                     name, self.__class__.__name__, **self.extra_kwargs):
                 continue
 
             yield db_obj
 
 
 class RepoList(_PermCheckIterator):
 
     def __init__(self, db_repo_list, perm_level, extra_kwargs=None):
         super(RepoList, self).__init__(obj_list=db_repo_list,
                     obj_attr='repo_name', perm_set=[perm_level],
                     perm_checker=HasRepoPermissionLevel,
                     extra_kwargs=extra_kwargs)
 
 
 class RepoGroupList(_PermCheckIterator):
 
     def __init__(self, db_repo_group_list, perm_level, extra_kwargs=None):
         super(RepoGroupList, self).__init__(obj_list=db_repo_group_list,
                     obj_attr='group_name', perm_set=[perm_level],
                     perm_checker=HasRepoGroupPermissionLevel,
                     extra_kwargs=extra_kwargs)
 
 
 class UserGroupList(_PermCheckIterator):
 
     def __init__(self, db_user_group_list, perm_level, extra_kwargs=None):
         super(UserGroupList, self).__init__(obj_list=db_user_group_list,
                     obj_attr='users_group_name', perm_set=[perm_level],
                     perm_checker=HasUserGroupPermissionLevel,
                     extra_kwargs=extra_kwargs)
 
 
 class ScmModel(object):
     """
     Generic Scm Model
     """
 
     def __get_repo(self, instance):
         cls = db.Repository
         if isinstance(instance, cls):
             return instance
         elif isinstance(instance, int):
             return cls.get(instance)
         elif isinstance(instance, str):
             if instance.isdigit():
                 return cls.get(int(instance))
             return cls.get_by_repo_name(instance)
         raise Exception('given object must be int, basestr or Instance'
                         ' of %s got %s' % (type(cls), type(instance)))
 
     @LazyProperty
     def repos_path(self):
         """
         Gets the repositories root path from database
         """
 
         q = db.Ui.query().filter(db.Ui.ui_key == '/').one()
 
         return q.ui_value
 
     def repo_scan(self, repos_path=None):
         """
         Listing of repositories in given path. This path should not be a
         repository itself. Return a dictionary of repository objects mapping to
         vcs instances.
 
         :param repos_path: path to directory containing repositories
         """
 
         if repos_path is None:
             repos_path = self.repos_path
 
         log.info('scanning for repositories in %s', repos_path)
 
         baseui = make_ui()
         repos = {}
 
         for name, path in get_filesystem_repos(repos_path):
             # name need to be decomposed and put back together using the /
             # since this is internal storage separator for kallithea
             name = db.Repository.normalize_repo_name(name)
 
             try:
                 if name in repos:
                     raise RepositoryError('Duplicate repository name %s '
                                           'found in %s' % (name, path))
                 else:
                     repos[name] = get_repo(path[1], baseui=baseui)
             except (OSError, VCSError):
                 continue
         log.debug('found %s paths with repositories', len(repos))
         return repos
 
     def get_repos(self, repos):
         """Return the repos the user has access to"""
         return RepoList(repos, perm_level='read')
 
     def get_repo_groups(self, groups=None):
         """Return the repo groups the user has access to
         If no groups are specified, use top level groups.
         """
         if groups is None:
             groups = db.RepoGroup.query() \
                 .filter(db.RepoGroup.parent_group_id == None).all()
         return RepoGroupList(groups, perm_level='read')
 
     def mark_for_invalidation(self, repo_name):
         """
         Mark caches of this repo invalid in the database.
 
         :param repo_name: the repo for which caches should be marked invalid
         """
         log.debug("Marking %s as invalidated and update cache", repo_name)
         repo = db.Repository.get_by_repo_name(repo_name)
         if repo is not None:
             repo.set_invalidate()
             repo.update_changeset_cache()
 
     def toggle_following_repo(self, follow_repo_id, user_id):
 
         f = db.UserFollowing.query() \
             .filter(db.UserFollowing.follows_repository_id == follow_repo_id) \
             .filter(db.UserFollowing.user_id == user_id).scalar()
 
         if f is not None:
             try:
                 meta.Session().delete(f)
                 userlog.action_logger(UserTemp(user_id),
                               'stopped_following_repo',
                               RepoTemp(follow_repo_id))
                 return
             except Exception:
                 log.error(traceback.format_exc())
                 raise
 
         try:
             f = db.UserFollowing()
             f.user_id = user_id
             f.follows_repository_id = follow_repo_id
             meta.Session().add(f)
 
             userlog.action_logger(UserTemp(user_id),
                           'started_following_repo',
                           RepoTemp(follow_repo_id))
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def toggle_following_user(self, follow_user_id, user_id):
         f = db.UserFollowing.query() \
             .filter(db.UserFollowing.follows_user_id == follow_user_id) \
             .filter(db.UserFollowing.user_id == user_id).scalar()
 
         if f is not None:
             try:
                 meta.Session().delete(f)
                 return
             except Exception:
                 log.error(traceback.format_exc())
                 raise
 
         try:
             f = db.UserFollowing()
             f.user_id = user_id
             f.follows_user_id = follow_user_id
             meta.Session().add(f)
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def is_following_repo(self, repo_name, user_id):
         r = db.Repository.query() \
             .filter(db.Repository.repo_name == repo_name).scalar()
 
         f = db.UserFollowing.query() \
             .filter(db.UserFollowing.follows_repository == r) \
             .filter(db.UserFollowing.user_id == user_id).scalar()
 
         return f is not None
 
     def is_following_user(self, username, user_id):
         u = db.User.get_by_username(username)
 
         f = db.UserFollowing.query() \
             .filter(db.UserFollowing.follows_user == u) \
             .filter(db.UserFollowing.user_id == user_id).scalar()
 
         return f is not None
 
     def get_followers(self, repo):
         repo = db.Repository.guess_instance(repo)
 
         return db.UserFollowing.query() \
                 .filter(db.UserFollowing.follows_repository == repo).count()
 
     def get_forks(self, repo):
         repo = db.Repository.guess_instance(repo)
         return db.Repository.query() \
                 .filter(db.Repository.fork == repo).count()
 
     def get_pull_requests(self, repo):
         repo = db.Repository.guess_instance(repo)
         return db.PullRequest.query() \
                 .filter(db.PullRequest.other_repo == repo) \
                 .filter(db.PullRequest.status != db.PullRequest.STATUS_CLOSED).count()
 
     def mark_as_fork(self, repo, fork, user):
         repo = self.__get_repo(repo)
         fork = self.__get_repo(fork)
         if fork and repo.repo_id == fork.repo_id:
             raise Exception("Cannot set repository as fork of itself")
 
         if fork and repo.repo_type != fork.repo_type:
             raise RepositoryError("Cannot set repository as fork of repository with other type")
 
         repo.fork = fork
         return repo
 
     def _handle_push(self, repo, username, ip_addr, action, repo_name, revisions):
         """
         Handle that the repository has changed.
         Adds an action log entry with the new revisions, and the head revision
         cache and in-memory caches are invalidated/updated.
 
         :param username: username who pushes
         :param action: push/push_local/push_remote
         :param repo_name: name of repo
         :param revisions: list of revisions that we pushed
         """
         set_hook_environment(username, ip_addr, repo_name, repo_alias=repo.alias, action=action)
         hooks.process_pushed_raw_ids(revisions) # also calls mark_for_invalidation
 
     def pull_changes(self, repo, username, ip_addr, clone_uri=None):
         """
         Pull from "clone URL" or fork origin.
         """
         dbrepo = self.__get_repo(repo)
         if clone_uri is None:
             clone_uri = dbrepo.clone_uri or dbrepo.fork and dbrepo.fork.repo_full_path
         if not clone_uri:
             raise Exception("This repository doesn't have a clone uri")
 
         repo = dbrepo.scm_instance
         repo_name = dbrepo.repo_name
         try:
             if repo.alias == 'git':
                 repo.fetch(clone_uri)
                 # git doesn't really have something like post-fetch action
                 # we fake that now.
                 # TODO: extract fetched revisions ... somehow ...
                 self._handle_push(repo,
                                   username=username,
                                   ip_addr=ip_addr,
                                   action='push_remote',
                                   repo_name=repo_name,
                                   revisions=[])
             else:
                 set_hook_environment(username, ip_addr, dbrepo.repo_name,
                                            repo.alias, action='push_remote')
                 repo.pull(clone_uri)
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def commit_change(self, repo, repo_name, cs, user, ip_addr, author, message,
                       content, f_path):
         """
         Commit a change to a single file
 
         :param repo: a db_repo.scm_instance
         """
         user = db.User.guess_instance(user)
         imc = repo.in_memory_changeset
         imc.change(FileNode(f_path, content, mode=cs.get_file_mode(f_path)))
         try:
             tip = imc.commit(message=message, author=author,
                              parents=[cs], branch=cs.branch)
         except Exception as e:
             log.error(traceback.format_exc())
             # clear caches - we also want a fresh object if commit fails
             self.mark_for_invalidation(repo_name)
             raise IMCCommitError(str(e))
         self._handle_push(repo,
                           username=user.username,
                           ip_addr=ip_addr,
                           action='push_local',
                           repo_name=repo_name,
                           revisions=[tip.raw_id])
         return tip
 
     def _sanitize_path(self, f_path):
         if f_path.startswith('/') or f_path.startswith('.') or '../' in f_path:
             raise NonRelativePathError('%s is not an relative path' % f_path)
         if f_path:
             f_path = posixpath.normpath(f_path)
         return f_path
 
     def get_nodes(self, repo_name, revision, root_path='/', flat=True):
         """
         Recursively walk root dir and return a set of all paths found.
 
         :param repo_name: name of repository
         :param revision: revision for which to list nodes
         :param root_path: root path to list
         :param flat: return as a list, if False returns a dict with description
 
         """
         _files = list()
         _dirs = list()
         try:
             _repo = self.__get_repo(repo_name)
             changeset = _repo.scm_instance.get_changeset(revision)
             root_path = root_path.lstrip('/')
             for topnode, dirs, files in changeset.walk(root_path):
                 for f in files:
                     _files.append(f.path if flat else {"name": f.path,
                                                        "type": "file"})
                 for d in dirs:
                     _dirs.append(d.path if flat else {"name": d.path,
                                                       "type": "dir"})
         except RepositoryError:
             log.debug(traceback.format_exc())
             raise
 
         return _dirs, _files
 
     def create_nodes(self, user, ip_addr, repo, message, nodes, parent_cs=None,
                      author=None, trigger_push_hook=True):
         """
         Commits specified nodes to repo.
 
         :param user: Kallithea User object or user_id, the committer
         :param repo: Kallithea Repository object
         :param message: commit message
         :param nodes: mapping {filename:{'content':content},...}
         :param parent_cs: parent changeset, can be empty than it's initial commit
         :param author: author of commit, cna be different that committer only for git
         :param trigger_push_hook: trigger push hooks
 
         :returns: new committed changeset
         """
 
         user = db.User.guess_instance(user)
         scm_instance = repo.scm_instance_no_cache()
 
         processed_nodes = []
         for f_path in nodes:
             content = nodes[f_path]['content']
             f_path = self._sanitize_path(f_path)
             if not isinstance(content, str) and not isinstance(content, bytes):
                 content = content.read()
             processed_nodes.append((f_path, content))
 
-        message = message
         committer = user.full_contact
         if not author:
             author = committer
 
         if not parent_cs:
             parent_cs = EmptyChangeset(alias=scm_instance.alias)
 
         if isinstance(parent_cs, EmptyChangeset):
             # EmptyChangeset means we we're editing empty repository
             parents = None
         else:
             parents = [parent_cs]
         # add multiple nodes
         imc = scm_instance.in_memory_changeset
         for path, content in processed_nodes:
             imc.add(FileNode(path, content=content))
 
         tip = imc.commit(message=message,
                          author=author,
                          parents=parents,
                          branch=parent_cs.branch)
 
         if trigger_push_hook:
             self._handle_push(scm_instance,
                               username=user.username,
                               ip_addr=ip_addr,
                               action='push_local',
                               repo_name=repo.repo_name,
                               revisions=[tip.raw_id])
         else:
             self.mark_for_invalidation(repo.repo_name)
         return tip
 
     def update_nodes(self, user, ip_addr, repo, message, nodes, parent_cs=None,
                      author=None, trigger_push_hook=True):
         """
         Commits specified nodes to repo. Again.
         """
         user = db.User.guess_instance(user)
         scm_instance = repo.scm_instance_no_cache()
 
-        message = message
         committer = user.full_contact
         if not author:
             author = committer
 
         if not parent_cs:
             parent_cs = EmptyChangeset(alias=scm_instance.alias)
 
         if isinstance(parent_cs, EmptyChangeset):
             # EmptyChangeset means we we're editing empty repository
             parents = None
         else:
             parents = [parent_cs]
 
         # add multiple nodes
         imc = scm_instance.in_memory_changeset
         for _filename, data in nodes.items():
             # new filename, can be renamed from the old one
             filename = self._sanitize_path(data['filename'])
             old_filename = self._sanitize_path(_filename)
             content = data['content']
 
             filenode = FileNode(old_filename, content=content)
             op = data['op']
             if op == 'add':
                 imc.add(filenode)
             elif op == 'del':
                 imc.remove(filenode)
             elif op == 'mod':
                 if filename != old_filename:
                     # TODO: handle renames, needs vcs lib changes
                     imc.remove(filenode)
                     imc.add(FileNode(filename, content=content))
                 else:
                     imc.change(filenode)
 
         # commit changes
         tip = imc.commit(message=message,
                          author=author,
                          parents=parents,
                          branch=parent_cs.branch)
 
         if trigger_push_hook:
             self._handle_push(scm_instance,
                               username=user.username,
                               ip_addr=ip_addr,
                               action='push_local',
                               repo_name=repo.repo_name,
                               revisions=[tip.raw_id])
         else:
             self.mark_for_invalidation(repo.repo_name)
 
     def delete_nodes(self, user, ip_addr, repo, message, nodes, parent_cs=None,
                      author=None, trigger_push_hook=True):
         """
         Deletes specified nodes from repo.
 
         :param user: Kallithea User object or user_id, the committer
         :param repo: Kallithea Repository object
         :param message: commit message
         :param nodes: mapping {filename:{'content':content},...}
         :param parent_cs: parent changeset, can be empty than it's initial commit
         :param author: author of commit, cna be different that committer only for git
         :param trigger_push_hook: trigger push hooks
 
         :returns: new committed changeset after deletion
         """
 
         user = db.User.guess_instance(user)
         scm_instance = repo.scm_instance_no_cache()
 
         processed_nodes = []
         for f_path in nodes:
             f_path = self._sanitize_path(f_path)
             # content can be empty but for compatibility it allows same dicts
             # structure as add_nodes
             content = nodes[f_path].get('content')
             processed_nodes.append((f_path, content))
 
-        message = message
         committer = user.full_contact
         if not author:
             author = committer
 
         if not parent_cs:
             parent_cs = EmptyChangeset(alias=scm_instance.alias)
 
         if isinstance(parent_cs, EmptyChangeset):
             # EmptyChangeset means we we're editing empty repository
             parents = None
         else:
             parents = [parent_cs]
         # add multiple nodes
         imc = scm_instance.in_memory_changeset
         for path, content in processed_nodes:
             imc.remove(FileNode(path, content=content))
 
         tip = imc.commit(message=message,
                          author=author,
                          parents=parents,
                          branch=parent_cs.branch)
 
         if trigger_push_hook:
             self._handle_push(scm_instance,
                               username=user.username,
                               ip_addr=ip_addr,
                               action='push_local',
                               repo_name=repo.repo_name,
                               revisions=[tip.raw_id])
         else:
             self.mark_for_invalidation(repo.repo_name)
         return tip
 
     def get_unread_journal(self):
         return db.UserLog.query().count()
 
     def get_repo_landing_revs(self, repo=None):
         """
         Generates select option with tags branches and bookmarks (for hg only)
         grouped by type
 
         :param repo:
         """
 
         hist_l = []
         choices = []
         hist_l.append(('rev:tip', _('latest tip')))
         choices.append('rev:tip')
         if repo is None:
             return choices, hist_l
 
         repo = self.__get_repo(repo)
         repo = repo.scm_instance
 
         branches_group = ([('branch:%s' % k, k) for k, v in
                            repo.branches.items()], _("Branches"))
         hist_l.append(branches_group)
         choices.extend([x[0] for x in branches_group[0]])
 
         if repo.alias == 'hg':
             bookmarks_group = ([('book:%s' % k, k) for k, v in
                                 repo.bookmarks.items()], _("Bookmarks"))
             hist_l.append(bookmarks_group)
             choices.extend([x[0] for x in bookmarks_group[0]])
 
         tags_group = ([('tag:%s' % k, k) for k, v in
                        repo.tags.items()], _("Tags"))
         hist_l.append(tags_group)
         choices.extend([x[0] for x in tags_group[0]])
 
         return choices, hist_l
 
     def _get_git_hook_interpreter(self):
         """Return a suitable interpreter for Git hooks.
 
         Return a suitable string to be written in the POSIX #! shebang line for
         Git hook scripts so they invoke Kallithea code with the right Python
         interpreter and in the right environment.
         """
         # Note: sys.executable might not point at a usable Python interpreter. For
         # example, when using uwsgi, it will point at the uwsgi program itself.
         # FIXME This may not work on Windows and may need a shell wrapper script.
         return (kallithea.CONFIG.get('git_hook_interpreter')
                 or sys.executable
                 or '/usr/bin/env python3')
 
     def install_git_hooks(self, repo, force=False):
         """
         Creates a kallithea hook inside a git repository
 
         :param repo: Instance of VCS repo
         :param force: Overwrite existing non-Kallithea hooks
         """
 
         hooks_path = os.path.join(repo.path, 'hooks')
         if not repo.bare:
             hooks_path = os.path.join(repo.path, '.git', 'hooks')
         if not os.path.isdir(hooks_path):
             os.makedirs(hooks_path)
 
         tmpl_post = b"#!%s\n" % safe_bytes(self._get_git_hook_interpreter())
         tmpl_post += pkg_resources.resource_string(
             'kallithea', os.path.join('templates', 'py', 'git_post_receive_hook.py')
         )
         tmpl_pre = b"#!%s\n" % safe_bytes(self._get_git_hook_interpreter())
         tmpl_pre += pkg_resources.resource_string(
             'kallithea', os.path.join('templates', 'py', 'git_pre_receive_hook.py')
         )
 
         for h_type, tmpl in [('pre-receive', tmpl_pre), ('post-receive', tmpl_post)]:
             hook_file = os.path.join(hooks_path, h_type)
             other_hook = False
             log.debug('Installing git hook in repo %s', repo)
             if os.path.islink(hook_file):
                 log.debug("Found symlink hook at %s", hook_file)
                 other_hook = True
             elif os.path.isfile(hook_file):
                 log.debug('hook exists, checking if it is from kallithea')
                 with open(hook_file, 'rb') as f:
                     data = f.read()
                     matches = re.search(br'^KALLITHEA_HOOK_VER\s*=\s*(.*)$', data, flags=re.MULTILINE)
                     if matches:
                         ver = safe_str(matches.group(1))
                         log.debug('Found Kallithea hook - it has KALLITHEA_HOOK_VER %s', ver)
                     else:
                         log.debug('Found non-Kallithea hook at %s', hook_file)
                         other_hook = True
             elif os.path.exists(hook_file):
                 log.debug("Found hook that isn't a regular file at %s", hook_file)
                 other_hook = True
             if other_hook and not force:
                 log.warning('skipping overwriting hook file %s', hook_file)
             else:
                 log.debug('writing %s hook file !', h_type)
                 try:
                     fh, fn = tempfile.mkstemp(prefix=hook_file + '.tmp.')
                     os.write(fh, tmpl.replace(b'_TMPL_', safe_bytes(kallithea.__version__)))
                     os.close(fh)
                     os.chmod(fn, 0o777 & ~umask)
                     os.rename(fn, hook_file)
                 except (OSError, IOError) as e:
                     log.error('error writing hook %s: %s', hook_file, e)
 
 
 def AvailableRepoGroupChoices(repo_group_perm_level, extras=()):
     """Return group_id,string tuples with choices for all the repo groups where
     the user has the necessary permissions.
 
     Top level is -1.
     """
     groups = db.RepoGroup.query().all()
     if HasPermissionAny('hg.admin')('available repo groups'):
         groups.append(None)
     else:
         groups = list(RepoGroupList(groups, perm_level=repo_group_perm_level))
         if HasPermissionAny('hg.create.repository')('available repo groups'):
             groups.append(None)
         for extra in extras:
             if not any(rg == extra for rg in groups):
                 groups.append(extra)
     return db.RepoGroup.groups_choices(groups=groups)

kallithea/tests/scripts/manual_test_crawler.py

@@ -1,191 +1,190 b''
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.tests.scripts.manual_test_crawler
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Test for crawling a project for memory usage
 This should be runned just as regular script together
 with a watch script that will show memory usage.
 
 watch -n1 ./kallithea/tests/mem_watch
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 21, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 import http.cookiejar
 import os
 import sys
 import tempfile
 import time
 import urllib.parse
 import urllib.request
 from os.path import dirname
 
 from kallithea.lib import vcs
 from kallithea.lib.compat import OrderedSet
 from kallithea.lib.vcs.exceptions import RepositoryError
 
 
 __here__ = os.path.abspath(__file__)
 __root__ = dirname(dirname(dirname(__here__)))
 sys.path.append(__root__)
 
 
 PASES = 3
 HOST = 'http://127.0.0.1'
 PORT = 5000
 BASE_URI = '%s:%s/' % (HOST, PORT)
 
 if len(sys.argv) == 2:
     BASE_URI = sys.argv[1]
 
 if not BASE_URI.endswith('/'):
     BASE_URI += '/'
 
 print('Crawling @ %s' % BASE_URI)
 BASE_URI += '%s'
 PROJECT_PATH = os.path.join('/', 'home', 'username', 'repos')
 PROJECTS = [
     # 'linux-magx-pbranch',
     'CPython',
     'kallithea',
 ]
 
 
 cj = http.cookiejar.FileCookieJar(os.path.join(tempfile.gettempdir(), 'test_cookie.txt'))
 o = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj))
 o.addheaders = [
     ('User-agent', 'kallithea-crawler'),
     ('Accept-Language', 'en - us, en;q = 0.5')
 ]
 
 urllib.request.install_opener(o)
 
 
 def _get_repo(proj):
     if isinstance(proj, str):
         repo = vcs.get_repo(os.path.join(PROJECT_PATH, proj))
-        proj = proj
     else:
         repo = proj
         proj = repo.name
 
     return repo, proj
 
 
 def test_changelog_walk(proj, pages=100):
     repo, proj = _get_repo(proj)
 
     total_time = 0
     for i in range(1, pages):
 
         page = '/'.join((proj, 'changelog',))
 
         full_uri = (BASE_URI % page) + '?' + urllib.parse.urlencode({'page': i})
         s = time.time()
         f = o.open(full_uri)
 
         assert f.url == full_uri, 'URL:%s does not match %s' % (f.url, full_uri)
 
         size = len(f.read())
         e = time.time() - s
         total_time += e
         print('visited %s size:%s req:%s ms' % (full_uri, size, e))
 
     print('total_time', total_time)
     print('average on req', total_time / float(pages))
 
 
 def test_changeset_walk(proj, limit=None):
     repo, proj = _get_repo(proj)
 
     print('processing', os.path.join(PROJECT_PATH, proj))
     total_time = 0
 
     cnt = 0
     for i in repo:
         cnt += 1
         raw_cs = '/'.join((proj, 'changeset', i.raw_id))
         if limit and limit == cnt:
             break
 
         full_uri = (BASE_URI % raw_cs)
         print('%s visiting %s/%s' % (cnt, full_uri, i))
         s = time.time()
         f = o.open(full_uri)
         size = len(f.read())
         e = time.time() - s
         total_time += e
         print('%s visited %s/%s size:%s req:%s ms' % (cnt, full_uri, i, size, e))
 
     print('total_time', total_time)
     print('average on req', total_time / float(cnt))
 
 
 def test_files_walk(proj, limit=100):
     repo, proj = _get_repo(proj)
 
     print('processing', os.path.join(PROJECT_PATH, proj))
     total_time = 0
 
     paths_ = OrderedSet([''])
     try:
         tip = repo.get_changeset('tip')
         for topnode, dirs, files in tip.walk('/'):
 
             for dir in dirs:
                 paths_.add(dir.path)
                 for f in dir:
                     paths_.add(f.path)
 
             for f in files:
                 paths_.add(f.path)
 
     except RepositoryError as e:
         pass
 
     cnt = 0
     for f in paths_:
         cnt += 1
         if limit and limit == cnt:
             break
 
         file_path = '/'.join((proj, 'files', 'tip', f))
         full_uri = (BASE_URI % file_path)
         print('%s visiting %s' % (cnt, full_uri))
         s = time.time()
         f = o.open(full_uri)
         size = len(f.read())
         e = time.time() - s
         total_time += e
         print('%s visited OK size:%s req:%s ms' % (cnt, size, e))
 
     print('total_time', total_time)
     print('average on req', total_time / float(cnt))
 
 if __name__ == '__main__':
     for path in PROJECTS:
         repo = vcs.get_repo(os.path.join(PROJECT_PATH, path))
         for i in range(PASES):
             print('PASS %s/%s' % (i, PASES))
             test_changelog_walk(repo, pages=80)
             test_changeset_walk(repo, limit=100)
             test_files_walk(repo, limit=100)