hooks: expose refs on push.

- allows easier pull request creation

rhodecode/lib/db_manage.py

@@ -1,619 +1,620 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Database creation, and setup module for RhodeCode Enterprise. Used for creation
 of database as well as for migration operations
 """
 
 import os
 import sys
 import time
 import uuid
 import logging
 import getpass
 from os.path import dirname as dn, join as jn
 
 from sqlalchemy.engine import create_engine
 
 from rhodecode import __dbversion__
 from rhodecode.model import init_model
 from rhodecode.model.user import UserModel
 from rhodecode.model.db import (
     User, Permission, RhodeCodeUi, RhodeCodeSetting, UserToPerm,
     DbMigrateVersion, RepoGroup, UserRepoGroupToPerm, CacheKey, Repository)
 from rhodecode.model.meta import Session, Base
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.settings import SettingsModel
 
 
 log = logging.getLogger(__name__)
 
 
 def notify(msg):
     """
     Notification for migrations messages
     """
     ml = len(msg) + (4 * 2)
     print('\n%s\n*** %s ***\n%s' % ('*' * ml, msg, '*' * ml)).upper()
 
 
 class DbManage(object):
 
     def __init__(self, log_sql, dbconf, root, tests=False,
                  SESSION=None, cli_args={}):
         self.dbname = dbconf.split('/')[-1]
         self.tests = tests
         self.root = root
         self.dburi = dbconf
         self.log_sql = log_sql
         self.db_exists = False
         self.cli_args = cli_args
         self.init_db(SESSION=SESSION)
         self.ask_ok = self.get_ask_ok_func(self.cli_args.get('force_ask'))
 
     def get_ask_ok_func(self, param):
         if param not in [None]:
             # return a function lambda that has a default set to param
             return lambda *args, **kwargs: param
         else:
             from rhodecode.lib.utils import ask_ok
             return ask_ok
 
     def init_db(self, SESSION=None):
         if SESSION:
             self.sa = SESSION
         else:
             # init new sessions
             engine = create_engine(self.dburi, echo=self.log_sql)
             init_model(engine)
             self.sa = Session()
 
     def create_tables(self, override=False):
         """
         Create a auth database
         """
 
         log.info("Existing database with the same name is going to be destroyed.")
         log.info("Setup command will run DROP ALL command on that database.")
         if self.tests:
             destroy = True
         else:
             destroy = self.ask_ok('Are you sure that you want to destroy the old database? [y/n]')
         if not destroy:
             log.info('Nothing done.')
             sys.exit(0)
         if destroy:
             Base.metadata.drop_all()
 
         checkfirst = not override
         Base.metadata.create_all(checkfirst=checkfirst)
         log.info('Created tables for %s' % self.dbname)
 
     def set_db_version(self):
         ver = DbMigrateVersion()
         ver.version = __dbversion__
         ver.repository_id = 'rhodecode_db_migrations'
         ver.repository_path = 'versions'
         self.sa.add(ver)
         log.info('db version set to: %s' % __dbversion__)
 
     def run_pre_migration_tasks(self):
         """
         Run various tasks before actually doing migrations
         """
         # delete cache keys on each upgrade
         total = CacheKey.query().count()
         log.info("Deleting (%s) cache keys now...", total)
         CacheKey.delete_all_cache()
 
     def upgrade(self):
         """
         Upgrades given database schema to given revision following
         all needed steps, to perform the upgrade
 
         """
 
         from rhodecode.lib.dbmigrate.migrate.versioning import api
         from rhodecode.lib.dbmigrate.migrate.exceptions import \
             DatabaseNotControlledError
 
         if 'sqlite' in self.dburi:
             print (
                '********************** WARNING **********************\n'
                'Make sure your version of sqlite is at least 3.7.X.  \n'
                'Earlier versions are known to fail on some migrations\n'
                '*****************************************************\n')
 
         upgrade = self.ask_ok(
             'You are about to perform a database upgrade. Make '
             'sure you have backed up your database. '
             'Continue ? [y/n]')
         if not upgrade:
             log.info('No upgrade performed')
             sys.exit(0)
 
         repository_path = jn(dn(dn(dn(os.path.realpath(__file__)))),
                              'rhodecode/lib/dbmigrate')
         db_uri = self.dburi
 
         try:
             curr_version = api.db_version(db_uri, repository_path)
             msg = ('Found current database under version '
                    'control with version %s' % curr_version)
 
         except (RuntimeError, DatabaseNotControlledError):
             curr_version = 1
             msg = ('Current database is not under version control. Setting '
                    'as version %s' % curr_version)
             api.version_control(db_uri, repository_path, curr_version)
 
         notify(msg)
 
         self.run_pre_migration_tasks()
 
         if curr_version == __dbversion__:
             log.info('This database is already at the newest version')
             sys.exit(0)
 
         upgrade_steps = range(curr_version + 1, __dbversion__ + 1)
         notify('attempting to upgrade database from '
                'version %s to version %s' % (curr_version, __dbversion__))
 
         # CALL THE PROPER ORDER OF STEPS TO PERFORM FULL UPGRADE
         _step = None
         for step in upgrade_steps:
             notify('performing upgrade step %s' % step)
             time.sleep(0.5)
 
             api.upgrade(db_uri, repository_path, step)
             self.sa.rollback()
             notify('schema upgrade for step %s completed' % (step,))
 
             _step = step
 
         notify('upgrade to version %s successful' % _step)
 
     def fix_repo_paths(self):
         """
         Fixes an old RhodeCode version path into new one without a '*'
         """
 
         paths = self.sa.query(RhodeCodeUi)\
                 .filter(RhodeCodeUi.ui_key == '/')\
                 .scalar()
 
         paths.ui_value = paths.ui_value.replace('*', '')
 
         try:
             self.sa.add(paths)
             self.sa.commit()
         except Exception:
             self.sa.rollback()
             raise
 
     def fix_default_user(self):
         """
         Fixes an old default user with some 'nicer' default values,
         used mostly for anonymous access
         """
         def_user = self.sa.query(User)\
                 .filter(User.username == User.DEFAULT_USER)\
                 .one()
 
         def_user.name = 'Anonymous'
         def_user.lastname = 'User'
         def_user.email = User.DEFAULT_USER_EMAIL
 
         try:
             self.sa.add(def_user)
             self.sa.commit()
         except Exception:
             self.sa.rollback()
             raise
 
     def fix_settings(self):
         """
         Fixes rhodecode settings and adds ga_code key for google analytics
         """
 
         hgsettings3 = RhodeCodeSetting('ga_code', '')
 
         try:
             self.sa.add(hgsettings3)
             self.sa.commit()
         except Exception:
             self.sa.rollback()
             raise
 
     def create_admin_and_prompt(self):
 
         # defaults
         defaults = self.cli_args
         username = defaults.get('username')
         password = defaults.get('password')
         email = defaults.get('email')
 
         if username is None:
             username = raw_input('Specify admin username:')
         if password is None:
             password = self._get_admin_password()
             if not password:
                 # second try
                 password = self._get_admin_password()
                 if not password:
                     sys.exit()
         if email is None:
             email = raw_input('Specify admin email:')
         api_key = self.cli_args.get('api_key')
         self.create_user(username, password, email, True,
                          strict_creation_check=False,
                          api_key=api_key)
 
     def _get_admin_password(self):
         password = getpass.getpass('Specify admin password '
                                    '(min 6 chars):')
         confirm = getpass.getpass('Confirm password:')
 
         if password != confirm:
             log.error('passwords mismatch')
             return False
         if len(password) < 6:
             log.error('password is too short - use at least 6 characters')
             return False
 
         return password
 
     def create_test_admin_and_users(self):
         log.info('creating admin and regular test users')
         from rhodecode.tests import TEST_USER_ADMIN_LOGIN, \
             TEST_USER_ADMIN_PASS, TEST_USER_ADMIN_EMAIL, \
             TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS, \
             TEST_USER_REGULAR_EMAIL, TEST_USER_REGULAR2_LOGIN, \
             TEST_USER_REGULAR2_PASS, TEST_USER_REGULAR2_EMAIL
 
         self.create_user(TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS,
                          TEST_USER_ADMIN_EMAIL, True, api_key=True)
 
         self.create_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS,
                          TEST_USER_REGULAR_EMAIL, False, api_key=True)
 
         self.create_user(TEST_USER_REGULAR2_LOGIN, TEST_USER_REGULAR2_PASS,
                          TEST_USER_REGULAR2_EMAIL, False, api_key=True)
 
     def create_ui_settings(self, repo_store_path):
         """
         Creates ui settings, fills out hooks
         and disables dotencode
         """
         settings_model = SettingsModel(sa=self.sa)
         from rhodecode.lib.vcs.backends.hg import largefiles_store
         from rhodecode.lib.vcs.backends.git import lfs_store
 
         # Build HOOKS
         hooks = [
             (RhodeCodeUi.HOOK_REPO_SIZE, 'python:vcsserver.hooks.repo_size'),
 
             # HG
             (RhodeCodeUi.HOOK_PRE_PULL, 'python:vcsserver.hooks.pre_pull'),
             (RhodeCodeUi.HOOK_PULL, 'python:vcsserver.hooks.log_pull_action'),
             (RhodeCodeUi.HOOK_PRE_PUSH, 'python:vcsserver.hooks.pre_push'),
             (RhodeCodeUi.HOOK_PRETX_PUSH, 'python:vcsserver.hooks.pre_push'),
             (RhodeCodeUi.HOOK_PUSH, 'python:vcsserver.hooks.log_push_action'),
+            (RhodeCodeUi.HOOK_PUSH_KEY, 'python:vcsserver.hooks.key_push'),
 
         ]
 
         for key, value in hooks:
             hook_obj = settings_model.get_ui_by_key(key)
             hooks2 = hook_obj if hook_obj else RhodeCodeUi()
             hooks2.ui_section = 'hooks'
             hooks2.ui_key = key
             hooks2.ui_value = value
             self.sa.add(hooks2)
 
         # enable largefiles
         largefiles = RhodeCodeUi()
         largefiles.ui_section = 'extensions'
         largefiles.ui_key = 'largefiles'
         largefiles.ui_value = ''
         self.sa.add(largefiles)
 
         # set default largefiles cache dir, defaults to
         # /repo_store_location/.cache/largefiles
         largefiles = RhodeCodeUi()
         largefiles.ui_section = 'largefiles'
         largefiles.ui_key = 'usercache'
         largefiles.ui_value = largefiles_store(repo_store_path)
 
         self.sa.add(largefiles)
 
         # set default lfs cache dir, defaults to
         # /repo_store_location/.cache/lfs_store
         lfsstore = RhodeCodeUi()
         lfsstore.ui_section = 'vcs_git_lfs'
         lfsstore.ui_key = 'store_location'
         lfsstore.ui_value = lfs_store(repo_store_path)
 
         self.sa.add(lfsstore)
 
         # enable hgsubversion disabled by default
         hgsubversion = RhodeCodeUi()
         hgsubversion.ui_section = 'extensions'
         hgsubversion.ui_key = 'hgsubversion'
         hgsubversion.ui_value = ''
         hgsubversion.ui_active = False
         self.sa.add(hgsubversion)
 
         # enable hgevolve disabled by default
         hgevolve = RhodeCodeUi()
         hgevolve.ui_section = 'extensions'
         hgevolve.ui_key = 'evolve'
         hgevolve.ui_value = ''
         hgevolve.ui_active = False
         self.sa.add(hgevolve)
 
         # enable hggit disabled by default
         hggit = RhodeCodeUi()
         hggit.ui_section = 'extensions'
         hggit.ui_key = 'hggit'
         hggit.ui_value = ''
         hggit.ui_active = False
         self.sa.add(hggit)
 
         # set svn branch defaults
         branches = ["/branches/*", "/trunk"]
         tags = ["/tags/*"]
 
         for branch in branches:
             settings_model.create_ui_section_value(
                 RhodeCodeUi.SVN_BRANCH_ID, branch)
 
         for tag in tags:
             settings_model.create_ui_section_value(RhodeCodeUi.SVN_TAG_ID, tag)
 
     def create_auth_plugin_options(self, skip_existing=False):
         """
         Create default auth plugin settings, and make it active
 
         :param skip_existing:
         """
 
         for k, v, t in [('auth_plugins', 'egg:rhodecode-enterprise-ce#rhodecode', 'list'),
                         ('auth_rhodecode_enabled', 'True', 'bool')]:
             if (skip_existing and
                     SettingsModel().get_setting_by_name(k) is not None):
                 log.debug('Skipping option %s' % k)
                 continue
             setting = RhodeCodeSetting(k, v, t)
             self.sa.add(setting)
 
     def create_default_options(self, skip_existing=False):
         """Creates default settings"""
 
         for k, v, t in [
             ('default_repo_enable_locking',  False, 'bool'),
             ('default_repo_enable_downloads', False, 'bool'),
             ('default_repo_enable_statistics', False, 'bool'),
             ('default_repo_private', False, 'bool'),
             ('default_repo_type', 'hg', 'unicode')]:
 
             if (skip_existing and
                     SettingsModel().get_setting_by_name(k) is not None):
                 log.debug('Skipping option %s' % k)
                 continue
             setting = RhodeCodeSetting(k, v, t)
             self.sa.add(setting)
 
     def fixup_groups(self):
         def_usr = User.get_default_user()
         for g in RepoGroup.query().all():
             g.group_name = g.get_new_name(g.name)
             self.sa.add(g)
             # get default perm
             default = UserRepoGroupToPerm.query()\
                 .filter(UserRepoGroupToPerm.group == g)\
                 .filter(UserRepoGroupToPerm.user == def_usr)\
                 .scalar()
 
             if default is None:
                 log.debug('missing default permission for group %s adding' % g)
                 perm_obj = RepoGroupModel()._create_default_perms(g)
                 self.sa.add(perm_obj)
 
     def reset_permissions(self, username):
         """
         Resets permissions to default state, useful when old systems had
         bad permissions, we must clean them up
 
         :param username:
         """
         default_user = User.get_by_username(username)
         if not default_user:
             return
 
         u2p = UserToPerm.query()\
             .filter(UserToPerm.user == default_user).all()
         fixed = False
         if len(u2p) != len(Permission.DEFAULT_USER_PERMISSIONS):
             for p in u2p:
                 Session().delete(p)
             fixed = True
             self.populate_default_permissions()
         return fixed
 
     def update_repo_info(self):
         RepoModel.update_repoinfo()
 
     def config_prompt(self, test_repo_path='', retries=3):
         defaults = self.cli_args
         _path = defaults.get('repos_location')
         if retries == 3:
             log.info('Setting up repositories config')
 
         if _path is not None:
             path = _path
         elif not self.tests and not test_repo_path:
             path = raw_input(
                  'Enter a valid absolute path to store repositories. '
                  'All repositories in that path will be added automatically:'
             )
         else:
             path = test_repo_path
         path_ok = True
 
         # check proper dir
         if not os.path.isdir(path):
             path_ok = False
             log.error('Given path %s is not a valid directory' % (path,))
 
         elif not os.path.isabs(path):
             path_ok = False
             log.error('Given path %s is not an absolute path' % (path,))
 
         # check if path is at least readable.
         if not os.access(path, os.R_OK):
             path_ok = False
             log.error('Given path %s is not readable' % (path,))
 
         # check write access, warn user about non writeable paths
         elif not os.access(path, os.W_OK) and path_ok:
             log.warning('No write permission to given path %s' % (path,))
 
             q = ('Given path %s is not writeable, do you want to '
                  'continue with read only mode ? [y/n]' % (path,))
             if not self.ask_ok(q):
                 log.error('Canceled by user')
                 sys.exit(-1)
 
         if retries == 0:
             sys.exit('max retries reached')
         if not path_ok:
             retries -= 1
             return self.config_prompt(test_repo_path, retries)
 
         real_path = os.path.normpath(os.path.realpath(path))
 
         if real_path != os.path.normpath(path):
             q = ('Path looks like a symlink, RhodeCode Enterprise will store '
                  'given path as %s ? [y/n]') % (real_path,)
             if not self.ask_ok(q):
                 log.error('Canceled by user')
                 sys.exit(-1)
 
         return real_path
 
     def create_settings(self, path):
 
         self.create_ui_settings(path)
 
         ui_config = [
             ('web', 'push_ssl', 'False'),
             ('web', 'allow_archive', 'gz zip bz2'),
             ('web', 'allow_push', '*'),
             ('web', 'baseurl', '/'),
             ('paths', '/', path),
             ('phases', 'publish', 'True')
         ]
         for section, key, value in ui_config:
             ui_conf = RhodeCodeUi()
             setattr(ui_conf, 'ui_section', section)
             setattr(ui_conf, 'ui_key', key)
             setattr(ui_conf, 'ui_value', value)
             self.sa.add(ui_conf)
 
         # rhodecode app settings
         settings = [
             ('realm', 'RhodeCode', 'unicode'),
             ('title', '', 'unicode'),
             ('pre_code', '', 'unicode'),
             ('post_code', '', 'unicode'),
             ('show_public_icon', True, 'bool'),
             ('show_private_icon', True, 'bool'),
             ('stylify_metatags', False, 'bool'),
             ('dashboard_items', 100, 'int'),
             ('admin_grid_items', 25, 'int'),
             ('show_version', True, 'bool'),
             ('use_gravatar', False, 'bool'),
             ('gravatar_url', User.DEFAULT_GRAVATAR_URL, 'unicode'),
             ('clone_uri_tmpl', Repository.DEFAULT_CLONE_URI, 'unicode'),
             ('support_url', '', 'unicode'),
             ('update_url', RhodeCodeSetting.DEFAULT_UPDATE_URL, 'unicode'),
             ('show_revision_number', True, 'bool'),
             ('show_sha_length', 12, 'int'),
         ]
 
         for key, val, type_ in settings:
             sett = RhodeCodeSetting(key, val, type_)
             self.sa.add(sett)
 
         self.create_auth_plugin_options()
         self.create_default_options()
 
         log.info('created ui config')
 
     def create_user(self, username, password, email='', admin=False,
                     strict_creation_check=True, api_key=None):
         log.info('creating user %s' % username)
         user = UserModel().create_or_update(
             username, password, email, firstname='RhodeCode', lastname='Admin',
             active=True, admin=admin, extern_type="rhodecode",
             strict_creation_check=strict_creation_check)
 
         if api_key:
             log.info('setting a provided api key for the user %s', username)
             from rhodecode.model.auth_token import AuthTokenModel
             AuthTokenModel().create(
                 user=user, description='BUILTIN TOKEN')
 
     def create_default_user(self):
         log.info('creating default user')
         # create default user for handling default permissions.
         user = UserModel().create_or_update(username=User.DEFAULT_USER,
                                             password=str(uuid.uuid1())[:20],
                                             email=User.DEFAULT_USER_EMAIL,
                                             firstname='Anonymous',
                                             lastname='User',
                                             strict_creation_check=False)
         # based on configuration options activate/deactive this user which
         # controlls anonymous access
         if self.cli_args.get('public_access') is False:
             log.info('Public access disabled')
             user.active = False
             Session().add(user)
             Session().commit()
 
     def create_permissions(self):
         """
         Creates all permissions defined in the system
         """
         # module.(access|create|change|delete)_[name]
         # module.(none|read|write|admin)
         log.info('creating permissions')
         PermissionModel(self.sa).create_permissions()
 
     def populate_default_permissions(self):
         """
         Populate default permissions. It will create only the default
         permissions that are missing, and not alter already defined ones
         """
         log.info('creating default user permissions')
         PermissionModel(self.sa).create_default_user_permissions(user=User.DEFAULT_USER)

rhodecode/lib/hooks_base.py

@@ -1,413 +1,425 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2013-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 Set of hooks run by RhodeCode Enterprise
 """
 
 import os
 import collections
 import logging
 
 import rhodecode
 from rhodecode import events
 from rhodecode.lib import helpers as h
 from rhodecode.lib import audit_logger
 from rhodecode.lib.utils2 import safe_str
 from rhodecode.lib.exceptions import HTTPLockedRC, UserCreationError
 from rhodecode.model.db import Repository, User
 
 log = logging.getLogger(__name__)
 
 
 HookResponse = collections.namedtuple('HookResponse', ('status', 'output'))
 
 
 def is_shadow_repo(extras):
     """
     Returns ``True`` if this is an action executed against a shadow repository.
     """
     return extras['is_shadow_repo']
 
 
 def _get_scm_size(alias, root_path):
 
     if not alias.startswith('.'):
         alias += '.'
 
     size_scm, size_root = 0, 0
     for path, unused_dirs, files in os.walk(safe_str(root_path)):
         if path.find(alias) != -1:
             for f in files:
                 try:
                     size_scm += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
         else:
             for f in files:
                 try:
                     size_root += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
 
     size_scm_f = h.format_byte_size_binary(size_scm)
     size_root_f = h.format_byte_size_binary(size_root)
     size_total_f = h.format_byte_size_binary(size_root + size_scm)
 
     return size_scm_f, size_root_f, size_total_f
 
 
 # actual hooks called by Mercurial internally, and GIT by our Python Hooks
 def repo_size(extras):
     """Present size of repository after push."""
     repo = Repository.get_by_repo_name(extras.repository)
     vcs_part = safe_str(u'.%s' % repo.repo_type)
     size_vcs, size_root, size_total = _get_scm_size(vcs_part,
                                                     repo.repo_full_path)
     msg = ('Repository `%s` size summary %s:%s repo:%s total:%s\n'
            % (repo.repo_name, vcs_part, size_vcs, size_root, size_total))
     return HookResponse(0, msg)
 
 
 def pre_push(extras):
     """
     Hook executed before pushing code.
 
     It bans pushing when the repository is locked.
     """
 
     usr = User.get_by_username(extras.username)
     output = ''
     if extras.locked_by[0] and usr.user_id != int(extras.locked_by[0]):
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         # this exception is interpreted in git/hg middlewares and based
         # on that proper return code is server to client
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output = _http_ret.title
         else:
             raise _http_ret
 
     # Propagate to external components. This is done after checking the
     # lock, for consistent behavior.
     if not is_shadow_repo(extras):
         pre_push_extension(repo_store_path=Repository.base_path(), **extras)
         events.trigger(events.RepoPrePushEvent(
             repo_name=extras.repository, extras=extras))
 
     return HookResponse(0, output)
 
 
 def pre_pull(extras):
     """
     Hook executed before pulling the code.
 
     It bans pulling when the repository is locked.
     """
 
     output = ''
     if extras.locked_by[0]:
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         # this exception is interpreted in git/hg middlewares and based
         # on that proper return code is server to client
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output = _http_ret.title
         else:
             raise _http_ret
 
     # Propagate to external components. This is done after checking the
     # lock, for consistent behavior.
     if not is_shadow_repo(extras):
         pre_pull_extension(**extras)
         events.trigger(events.RepoPrePullEvent(
             repo_name=extras.repository, extras=extras))
 
     return HookResponse(0, output)
 
 
 def post_pull(extras):
     """Hook executed after client pulls the code."""
 
     audit_user = audit_logger.UserWrap(
         username=extras.username,
         ip_addr=extras.ip)
     repo = audit_logger.RepoWrap(repo_name=extras.repository)
     audit_logger.store(
         action='user.pull', action_data={
             'user_agent': extras.user_agent},
         user=audit_user, repo=repo, commit=True)
 
     # Propagate to external components.
     if not is_shadow_repo(extras):
         post_pull_extension(**extras)
         events.trigger(events.RepoPullEvent(
             repo_name=extras.repository, extras=extras))
 
     output = ''
     # make lock is a tri state False, True, None. We only make lock on True
     if extras.make_lock is True and not is_shadow_repo(extras):
         user = User.get_by_username(extras.username)
         Repository.lock(Repository.get_by_repo_name(extras.repository),
                         user.user_id,
                         lock_reason=Repository.LOCK_PULL)
         msg = 'Made lock on repo `%s`' % (extras.repository,)
         output += msg
 
     if extras.locked_by[0]:
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output += _http_ret.title
 
     return HookResponse(0, output)
 
 
 def post_push(extras):
     """Hook executed after user pushes to the repository."""
     commit_ids = extras.commit_ids
 
     # log the push call
     audit_user = audit_logger.UserWrap(
         username=extras.username, ip_addr=extras.ip)
     repo = audit_logger.RepoWrap(repo_name=extras.repository)
     audit_logger.store(
         action='user.push', action_data={
             'user_agent': extras.user_agent,
             'commit_ids': commit_ids[:10000]},
         user=audit_user, repo=repo, commit=True)
 
     # Propagate to external components.
     if not is_shadow_repo(extras):
         post_push_extension(
             repo_store_path=Repository.base_path(),
             pushed_revs=commit_ids,
             **extras)
         events.trigger(events.RepoPushEvent(
             repo_name=extras.repository,
             pushed_commit_ids=commit_ids,
             extras=extras))
 
     output = ''
     # make lock is a tri state False, True, None. We only release lock on False
     if extras.make_lock is False and not is_shadow_repo(extras):
         Repository.unlock(Repository.get_by_repo_name(extras.repository))
         msg = 'Released lock on repo `%s`\n' % extras.repository
         output += msg
 
     if extras.locked_by[0]:
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         # TODO: johbo: if not?
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output += _http_ret.title
 
+    if extras.new_refs:
+        tmpl = \
+            extras.server_url + '/' + \
+            extras.repository + \
+            "/pull-request/new?{ref_type}={ref_name}"
+        for branch_name in extras.new_refs['branches']:
+            output += 'RhodeCode: open pull request link: {}\n'.format(
+                tmpl.format(ref_type='branch', ref_name=branch_name))
+
+        for book_name in extras.new_refs['bookmarks']:
+            output += 'RhodeCode: open pull request link: {}\n'.format(
+                tmpl.format(ref_type='bookmark', ref_name=book_name))
+
     output += 'RhodeCode: push completed\n'
-
     return HookResponse(0, output)
 
 
 def _locked_by_explanation(repo_name, user_name, reason):
     message = (
         'Repository `%s` locked by user `%s`. Reason:`%s`'
         % (repo_name, user_name, reason))
     return message
 
 
 def check_allowed_create_user(user_dict, created_by, **kwargs):
     # pre create hooks
     if pre_create_user.is_active():
         allowed, reason = pre_create_user(created_by=created_by, **user_dict)
         if not allowed:
             raise UserCreationError(reason)
 
 
 class ExtensionCallback(object):
     """
     Forwards a given call to rcextensions, sanitizes keyword arguments.
 
     Does check if there is an extension active for that hook. If it is
     there, it will forward all `kwargs_keys` keyword arguments to the
     extension callback.
     """
 
     def __init__(self, hook_name, kwargs_keys):
         self._hook_name = hook_name
         self._kwargs_keys = set(kwargs_keys)
 
     def __call__(self, *args, **kwargs):
         log.debug('Calling extension callback for %s', self._hook_name)
 
         kwargs_to_pass = dict((key, kwargs[key]) for key in self._kwargs_keys)
         # backward compat for removed api_key for old hooks. THis was it works
         # with older rcextensions that require api_key present
         if self._hook_name in ['CREATE_USER_HOOK', 'DELETE_USER_HOOK']:
             kwargs_to_pass['api_key'] = '_DEPRECATED_'
 
         callback = self._get_callback()
         if callback:
             return callback(**kwargs_to_pass)
         else:
             log.debug('extensions callback not found skipping...')
 
     def is_active(self):
         return hasattr(rhodecode.EXTENSIONS, self._hook_name)
 
     def _get_callback(self):
         return getattr(rhodecode.EXTENSIONS, self._hook_name, None)
 
 
 pre_pull_extension = ExtensionCallback(
     hook_name='PRE_PULL_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository'))
 
 
 post_pull_extension = ExtensionCallback(
     hook_name='PULL_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository'))
 
 
 pre_push_extension = ExtensionCallback(
     hook_name='PRE_PUSH_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'repo_store_path', 'commit_ids'))
 
 
 post_push_extension = ExtensionCallback(
     hook_name='PUSH_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'repo_store_path', 'pushed_revs'))
 
 
 pre_create_user = ExtensionCallback(
     hook_name='PRE_CREATE_USER_HOOK',
     kwargs_keys=(
         'username', 'password', 'email', 'firstname', 'lastname', 'active',
         'admin', 'created_by'))
 
 
 log_create_pull_request = ExtensionCallback(
     hook_name='CREATE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 log_merge_pull_request = ExtensionCallback(
     hook_name='MERGE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 log_close_pull_request = ExtensionCallback(
     hook_name='CLOSE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 log_review_pull_request = ExtensionCallback(
     hook_name='REVIEW_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 log_update_pull_request = ExtensionCallback(
     hook_name='UPDATE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 log_create_user = ExtensionCallback(
     hook_name='CREATE_USER_HOOK',
     kwargs_keys=(
         'username', 'full_name_or_username', 'full_contact', 'user_id',
         'name', 'firstname', 'short_contact', 'admin', 'lastname',
         'ip_addresses', 'extern_type', 'extern_name',
         'email', 'api_keys', 'last_login',
         'full_name', 'active', 'password', 'emails',
         'inherit_default_permissions', 'created_by', 'created_on'))
 
 
 log_delete_user = ExtensionCallback(
     hook_name='DELETE_USER_HOOK',
     kwargs_keys=(
         'username', 'full_name_or_username', 'full_contact', 'user_id',
         'name', 'firstname', 'short_contact', 'admin', 'lastname',
         'ip_addresses',
         'email', 'last_login',
         'full_name', 'active', 'password', 'emails',
         'inherit_default_permissions', 'deleted_by'))
 
 
 log_create_repository = ExtensionCallback(
     hook_name='CREATE_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id', 'created_by'))
 
 
 log_delete_repository = ExtensionCallback(
     hook_name='DELETE_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id', 'deleted_by', 'deleted_on'))
 
 
 log_create_repository_group = ExtensionCallback(
     hook_name='CREATE_REPO_GROUP_HOOK',
     kwargs_keys=(
         'group_name', 'group_parent_id', 'group_description',
         'group_id', 'user_id', 'created_by', 'created_on',
         'enable_locking'))

rhodecode/lib/utils.py

@@ -1,1042 +1,1043 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Utilities library for RhodeCode
 """
 
 import datetime
 import decorator
 import json
 import logging
 import os
 import re
 import shutil
 import tempfile
 import traceback
 import tarfile
 import warnings
 import hashlib
 from os.path import join as jn
 
 import paste
 import pkg_resources
 from paste.script.command import Command, BadCommand
 from webhelpers.text import collapse, remove_formatting, strip_tags
 from mako import exceptions
 from pyramid.threadlocal import get_current_registry
 from pyramid.request import Request
 
 from rhodecode.lib.fakemod import create_module
 from rhodecode.lib.vcs.backends.base import Config
 from rhodecode.lib.vcs.exceptions import VCSError
 from rhodecode.lib.vcs.utils.helpers import get_scm, get_scm_backend
 from rhodecode.lib.utils2 import (
     safe_str, safe_unicode, get_current_rhodecode_user, md5)
 from rhodecode.model import meta
 from rhodecode.model.db import (
     Repository, User, RhodeCodeUi, UserLog, RepoGroup, UserGroup)
 from rhodecode.model.meta import Session
 
 
 log = logging.getLogger(__name__)
 
 REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}__.*')
 
 # String which contains characters that are not allowed in slug names for
 # repositories or repository groups. It is properly escaped to use it in
 # regular expressions.
 SLUG_BAD_CHARS = re.escape('`?=[]\;\'"<>,/~!@#$%^&*()+{}|:')
 
 # Regex that matches forbidden characters in repo/group slugs.
 SLUG_BAD_CHAR_RE = re.compile('[{}]'.format(SLUG_BAD_CHARS))
 
 # Regex that matches allowed characters in repo/group slugs.
 SLUG_GOOD_CHAR_RE = re.compile('[^{}]'.format(SLUG_BAD_CHARS))
 
 # Regex that matches whole repo/group slugs.
 SLUG_RE = re.compile('[^{}]+'.format(SLUG_BAD_CHARS))
 
 _license_cache = None
 
 
 def repo_name_slug(value):
     """
     Return slug of name of repository
     This function is called on each creation/modification
     of repository to prevent bad names in repo
     """
     replacement_char = '-'
 
     slug = remove_formatting(value)
     slug = SLUG_BAD_CHAR_RE.sub('', slug)
     slug = re.sub('[\s]+', '-', slug)
     slug = collapse(slug, replacement_char)
     return slug
 
 
 #==============================================================================
 # PERM DECORATOR HELPERS FOR EXTRACTING NAMES FOR PERM CHECKS
 #==============================================================================
 def get_repo_slug(request):
     if isinstance(request, Request) and getattr(request, 'matchdict', None):
         # pyramid
         _repo = request.matchdict.get('repo_name')
     else:
         _repo = request.environ['pylons.routes_dict'].get('repo_name')
 
     if _repo:
         _repo = _repo.rstrip('/')
     return _repo
 
 
 def get_repo_group_slug(request):
     if isinstance(request, Request) and getattr(request, 'matchdict', None):
         # pyramid
         _group = request.matchdict.get('group_name')
     else:
         _group = request.environ['pylons.routes_dict'].get('group_name')
 
     if _group:
         _group = _group.rstrip('/')
     return _group
 
 
 def get_user_group_slug(request):
     if isinstance(request, Request) and getattr(request, 'matchdict', None):
         # pyramid
         _group = request.matchdict.get('user_group_id')
     else:
         _group = request.environ['pylons.routes_dict'].get('user_group_id')
 
     try:
         _group = UserGroup.get(_group)
         if _group:
             _group = _group.users_group_name
     except Exception:
         log.debug(traceback.format_exc())
         # catch all failures here
         pass
 
     return _group
 
 
 def action_logger(user, action, repo, ipaddr='', sa=None, commit=False):
     """
     Action logger for various actions made by users
 
     :param user: user that made this action, can be a unique username string or
         object containing user_id attribute
     :param action: action to log, should be on of predefined unique actions for
         easy translations
     :param repo: string name of repository or object containing repo_id,
         that action was made on
     :param ipaddr: optional ip address from what the action was made
     :param sa: optional sqlalchemy session
 
     """
 
     if not sa:
         sa = meta.Session()
     # if we don't get explicit IP address try to get one from registered user
     # in tmpl context var
     if not ipaddr:
         ipaddr = getattr(get_current_rhodecode_user(), 'ip_addr', '')
 
     try:
         if getattr(user, 'user_id', None):
             user_obj = User.get(user.user_id)
         elif isinstance(user, basestring):
             user_obj = User.get_by_username(user)
         else:
             raise Exception('You have to provide a user object or a username')
 
         if getattr(repo, 'repo_id', None):
             repo_obj = Repository.get(repo.repo_id)
             repo_name = repo_obj.repo_name
         elif isinstance(repo, basestring):
             repo_name = repo.lstrip('/')
             repo_obj = Repository.get_by_repo_name(repo_name)
         else:
             repo_obj = None
             repo_name = ''
 
         user_log = UserLog()
         user_log.user_id = user_obj.user_id
         user_log.username = user_obj.username
         action = safe_unicode(action)
         user_log.action = action[:1200000]
 
         user_log.repository = repo_obj
         user_log.repository_name = repo_name
 
         user_log.action_date = datetime.datetime.now()
         user_log.user_ip = ipaddr
         sa.add(user_log)
 
         log.info('Logging action:`%s` on repo:`%s` by user:%s ip:%s',
                  action, safe_unicode(repo), user_obj, ipaddr)
         if commit:
             sa.commit()
     except Exception:
         log.error(traceback.format_exc())
         raise
 
 
 def get_filesystem_repos(path, recursive=False, skip_removed_repos=True):
     """
     Scans given path for repos and return (name,(type,path)) tuple
 
     :param path: path to scan for repositories
     :param recursive: recursive search and return names with subdirs in front
     """
 
     # remove ending slash for better results
     path = path.rstrip(os.sep)
     log.debug('now scanning in %s location recursive:%s...', path, recursive)
 
     def _get_repos(p):
         dirpaths = _get_dirpaths(p)
         if not _is_dir_writable(p):
             log.warning('repo path without write access: %s', p)
 
         for dirpath in dirpaths:
             if os.path.isfile(os.path.join(p, dirpath)):
                 continue
             cur_path = os.path.join(p, dirpath)
 
             # skip removed repos
             if skip_removed_repos and REMOVED_REPO_PAT.match(dirpath):
                 continue
 
             #skip .<somethin> dirs
             if dirpath.startswith('.'):
                 continue
 
             try:
                 scm_info = get_scm(cur_path)
                 yield scm_info[1].split(path, 1)[-1].lstrip(os.sep), scm_info
             except VCSError:
                 if not recursive:
                     continue
                 #check if this dir containts other repos for recursive scan
                 rec_path = os.path.join(p, dirpath)
                 if os.path.isdir(rec_path):
                     for inner_scm in _get_repos(rec_path):
                         yield inner_scm
 
     return _get_repos(path)
 
 
 def _get_dirpaths(p):
     try:
         # OS-independable way of checking if we have at least read-only
         # access or not.
         dirpaths = os.listdir(p)
     except OSError:
         log.warning('ignoring repo path without read access: %s', p)
         return []
 
     # os.listpath has a tweak: If a unicode is passed into it, then it tries to
     # decode paths and suddenly returns unicode objects itself. The items it
     # cannot decode are returned as strings and cause issues.
     #
     # Those paths are ignored here until a solid solution for path handling has
     # been built.
     expected_type = type(p)
 
     def _has_correct_type(item):
         if type(item) is not expected_type:
             log.error(
                 u"Ignoring path %s since it cannot be decoded into unicode.",
                 # Using "repr" to make sure that we see the byte value in case
                 # of support.
                 repr(item))
             return False
         return True
 
     dirpaths = [item for item in dirpaths if _has_correct_type(item)]
 
     return dirpaths
 
 
 def _is_dir_writable(path):
     """
     Probe if `path` is writable.
 
     Due to trouble on Cygwin / Windows, this is actually probing if it is
     possible to create a file inside of `path`, stat does not produce reliable
     results in this case.
     """
     try:
         with tempfile.TemporaryFile(dir=path):
             pass
     except OSError:
         return False
     return True
 
 
 def is_valid_repo(repo_name, base_path, expect_scm=None, explicit_scm=None):
     """
     Returns True if given path is a valid repository False otherwise.
     If expect_scm param is given also, compare if given scm is the same
     as expected from scm parameter. If explicit_scm is given don't try to
     detect the scm, just use the given one to check if repo is valid
 
     :param repo_name:
     :param base_path:
     :param expect_scm:
     :param explicit_scm:
 
     :return True: if given path is a valid repository
     """
     full_path = os.path.join(safe_str(base_path), safe_str(repo_name))
     log.debug('Checking if `%s` is a valid path for repository. '
               'Explicit type: %s', repo_name, explicit_scm)
 
     try:
         if explicit_scm:
             detected_scms = [get_scm_backend(explicit_scm)]
         else:
             detected_scms = get_scm(full_path)
 
         if expect_scm:
             return detected_scms[0] == expect_scm
         log.debug('path: %s is an vcs object:%s', full_path, detected_scms)
         return True
     except VCSError:
         log.debug('path: %s is not a valid repo !', full_path)
         return False
 
 
 def is_valid_repo_group(repo_group_name, base_path, skip_path_check=False):
     """
     Returns True if given path is a repository group, False otherwise
 
     :param repo_name:
     :param base_path:
     """
     full_path = os.path.join(safe_str(base_path), safe_str(repo_group_name))
     log.debug('Checking if `%s` is a valid path for repository group',
               repo_group_name)
 
     # check if it's not a repo
     if is_valid_repo(repo_group_name, base_path):
         log.debug('Repo called %s exist, it is not a valid '
                   'repo group' % repo_group_name)
         return False
 
     try:
         # we need to check bare git repos at higher level
         # since we might match branches/hooks/info/objects or possible
         # other things inside bare git repo
         scm_ = get_scm(os.path.dirname(full_path))
         log.debug('path: %s is a vcs object:%s, not valid '
                   'repo group' % (full_path, scm_))
         return False
     except VCSError:
         pass
 
     # check if it's a valid path
     if skip_path_check or os.path.isdir(full_path):
         log.debug('path: %s is a valid repo group !', full_path)
         return True
 
     log.debug('path: %s is not a valid repo group !', full_path)
     return False
 
 
 def ask_ok(prompt, retries=4, complaint='[y]es or [n]o please!'):
     while True:
         ok = raw_input(prompt)
         if ok.lower() in ('y', 'ye', 'yes'):
             return True
         if ok.lower() in ('n', 'no', 'nop', 'nope'):
             return False
         retries = retries - 1
         if retries < 0:
             raise IOError
         print(complaint)
 
 # propagated from mercurial documentation
 ui_sections = [
     'alias', 'auth',
     'decode/encode', 'defaults',
     'diff', 'email',
     'extensions', 'format',
     'merge-patterns', 'merge-tools',
     'hooks', 'http_proxy',
     'smtp', 'patch',
     'paths', 'profiling',
     'server', 'trusted',
     'ui', 'web', ]
 
 
 def config_data_from_db(clear_session=True, repo=None):
     """
     Read the configuration data from the database and return configuration
     tuples.
     """
     from rhodecode.model.settings import VcsSettingsModel
 
     config = []
 
     sa = meta.Session()
     settings_model = VcsSettingsModel(repo=repo, sa=sa)
 
     ui_settings = settings_model.get_ui_settings()
 
     for setting in ui_settings:
         if setting.active:
             log.debug(
                 'settings ui from db: [%s] %s=%s',
                 setting.section, setting.key, setting.value)
             config.append((
                 safe_str(setting.section), safe_str(setting.key),
                 safe_str(setting.value)))
         if setting.key == 'push_ssl':
             # force set push_ssl requirement to False, rhodecode
             # handles that
             config.append((
                 safe_str(setting.section), safe_str(setting.key), False))
     if clear_session:
         meta.Session.remove()
 
     # TODO: mikhail: probably it makes no sense to re-read hooks information.
     # It's already there and activated/deactivated
     skip_entries = []
     enabled_hook_classes = get_enabled_hook_classes(ui_settings)
     if 'pull' not in enabled_hook_classes:
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PULL))
     if 'push' not in enabled_hook_classes:
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PUSH))
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRETX_PUSH))
+        skip_entries.append(('hooks', RhodeCodeUi.HOOK_PUSH_KEY))
 
     config = [entry for entry in config if entry[:2] not in skip_entries]
 
     return config
 
 
 def make_db_config(clear_session=True, repo=None):
     """
     Create a :class:`Config` instance based on the values in the database.
     """
     config = Config()
     config_data = config_data_from_db(clear_session=clear_session, repo=repo)
     for section, option, value in config_data:
         config.set(section, option, value)
     return config
 
 
 def get_enabled_hook_classes(ui_settings):
     """
     Return the enabled hook classes.
 
     :param ui_settings: List of ui_settings as returned
         by :meth:`VcsSettingsModel.get_ui_settings`
 
     :return: a list with the enabled hook classes. The order is not guaranteed.
     :rtype: list
     """
     enabled_hooks = []
     active_hook_keys = [
         key for section, key, value, active in ui_settings
         if section == 'hooks' and active]
 
     hook_names = {
         RhodeCodeUi.HOOK_PUSH: 'push',
         RhodeCodeUi.HOOK_PULL: 'pull',
         RhodeCodeUi.HOOK_REPO_SIZE: 'repo_size'
     }
 
     for key in active_hook_keys:
         hook = hook_names.get(key)
         if hook:
             enabled_hooks.append(hook)
 
     return enabled_hooks
 
 
 def set_rhodecode_config(config):
     """
     Updates pylons config with new settings from database
 
     :param config:
     """
     from rhodecode.model.settings import SettingsModel
     app_settings = SettingsModel().get_all_settings()
 
     for k, v in app_settings.items():
         config[k] = v
 
 
 def get_rhodecode_realm():
     """
     Return the rhodecode realm from database.
     """
     from rhodecode.model.settings import SettingsModel
     realm = SettingsModel().get_setting_by_name('realm')
     return safe_str(realm.app_settings_value)
 
 
 def get_rhodecode_base_path():
     """
     Returns the base path. The base path is the filesystem path which points
     to the repository store.
     """
     from rhodecode.model.settings import SettingsModel
     paths_ui = SettingsModel().get_ui_by_section_and_key('paths', '/')
     return safe_str(paths_ui.ui_value)
 
 
 def map_groups(path):
     """
     Given a full path to a repository, create all nested groups that this
     repo is inside. This function creates parent-child relationships between
     groups and creates default perms for all new groups.
 
     :param paths: full path to repository
     """
     from rhodecode.model.repo_group import RepoGroupModel
     sa = meta.Session()
     groups = path.split(Repository.NAME_SEP)
     parent = None
     group = None
 
     # last element is repo in nested groups structure
     groups = groups[:-1]
     rgm = RepoGroupModel(sa)
     owner = User.get_first_super_admin()
     for lvl, group_name in enumerate(groups):
         group_name = '/'.join(groups[:lvl] + [group_name])
         group = RepoGroup.get_by_group_name(group_name)
         desc = '%s group' % group_name
 
         # skip folders that are now removed repos
         if REMOVED_REPO_PAT.match(group_name):
             break
 
         if group is None:
             log.debug('creating group level: %s group_name: %s',
                        lvl, group_name)
             group = RepoGroup(group_name, parent)
             group.group_description = desc
             group.user = owner
             sa.add(group)
             perm_obj = rgm._create_default_perms(group)
             sa.add(perm_obj)
             sa.flush()
 
         parent = group
     return group
 
 
 def repo2db_mapper(initial_repo_list, remove_obsolete=False):
     """
     maps all repos given in initial_repo_list, non existing repositories
     are created, if remove_obsolete is True it also checks for db entries
     that are not in initial_repo_list and removes them.
 
     :param initial_repo_list: list of repositories found by scanning methods
     :param remove_obsolete: check for obsolete entries in database
     """
     from rhodecode.model.repo import RepoModel
     from rhodecode.model.scm import ScmModel
     from rhodecode.model.repo_group import RepoGroupModel
     from rhodecode.model.settings import SettingsModel
 
     sa = meta.Session()
     repo_model = RepoModel()
     user = User.get_first_super_admin()
     added = []
 
     # creation defaults
     defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
     enable_statistics = defs.get('repo_enable_statistics')
     enable_locking = defs.get('repo_enable_locking')
     enable_downloads = defs.get('repo_enable_downloads')
     private = defs.get('repo_private')
 
     for name, repo in initial_repo_list.items():
         group = map_groups(name)
         unicode_name = safe_unicode(name)
         db_repo = repo_model.get_by_repo_name(unicode_name)
         # found repo that is on filesystem not in RhodeCode database
         if not db_repo:
             log.info('repository %s not found, creating now', name)
             added.append(name)
             desc = (repo.description
                     if repo.description != 'unknown'
                     else '%s repository' % name)
 
             db_repo = repo_model._create_repo(
                 repo_name=name,
                 repo_type=repo.alias,
                 description=desc,
                 repo_group=getattr(group, 'group_id', None),
                 owner=user,
                 enable_locking=enable_locking,
                 enable_downloads=enable_downloads,
                 enable_statistics=enable_statistics,
                 private=private,
                 state=Repository.STATE_CREATED
             )
             sa.commit()
             # we added that repo just now, and make sure we updated server info
             if db_repo.repo_type == 'git':
                 git_repo = db_repo.scm_instance()
                 # update repository server-info
                 log.debug('Running update server info')
                 git_repo._update_server_info()
 
             db_repo.update_commit_cache()
 
         config = db_repo._config
         config.set('extensions', 'largefiles', '')
         ScmModel().install_hooks(
             db_repo.scm_instance(config=config),
             repo_type=db_repo.repo_type)
 
     removed = []
     if remove_obsolete:
         # remove from database those repositories that are not in the filesystem
         for repo in sa.query(Repository).all():
             if repo.repo_name not in initial_repo_list.keys():
                 log.debug("Removing non-existing repository found in db `%s`",
                           repo.repo_name)
                 try:
                     RepoModel(sa).delete(repo, forks='detach', fs_remove=False)
                     sa.commit()
                     removed.append(repo.repo_name)
                 except Exception:
                     # don't hold further removals on error
                     log.error(traceback.format_exc())
                     sa.rollback()
 
         def splitter(full_repo_name):
             _parts = full_repo_name.rsplit(RepoGroup.url_sep(), 1)
             gr_name = None
             if len(_parts) == 2:
                 gr_name = _parts[0]
             return gr_name
 
         initial_repo_group_list = [splitter(x) for x in
                                    initial_repo_list.keys() if splitter(x)]
 
         # remove from database those repository groups that are not in the
         # filesystem due to parent child relationships we need to delete them
         # in a specific order of most nested first
         all_groups = [x.group_name for x in sa.query(RepoGroup).all()]
         nested_sort = lambda gr: len(gr.split('/'))
         for group_name in sorted(all_groups, key=nested_sort, reverse=True):
             if group_name not in initial_repo_group_list:
                 repo_group = RepoGroup.get_by_group_name(group_name)
                 if (repo_group.children.all() or
                     not RepoGroupModel().check_exist_filesystem(
                         group_name=group_name, exc_on_failure=False)):
                     continue
 
                 log.info(
                     'Removing non-existing repository group found in db `%s`',
                     group_name)
                 try:
                     RepoGroupModel(sa).delete(group_name, fs_remove=False)
                     sa.commit()
                     removed.append(group_name)
                 except Exception:
                     # don't hold further removals on error
                     log.exception(
                         'Unable to remove repository group `%s`',
                         group_name)
                     sa.rollback()
                     raise
 
     return added, removed
 
 
 def get_default_cache_settings(settings):
     cache_settings = {}
     for key in settings.keys():
         for prefix in ['beaker.cache.', 'cache.']:
             if key.startswith(prefix):
                 name = key.split(prefix)[1].strip()
                 cache_settings[name] = settings[key].strip()
     return cache_settings
 
 
 # set cache regions for beaker so celery can utilise it
 def add_cache(settings):
     from rhodecode.lib import caches
     cache_settings = {'regions': None}
     # main cache settings used as default ...
     cache_settings.update(get_default_cache_settings(settings))
 
     if cache_settings['regions']:
         for region in cache_settings['regions'].split(','):
             region = region.strip()
             region_settings = {}
             for key, value in cache_settings.items():
                 if key.startswith(region):
                     region_settings[key.split('.')[1]] = value
 
             caches.configure_cache_region(
                 region, region_settings, cache_settings)
 
 
 def load_rcextensions(root_path):
     import rhodecode
     from rhodecode.config import conf
 
     path = os.path.join(root_path, 'rcextensions', '__init__.py')
     if os.path.isfile(path):
         rcext = create_module('rc', path)
         EXT = rhodecode.EXTENSIONS = rcext
         log.debug('Found rcextensions now loading %s...', rcext)
 
         # Additional mappings that are not present in the pygments lexers
         conf.LANGUAGES_EXTENSIONS_MAP.update(getattr(EXT, 'EXTRA_MAPPINGS', {}))
 
         # auto check if the module is not missing any data, set to default if is
         # this will help autoupdate new feature of rcext module
         #from rhodecode.config import rcextensions
         #for k in dir(rcextensions):
         #    if not k.startswith('_') and not hasattr(EXT, k):
         #        setattr(EXT, k, getattr(rcextensions, k))
 
 
 def get_custom_lexer(extension):
     """
     returns a custom lexer if it is defined in rcextensions module, or None
     if there's no custom lexer defined
     """
     import rhodecode
     from pygments import lexers
 
     # custom override made by RhodeCode
     if extension in ['mako']:
         return lexers.get_lexer_by_name('html+mako')
 
     # check if we didn't define this extension as other lexer
     extensions = rhodecode.EXTENSIONS and getattr(rhodecode.EXTENSIONS, 'EXTRA_LEXERS', None)
     if extensions and extension in rhodecode.EXTENSIONS.EXTRA_LEXERS:
         _lexer_name = rhodecode.EXTENSIONS.EXTRA_LEXERS[extension]
         return lexers.get_lexer_by_name(_lexer_name)
 
 
 #==============================================================================
 # TEST FUNCTIONS AND CREATORS
 #==============================================================================
 def create_test_index(repo_location, config):
     """
     Makes default test index.
     """
     import rc_testdata
 
     rc_testdata.extract_search_index(
         'vcs_search_index', os.path.dirname(config['search.location']))
 
 
 def create_test_directory(test_path):
     """
     Create test directory if it doesn't exist.
     """
     if not os.path.isdir(test_path):
         log.debug('Creating testdir %s', test_path)
         os.makedirs(test_path)
 
 
 def create_test_database(test_path, config):
     """
     Makes a fresh database.
     """
     from rhodecode.lib.db_manage import DbManage
 
     # PART ONE create db
     dbconf = config['sqlalchemy.db1.url']
     log.debug('making test db %s', dbconf)
 
     dbmanage = DbManage(log_sql=False, dbconf=dbconf, root=config['here'],
                         tests=True, cli_args={'force_ask': True})
     dbmanage.create_tables(override=True)
     dbmanage.set_db_version()
     # for tests dynamically set new root paths based on generated content
     dbmanage.create_settings(dbmanage.config_prompt(test_path))
     dbmanage.create_default_user()
     dbmanage.create_test_admin_and_users()
     dbmanage.create_permissions()
     dbmanage.populate_default_permissions()
     Session().commit()
 
 
 def create_test_repositories(test_path, config):
     """
     Creates test repositories in the temporary directory. Repositories are
     extracted from archives within the rc_testdata package.
     """
     import rc_testdata
     from rhodecode.tests import HG_REPO, GIT_REPO, SVN_REPO
 
     log.debug('making test vcs repositories')
 
     idx_path = config['search.location']
     data_path = config['cache_dir']
 
     # clean index and data
     if idx_path and os.path.exists(idx_path):
         log.debug('remove %s', idx_path)
         shutil.rmtree(idx_path)
 
     if data_path and os.path.exists(data_path):
         log.debug('remove %s', data_path)
         shutil.rmtree(data_path)
 
     rc_testdata.extract_hg_dump('vcs_test_hg', jn(test_path, HG_REPO))
     rc_testdata.extract_git_dump('vcs_test_git', jn(test_path, GIT_REPO))
 
     # Note: Subversion is in the process of being integrated with the system,
     # until we have a properly packed version of the test svn repository, this
     # tries to copy over the repo from a package "rc_testdata"
     svn_repo_path = rc_testdata.get_svn_repo_archive()
     with tarfile.open(svn_repo_path) as tar:
         tar.extractall(jn(test_path, SVN_REPO))
 
 
 #==============================================================================
 # PASTER COMMANDS
 #==============================================================================
 class BasePasterCommand(Command):
     """
     Abstract Base Class for paster commands.
 
     The celery commands are somewhat aggressive about loading
     celery.conf, and since our module sets the `CELERY_LOADER`
     environment variable to our loader, we have to bootstrap a bit and
     make sure we've had a chance to load the pylons config off of the
     command line, otherwise everything fails.
     """
     min_args = 1
     min_args_error = "Please provide a paster config file as an argument."
     takes_config_file = 1
     requires_config_file = True
 
     def notify_msg(self, msg, log=False):
         """Make a notification to user, additionally if logger is passed
         it logs this action using given logger
 
         :param msg: message that will be printed to user
         :param log: logging instance, to use to additionally log this message
 
         """
         if log and isinstance(log, logging):
             log(msg)
 
     def run(self, args):
         """
         Overrides Command.run
 
         Checks for a config file argument and loads it.
         """
         if len(args) < self.min_args:
             raise BadCommand(
                 self.min_args_error % {'min_args': self.min_args,
                                        'actual_args': len(args)})
 
         # Decrement because we're going to lob off the first argument.
         # @@ This is hacky
         self.min_args -= 1
         self.bootstrap_config(args[0])
         self.update_parser()
         return super(BasePasterCommand, self).run(args[1:])
 
     def update_parser(self):
         """
         Abstract method.  Allows for the class' parser to be updated
         before the superclass' `run` method is called.  Necessary to
         allow options/arguments to be passed through to the underlying
         celery command.
         """
         raise NotImplementedError("Abstract Method.")
 
     def bootstrap_config(self, conf):
         """
         Loads the pylons configuration.
         """
         from pylons import config as pylonsconfig
 
         self.path_to_ini_file = os.path.realpath(conf)
         conf = paste.deploy.appconfig('config:' + self.path_to_ini_file)
         pylonsconfig.init_app(conf.global_conf, conf.local_conf)
 
     def _init_session(self):
         """
         Inits SqlAlchemy Session
         """
         logging.config.fileConfig(self.path_to_ini_file)
         from pylons import config
         from rhodecode.config.utils import initialize_database
 
         # get to remove repos !!
         add_cache(config)
         initialize_database(config)
 
 
 @decorator.decorator
 def jsonify(func, *args, **kwargs):
     """Action decorator that formats output for JSON
 
     Given a function that will return content, this decorator will turn
     the result into JSON, with a content-type of 'application/json' and
     output it.
 
     """
     from pylons.decorators.util import get_pylons
     from rhodecode.lib.ext_json import json
     pylons = get_pylons(args)
     pylons.response.headers['Content-Type'] = 'application/json; charset=utf-8'
     data = func(*args, **kwargs)
     if isinstance(data, (list, tuple)):
         msg = "JSON responses with Array envelopes are susceptible to " \
               "cross-site data leak attacks, see " \
               "http://wiki.pylonshq.com/display/pylonsfaq/Warnings"
         warnings.warn(msg, Warning, 2)
         log.warning(msg)
     log.debug("Returning JSON wrapped action output")
     return json.dumps(data, encoding='utf-8')
 
 
 class PartialRenderer(object):
     """
     Partial renderer used to render chunks of html used in datagrids
     use like::
 
         _render = PartialRenderer('data_table/_dt_elements.mako')
         _render('quick_menu', args, kwargs)
         PartialRenderer.h,
                         c,
                         _,
                         ungettext
         are the template stuff initialized inside and can be re-used later
 
     :param tmpl_name: template path relate to /templates/ dir
     """
 
     def __init__(self, tmpl_name):
         import rhodecode
         from pylons import request, tmpl_context as c
         from pylons.i18n.translation import _, ungettext
         from rhodecode.lib import helpers as h
 
         self.tmpl_name = tmpl_name
         self.rhodecode = rhodecode
         self.c = c
         self._ = _
         self.ungettext = ungettext
         self.h = h
         self.request = request
 
     def _mako_lookup(self):
         _tmpl_lookup = self.rhodecode.CONFIG['pylons.app_globals'].mako_lookup
         return _tmpl_lookup.get_template(self.tmpl_name)
 
     def _update_kwargs_for_render(self, kwargs):
         """
         Inject params required for Mako rendering
         """
         _kwargs = {
             '_': self._,
             'h': self.h,
             'c': self.c,
             'request': self.request,
             'ungettext': self.ungettext,
         }
         _kwargs.update(kwargs)
         return _kwargs
 
     def _render_with_exc(self, render_func, args, kwargs):
         try:
             return render_func.render(*args, **kwargs)
         except:
             log.error(exceptions.text_error_template().render())
             raise
 
     def _get_template(self, template_obj, def_name):
         if def_name:
             tmpl = template_obj.get_def(def_name)
         else:
             tmpl = template_obj
         return tmpl
 
     def render(self, def_name, *args, **kwargs):
         lookup_obj = self._mako_lookup()
         tmpl = self._get_template(lookup_obj, def_name=def_name)
         kwargs = self._update_kwargs_for_render(kwargs)
         return self._render_with_exc(tmpl, args, kwargs)
 
     def __call__(self, tmpl, *args, **kwargs):
         return self.render(tmpl, *args, **kwargs)
 
 
 def password_changed(auth_user, session):
     # Never report password change in case of default user or anonymous user.
     if auth_user.username == User.DEFAULT_USER or auth_user.user_id is None:
         return False
 
     password_hash = md5(auth_user.password) if auth_user.password else None
     rhodecode_user = session.get('rhodecode_user', {})
     session_password_hash = rhodecode_user.get('password', '')
     return password_hash != session_password_hash
 
 
 def read_opensource_licenses():
     global _license_cache
 
     if not _license_cache:
         licenses = pkg_resources.resource_string(
             'rhodecode', 'config/licenses.json')
         _license_cache = json.loads(licenses)
 
     return _license_cache
 
 
 def get_registry(request):
     """
     Utility to get the pyramid registry from a request. During migration to
     pyramid we sometimes want to use the pyramid registry from pylons context.
     Therefore this utility returns `request.registry` for pyramid requests and
     uses `get_current_registry()` for pylons requests.
     """
     try:
         return request.registry
     except AttributeError:
         return get_current_registry()
 
 
 def generate_platform_uuid():
     """
     Generates platform UUID based on it's name
     """
     import platform
 
     try:
         uuid_list = [platform.platform()]
         return hashlib.sha256(':'.join(uuid_list)).hexdigest()
     except Exception as e:
         log.error('Failed to generate host uuid: %s' % e)
         return 'UNDEFINED'

rhodecode/model/settings.py

@@ -1,810 +1,811 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import hashlib
 import logging
 from collections import namedtuple
 from functools import wraps
 
 from rhodecode.lib import caches
 from rhodecode.lib.utils2 import (
     Optional, AttributeDict, safe_str, remove_prefix, str2bool)
 from rhodecode.lib.vcs.backends import base
 from rhodecode.model import BaseModel
 from rhodecode.model.db import (
     RepoRhodeCodeUi, RepoRhodeCodeSetting, RhodeCodeUi, RhodeCodeSetting)
 from rhodecode.model.meta import Session
 
 
 log = logging.getLogger(__name__)
 
 
 UiSetting = namedtuple(
     'UiSetting', ['section', 'key', 'value', 'active'])
 
 SOCIAL_PLUGINS_LIST = ['github', 'bitbucket', 'twitter', 'google']
 
 
 class SettingNotFound(Exception):
     def __init__(self):
         super(SettingNotFound, self).__init__('Setting is not found')
 
 
 class SettingsModel(BaseModel):
     BUILTIN_HOOKS = (
         RhodeCodeUi.HOOK_REPO_SIZE, RhodeCodeUi.HOOK_PUSH,
         RhodeCodeUi.HOOK_PRE_PUSH, RhodeCodeUi.HOOK_PRETX_PUSH,
-        RhodeCodeUi.HOOK_PULL, RhodeCodeUi.HOOK_PRE_PULL)
+        RhodeCodeUi.HOOK_PULL, RhodeCodeUi.HOOK_PRE_PULL,
+        RhodeCodeUi.HOOK_PUSH_KEY,)
     HOOKS_SECTION = 'hooks'
 
     def __init__(self, sa=None, repo=None):
         self.repo = repo
         self.UiDbModel = RepoRhodeCodeUi if repo else RhodeCodeUi
         self.SettingsDbModel = (
             RepoRhodeCodeSetting if repo else RhodeCodeSetting)
         super(SettingsModel, self).__init__(sa)
 
     def get_ui_by_key(self, key):
         q = self.UiDbModel.query()
         q = q.filter(self.UiDbModel.ui_key == key)
         q = self._filter_by_repo(RepoRhodeCodeUi, q)
         return q.scalar()
 
     def get_ui_by_section(self, section):
         q = self.UiDbModel.query()
         q = q.filter(self.UiDbModel.ui_section == section)
         q = self._filter_by_repo(RepoRhodeCodeUi, q)
         return q.all()
 
     def get_ui_by_section_and_key(self, section, key):
         q = self.UiDbModel.query()
         q = q.filter(self.UiDbModel.ui_section == section)
         q = q.filter(self.UiDbModel.ui_key == key)
         q = self._filter_by_repo(RepoRhodeCodeUi, q)
         return q.scalar()
 
     def get_ui(self, section=None, key=None):
         q = self.UiDbModel.query()
         q = self._filter_by_repo(RepoRhodeCodeUi, q)
 
         if section:
             q = q.filter(self.UiDbModel.ui_section == section)
         if key:
             q = q.filter(self.UiDbModel.ui_key == key)
 
         # TODO: mikhail: add caching
         result = [
             UiSetting(
                 section=safe_str(r.ui_section), key=safe_str(r.ui_key),
                 value=safe_str(r.ui_value), active=r.ui_active
             )
             for r in q.all()
         ]
         return result
 
     def get_builtin_hooks(self):
         q = self.UiDbModel.query()
         q = q.filter(self.UiDbModel.ui_key.in_(self.BUILTIN_HOOKS))
         return self._get_hooks(q)
 
     def get_custom_hooks(self):
         q = self.UiDbModel.query()
         q = q.filter(~self.UiDbModel.ui_key.in_(self.BUILTIN_HOOKS))
         return self._get_hooks(q)
 
     def create_ui_section_value(self, section, val, key=None, active=True):
         new_ui = self.UiDbModel()
         new_ui.ui_section = section
         new_ui.ui_value = val
         new_ui.ui_active = active
 
         if self.repo:
             repo = self._get_repo(self.repo)
             repository_id = repo.repo_id
             new_ui.repository_id = repository_id
 
         if not key:
             # keys are unique so they need appended info
             if self.repo:
                 key = hashlib.sha1(
                     '{}{}{}'.format(section, val, repository_id)).hexdigest()
             else:
                 key = hashlib.sha1('{}{}'.format(section, val)).hexdigest()
 
         new_ui.ui_key = key
 
         Session().add(new_ui)
         return new_ui
 
     def create_or_update_hook(self, key, value):
         ui = (
             self.get_ui_by_section_and_key(self.HOOKS_SECTION, key) or
             self.UiDbModel())
         ui.ui_section = self.HOOKS_SECTION
         ui.ui_active = True
         ui.ui_key = key
         ui.ui_value = value
 
         if self.repo:
             repo = self._get_repo(self.repo)
             repository_id = repo.repo_id
             ui.repository_id = repository_id
 
         Session().add(ui)
         return ui
 
     def delete_ui(self, id_):
         ui = self.UiDbModel.get(id_)
         if not ui:
             raise SettingNotFound()
         Session().delete(ui)
 
     def get_setting_by_name(self, name):
         q = self._get_settings_query()
         q = q.filter(self.SettingsDbModel.app_settings_name == name)
         return q.scalar()
 
     def create_or_update_setting(
             self, name, val=Optional(''), type_=Optional('unicode')):
         """
         Creates or updates RhodeCode setting. If updates is triggered it will
         only update parameters that are explicityl set Optional instance will
         be skipped
 
         :param name:
         :param val:
         :param type_:
         :return:
         """
 
         res = self.get_setting_by_name(name)
         repo = self._get_repo(self.repo) if self.repo else None
 
         if not res:
             val = Optional.extract(val)
             type_ = Optional.extract(type_)
 
             args = (
                 (repo.repo_id, name, val, type_)
                 if repo else (name, val, type_))
             res = self.SettingsDbModel(*args)
 
         else:
             if self.repo:
                 res.repository_id = repo.repo_id
 
             res.app_settings_name = name
             if not isinstance(type_, Optional):
                 # update if set
                 res.app_settings_type = type_
             if not isinstance(val, Optional):
                 # update if set
                 res.app_settings_value = val
 
         Session().add(res)
         return res
 
     def invalidate_settings_cache(self):
         namespace = 'rhodecode_settings'
         cache_manager = caches.get_cache_manager('sql_cache_short', namespace)
         caches.clear_cache_manager(cache_manager)
 
     def get_all_settings(self, cache=False):
 
         def _compute():
             q = self._get_settings_query()
             if not q:
                 raise Exception('Could not get application settings !')
 
             settings = {
                 'rhodecode_' + result.app_settings_name: result.app_settings_value
                 for result in q
             }
             return settings
 
         if cache:
             log.debug('Fetching app settings using cache')
             repo = self._get_repo(self.repo) if self.repo else None
             namespace = 'rhodecode_settings'
             cache_manager = caches.get_cache_manager(
                 'sql_cache_short', namespace)
             _cache_key = (
                 "get_repo_{}_settings".format(repo.repo_id)
                 if repo else "get_app_settings")
 
             return cache_manager.get(_cache_key, createfunc=_compute)
 
         else:
             return _compute()
 
     def get_auth_settings(self):
         q = self._get_settings_query()
         q = q.filter(
             self.SettingsDbModel.app_settings_name.startswith('auth_'))
         rows = q.all()
         auth_settings = {
             row.app_settings_name: row.app_settings_value for row in rows}
         return auth_settings
 
     def get_auth_plugins(self):
         auth_plugins = self.get_setting_by_name("auth_plugins")
         return auth_plugins.app_settings_value
 
     def get_default_repo_settings(self, strip_prefix=False):
         q = self._get_settings_query()
         q = q.filter(
             self.SettingsDbModel.app_settings_name.startswith('default_'))
         rows = q.all()
 
         result = {}
         for row in rows:
             key = row.app_settings_name
             if strip_prefix:
                 key = remove_prefix(key, prefix='default_')
             result.update({key: row.app_settings_value})
         return result
 
     def get_repo(self):
         repo = self._get_repo(self.repo)
         if not repo:
             raise Exception(
                 'Repository `{}` cannot be found inside the database'.format(
                     self.repo))
         return repo
 
     def _filter_by_repo(self, model, query):
         if self.repo:
             repo = self.get_repo()
             query = query.filter(model.repository_id == repo.repo_id)
         return query
 
     def _get_hooks(self, query):
         query = query.filter(self.UiDbModel.ui_section == self.HOOKS_SECTION)
         query = self._filter_by_repo(RepoRhodeCodeUi, query)
         return query.all()
 
     def _get_settings_query(self):
         q = self.SettingsDbModel.query()
         return self._filter_by_repo(RepoRhodeCodeSetting, q)
 
     def list_enabled_social_plugins(self, settings):
         enabled = []
         for plug in SOCIAL_PLUGINS_LIST:
             if str2bool(settings.get('rhodecode_auth_{}_enabled'.format(plug)
                                      )):
                 enabled.append(plug)
         return enabled
 
 
 def assert_repo_settings(func):
     @wraps(func)
     def _wrapper(self, *args, **kwargs):
         if not self.repo_settings:
             raise Exception('Repository is not specified')
         return func(self, *args, **kwargs)
     return _wrapper
 
 
 class IssueTrackerSettingsModel(object):
     INHERIT_SETTINGS = 'inherit_issue_tracker_settings'
     SETTINGS_PREFIX = 'issuetracker_'
 
     def __init__(self, sa=None, repo=None):
         self.global_settings = SettingsModel(sa=sa)
         self.repo_settings = SettingsModel(sa=sa, repo=repo) if repo else None
 
     @property
     def inherit_global_settings(self):
         if not self.repo_settings:
             return True
         setting = self.repo_settings.get_setting_by_name(self.INHERIT_SETTINGS)
         return setting.app_settings_value if setting else True
 
     @inherit_global_settings.setter
     def inherit_global_settings(self, value):
         if self.repo_settings:
             settings = self.repo_settings.create_or_update_setting(
                 self.INHERIT_SETTINGS, value, type_='bool')
             Session().add(settings)
 
     def _get_keyname(self, key, uid, prefix=''):
         return '{0}{1}{2}_{3}'.format(
             prefix, self.SETTINGS_PREFIX, key, uid)
 
     def _make_dict_for_settings(self, qs):
         prefix_match = self._get_keyname('pat', '', 'rhodecode_')
 
         issuetracker_entries = {}
         # create keys
         for k, v in qs.items():
             if k.startswith(prefix_match):
                 uid = k[len(prefix_match):]
                 issuetracker_entries[uid] = None
 
         # populate
         for uid in issuetracker_entries:
             issuetracker_entries[uid] = AttributeDict({
                 'pat': qs.get(self._get_keyname('pat', uid, 'rhodecode_')),
                 'url': qs.get(self._get_keyname('url', uid, 'rhodecode_')),
                 'pref': qs.get(self._get_keyname('pref', uid, 'rhodecode_')),
                 'desc': qs.get(self._get_keyname('desc', uid, 'rhodecode_')),
             })
         return issuetracker_entries
 
     def get_global_settings(self, cache=False):
         """
         Returns list of global issue tracker settings
         """
         defaults = self.global_settings.get_all_settings(cache=cache)
         settings = self._make_dict_for_settings(defaults)
         return settings
 
     def get_repo_settings(self, cache=False):
         """
         Returns list of issue tracker settings per repository
         """
         if not self.repo_settings:
             raise Exception('Repository is not specified')
         all_settings = self.repo_settings.get_all_settings(cache=cache)
         settings = self._make_dict_for_settings(all_settings)
         return settings
 
     def get_settings(self, cache=False):
         if self.inherit_global_settings:
             return self.get_global_settings(cache=cache)
         else:
             return self.get_repo_settings(cache=cache)
 
     def delete_entries(self, uid):
         if self.repo_settings:
             all_patterns = self.get_repo_settings()
             settings_model = self.repo_settings
         else:
             all_patterns = self.get_global_settings()
             settings_model = self.global_settings
         entries = all_patterns.get(uid)
 
         for del_key in entries:
             setting_name = self._get_keyname(del_key, uid)
             entry = settings_model.get_setting_by_name(setting_name)
             if entry:
                 Session().delete(entry)
 
         Session().commit()
 
     def create_or_update_setting(
             self, name, val=Optional(''), type_=Optional('unicode')):
         if self.repo_settings:
             setting = self.repo_settings.create_or_update_setting(
                 name, val, type_)
         else:
             setting = self.global_settings.create_or_update_setting(
                 name, val, type_)
         return setting
 
 
 class VcsSettingsModel(object):
 
     INHERIT_SETTINGS = 'inherit_vcs_settings'
     GENERAL_SETTINGS = (
         'use_outdated_comments',
         'pr_merge_enabled',
         'hg_use_rebase_for_merging')
 
     HOOKS_SETTINGS = (
         ('hooks', 'changegroup.repo_size'),
         ('hooks', 'changegroup.push_logger'),
         ('hooks', 'outgoing.pull_logger'),)
     HG_SETTINGS = (
         ('extensions', 'largefiles'),
         ('phases', 'publish'),
         ('extensions', 'evolve'),)
     GIT_SETTINGS = (
         ('vcs_git_lfs', 'enabled'),)
     GLOBAL_HG_SETTINGS = (
         ('extensions', 'largefiles'),
         ('largefiles', 'usercache'),
         ('phases', 'publish'),
         ('extensions', 'hgsubversion'),
         ('extensions', 'evolve'),)
     GLOBAL_GIT_SETTINGS = (
         ('vcs_git_lfs', 'enabled'),
         ('vcs_git_lfs', 'store_location'))
     GLOBAL_SVN_SETTINGS = (
         ('vcs_svn_proxy', 'http_requests_enabled'),
         ('vcs_svn_proxy', 'http_server_url'))
 
     SVN_BRANCH_SECTION = 'vcs_svn_branch'
     SVN_TAG_SECTION = 'vcs_svn_tag'
     SSL_SETTING = ('web', 'push_ssl')
     PATH_SETTING = ('paths', '/')
 
     def __init__(self, sa=None, repo=None):
         self.global_settings = SettingsModel(sa=sa)
         self.repo_settings = SettingsModel(sa=sa, repo=repo) if repo else None
         self._ui_settings = (
             self.HG_SETTINGS + self.GIT_SETTINGS + self.HOOKS_SETTINGS)
         self._svn_sections = (self.SVN_BRANCH_SECTION, self.SVN_TAG_SECTION)
 
     @property
     @assert_repo_settings
     def inherit_global_settings(self):
         setting = self.repo_settings.get_setting_by_name(self.INHERIT_SETTINGS)
         return setting.app_settings_value if setting else True
 
     @inherit_global_settings.setter
     @assert_repo_settings
     def inherit_global_settings(self, value):
         self.repo_settings.create_or_update_setting(
             self.INHERIT_SETTINGS, value, type_='bool')
 
     def get_global_svn_branch_patterns(self):
         return self.global_settings.get_ui_by_section(self.SVN_BRANCH_SECTION)
 
     @assert_repo_settings
     def get_repo_svn_branch_patterns(self):
         return self.repo_settings.get_ui_by_section(self.SVN_BRANCH_SECTION)
 
     def get_global_svn_tag_patterns(self):
         return self.global_settings.get_ui_by_section(self.SVN_TAG_SECTION)
 
     @assert_repo_settings
     def get_repo_svn_tag_patterns(self):
         return self.repo_settings.get_ui_by_section(self.SVN_TAG_SECTION)
 
     def get_global_settings(self):
         return self._collect_all_settings(global_=True)
 
     @assert_repo_settings
     def get_repo_settings(self):
         return self._collect_all_settings(global_=False)
 
     @assert_repo_settings
     def create_or_update_repo_settings(
             self, data, inherit_global_settings=False):
         from rhodecode.model.scm import ScmModel
 
         self.inherit_global_settings = inherit_global_settings
 
         repo = self.repo_settings.get_repo()
         if not inherit_global_settings:
             if repo.repo_type == 'svn':
                 self.create_repo_svn_settings(data)
             else:
                 self.create_or_update_repo_hook_settings(data)
                 self.create_or_update_repo_pr_settings(data)
 
             if repo.repo_type == 'hg':
                 self.create_or_update_repo_hg_settings(data)
 
             if repo.repo_type == 'git':
                 self.create_or_update_repo_git_settings(data)
 
         ScmModel().mark_for_invalidation(repo.repo_name, delete=True)
 
     @assert_repo_settings
     def create_or_update_repo_hook_settings(self, data):
         for section, key in self.HOOKS_SETTINGS:
             data_key = self._get_form_ui_key(section, key)
             if data_key not in data:
                 raise ValueError(
                     'The given data does not contain {} key'.format(data_key))
 
             active = data.get(data_key)
             repo_setting = self.repo_settings.get_ui_by_section_and_key(
                 section, key)
             if not repo_setting:
                 global_setting = self.global_settings.\
                     get_ui_by_section_and_key(section, key)
                 self.repo_settings.create_ui_section_value(
                     section, global_setting.ui_value, key=key, active=active)
             else:
                 repo_setting.ui_active = active
                 Session().add(repo_setting)
 
     def update_global_hook_settings(self, data):
         for section, key in self.HOOKS_SETTINGS:
             data_key = self._get_form_ui_key(section, key)
             if data_key not in data:
                 raise ValueError(
                     'The given data does not contain {} key'.format(data_key))
             active = data.get(data_key)
             repo_setting = self.global_settings.get_ui_by_section_and_key(
                 section, key)
             repo_setting.ui_active = active
             Session().add(repo_setting)
 
     @assert_repo_settings
     def create_or_update_repo_pr_settings(self, data):
         return self._create_or_update_general_settings(
             self.repo_settings, data)
 
     def create_or_update_global_pr_settings(self, data):
         return self._create_or_update_general_settings(
             self.global_settings, data)
 
     @assert_repo_settings
     def create_repo_svn_settings(self, data):
         return self._create_svn_settings(self.repo_settings, data)
 
     @assert_repo_settings
     def create_or_update_repo_hg_settings(self, data):
         largefiles, phases, evolve = \
             self.HG_SETTINGS
         largefiles_key, phases_key, evolve_key = \
             self._get_settings_keys(self.HG_SETTINGS, data)
 
         self._create_or_update_ui(
             self.repo_settings, *largefiles, value='',
             active=data[largefiles_key])
         self._create_or_update_ui(
             self.repo_settings, *evolve, value='',
             active=data[evolve_key])
         self._create_or_update_ui(
             self.repo_settings, *phases, value=safe_str(data[phases_key]))
 
     def create_or_update_global_hg_settings(self, data):
         largefiles, largefiles_store, phases, hgsubversion, evolve \
             = self.GLOBAL_HG_SETTINGS
         largefiles_key, largefiles_store_key, phases_key, subversion_key, evolve_key \
             = self._get_settings_keys(self.GLOBAL_HG_SETTINGS, data)
 
         self._create_or_update_ui(
             self.global_settings, *largefiles, value='',
             active=data[largefiles_key])
         self._create_or_update_ui(
             self.global_settings, *largefiles_store,
             value=data[largefiles_store_key])
         self._create_or_update_ui(
             self.global_settings, *phases, value=safe_str(data[phases_key]))
         self._create_or_update_ui(
             self.global_settings, *hgsubversion, active=data[subversion_key])
         self._create_or_update_ui(
             self.global_settings, *evolve, value='',
             active=data[evolve_key])
 
     def create_or_update_repo_git_settings(self, data):
         # NOTE(marcink): # comma make unpack work properly
         lfs_enabled, \
             = self.GIT_SETTINGS
 
         lfs_enabled_key, \
             = self._get_settings_keys(self.GIT_SETTINGS, data)
 
         self._create_or_update_ui(
             self.repo_settings, *lfs_enabled, value=data[lfs_enabled_key],
             active=data[lfs_enabled_key])
 
     def create_or_update_global_git_settings(self, data):
         lfs_enabled, lfs_store_location \
             = self.GLOBAL_GIT_SETTINGS
         lfs_enabled_key, lfs_store_location_key \
             = self._get_settings_keys(self.GLOBAL_GIT_SETTINGS, data)
 
         self._create_or_update_ui(
             self.global_settings, *lfs_enabled, value=data[lfs_enabled_key],
             active=data[lfs_enabled_key])
         self._create_or_update_ui(
             self.global_settings, *lfs_store_location,
             value=data[lfs_store_location_key])
 
     def create_or_update_global_svn_settings(self, data):
         # branch/tags patterns
         self._create_svn_settings(self.global_settings, data)
 
         http_requests_enabled, http_server_url = self.GLOBAL_SVN_SETTINGS
         http_requests_enabled_key, http_server_url_key = self._get_settings_keys(
             self.GLOBAL_SVN_SETTINGS, data)
 
         self._create_or_update_ui(
             self.global_settings, *http_requests_enabled,
             value=safe_str(data[http_requests_enabled_key]))
         self._create_or_update_ui(
             self.global_settings, *http_server_url,
             value=data[http_server_url_key])
 
     def update_global_ssl_setting(self, value):
         self._create_or_update_ui(
             self.global_settings, *self.SSL_SETTING, value=value)
 
     def update_global_path_setting(self, value):
         self._create_or_update_ui(
             self.global_settings, *self.PATH_SETTING, value=value)
 
     @assert_repo_settings
     def delete_repo_svn_pattern(self, id_):
         self.repo_settings.delete_ui(id_)
 
     def delete_global_svn_pattern(self, id_):
         self.global_settings.delete_ui(id_)
 
     @assert_repo_settings
     def get_repo_ui_settings(self, section=None, key=None):
         global_uis = self.global_settings.get_ui(section, key)
         repo_uis = self.repo_settings.get_ui(section, key)
         filtered_repo_uis = self._filter_ui_settings(repo_uis)
         filtered_repo_uis_keys = [
             (s.section, s.key) for s in filtered_repo_uis]
 
         def _is_global_ui_filtered(ui):
             return (
                 (ui.section, ui.key) in filtered_repo_uis_keys
                 or ui.section in self._svn_sections)
 
         filtered_global_uis = [
             ui for ui in global_uis if not _is_global_ui_filtered(ui)]
 
         return filtered_global_uis + filtered_repo_uis
 
     def get_global_ui_settings(self, section=None, key=None):
         return self.global_settings.get_ui(section, key)
 
     def get_ui_settings_as_config_obj(self, section=None, key=None):
         config = base.Config()
 
         ui_settings = self.get_ui_settings(section=section, key=key)
 
         for entry in ui_settings:
             config.set(entry.section, entry.key, entry.value)
 
         return config
 
     def get_ui_settings(self, section=None, key=None):
         if not self.repo_settings or self.inherit_global_settings:
             return self.get_global_ui_settings(section, key)
         else:
             return self.get_repo_ui_settings(section, key)
 
     def get_svn_patterns(self, section=None):
         if not self.repo_settings:
             return self.get_global_ui_settings(section)
         else:
             return self.get_repo_ui_settings(section)
 
     @assert_repo_settings
     def get_repo_general_settings(self):
         global_settings = self.global_settings.get_all_settings()
         repo_settings = self.repo_settings.get_all_settings()
         filtered_repo_settings = self._filter_general_settings(repo_settings)
         global_settings.update(filtered_repo_settings)
         return global_settings
 
     def get_global_general_settings(self):
         return self.global_settings.get_all_settings()
 
     def get_general_settings(self):
         if not self.repo_settings or self.inherit_global_settings:
             return self.get_global_general_settings()
         else:
             return self.get_repo_general_settings()
 
     def get_repos_location(self):
         return self.global_settings.get_ui_by_key('/').ui_value
 
     def _filter_ui_settings(self, settings):
         filtered_settings = [
             s for s in settings if self._should_keep_setting(s)]
         return filtered_settings
 
     def _should_keep_setting(self, setting):
         keep = (
             (setting.section, setting.key) in self._ui_settings or
             setting.section in self._svn_sections)
         return keep
 
     def _filter_general_settings(self, settings):
         keys = ['rhodecode_{}'.format(key) for key in self.GENERAL_SETTINGS]
         return {
             k: settings[k]
             for k in settings if k in keys}
 
     def _collect_all_settings(self, global_=False):
         settings = self.global_settings if global_ else self.repo_settings
         result = {}
 
         for section, key in self._ui_settings:
             ui = settings.get_ui_by_section_and_key(section, key)
             result_key = self._get_form_ui_key(section, key)
 
             if ui:
                 if section in ('hooks', 'extensions'):
                     result[result_key] = ui.ui_active
                 elif result_key in ['vcs_git_lfs_enabled']:
                     result[result_key] = ui.ui_active
                 else:
                     result[result_key] = ui.ui_value
 
         for name in self.GENERAL_SETTINGS:
             setting = settings.get_setting_by_name(name)
             if setting:
                 result_key = 'rhodecode_{}'.format(name)
                 result[result_key] = setting.app_settings_value
 
         return result
 
     def _get_form_ui_key(self, section, key):
         return '{section}_{key}'.format(
             section=section, key=key.replace('.', '_'))
 
     def _create_or_update_ui(
             self, settings, section, key, value=None, active=None):
         ui = settings.get_ui_by_section_and_key(section, key)
         if not ui:
             active = True if active is None else active
             settings.create_ui_section_value(
                 section, value, key=key, active=active)
         else:
             if active is not None:
                 ui.ui_active = active
             if value is not None:
                 ui.ui_value = value
             Session().add(ui)
 
     def _create_svn_settings(self, settings, data):
         svn_settings = {
             'new_svn_branch': self.SVN_BRANCH_SECTION,
             'new_svn_tag': self.SVN_TAG_SECTION
         }
         for key in svn_settings:
             if data.get(key):
                 settings.create_ui_section_value(svn_settings[key], data[key])
 
     def _create_or_update_general_settings(self, settings, data):
         for name in self.GENERAL_SETTINGS:
             data_key = 'rhodecode_{}'.format(name)
             if data_key not in data:
                 raise ValueError(
                     'The given data does not contain {} key'.format(data_key))
             setting = settings.create_or_update_setting(
                 name, data[data_key], 'bool')
             Session().add(setting)
 
     def _get_settings_keys(self, settings, data):
         data_keys = [self._get_form_ui_key(*s) for s in settings]
         for data_key in data_keys:
             if data_key not in data:
                 raise ValueError(
                     'The given data does not contain {} key'.format(data_key))
         return data_keys
 
     def create_largeobjects_dirs_if_needed(self, repo_store_path):
         """
         This is subscribed to the `pyramid.events.ApplicationCreated` event. It
         does a repository scan if enabled in the settings.
         """
 
         from rhodecode.lib.vcs.backends.hg import largefiles_store
         from rhodecode.lib.vcs.backends.git import lfs_store
 
         paths = [
             largefiles_store(repo_store_path),
             lfs_store(repo_store_path)]
 
         for path in paths:
             if os.path.isdir(path):
                 continue
             if os.path.isfile(path):
                 continue
             # not a file nor dir, we try to create it
             try:
                 os.makedirs(path)
             except Exception:
                 log.warning('Failed to create largefiles dir:%s', path)

rhodecode/tests/lib/test_utils.py

@@ -1,471 +1,473 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import json
 import multiprocessing
 import os
 
 import mock
 import py
 import pytest
 
 from rhodecode.lib import caching_query
 from rhodecode.lib import utils
 from rhodecode.lib.utils2 import md5
 from rhodecode.model import settings
 from rhodecode.model import db
 from rhodecode.model import meta
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.settings import UiSetting, SettingsModel
 from rhodecode.tests.fixture import Fixture
 from rhodecode.tests import TEST_USER_ADMIN_LOGIN
 
 
 fixture = Fixture()
 
 
 def extract_hooks(config):
     """Return a dictionary with the hook entries of the given config."""
     hooks = {}
     config_items = config.serialize()
     for section, name, value in config_items:
         if section != 'hooks':
             continue
         hooks[name] = value
 
     return hooks
 
 
 def disable_hooks(request, hooks):
     """Disables the given hooks from the UI settings."""
     session = meta.Session()
 
     model = SettingsModel()
     for hook_key in hooks:
         sett = model.get_ui_by_key(hook_key)
         sett.ui_active = False
         session.add(sett)
 
     # Invalidate cache
     ui_settings = session.query(db.RhodeCodeUi).options(
         caching_query.FromCache('sql_cache_short', 'get_hg_ui_settings'))
     ui_settings.invalidate()
 
     ui_settings = session.query(db.RhodeCodeUi).options(
         caching_query.FromCache(
             'sql_cache_short', 'get_hook_settings', 'get_hook_settings'))
     ui_settings.invalidate()
 
     @request.addfinalizer
     def rollback():
         session.rollback()
 
 
 HOOK_PRE_PUSH = db.RhodeCodeUi.HOOK_PRE_PUSH
 HOOK_PRETX_PUSH = db.RhodeCodeUi.HOOK_PRETX_PUSH
 HOOK_PUSH = db.RhodeCodeUi.HOOK_PUSH
 HOOK_PRE_PULL = db.RhodeCodeUi.HOOK_PRE_PULL
 HOOK_PULL = db.RhodeCodeUi.HOOK_PULL
 HOOK_REPO_SIZE = db.RhodeCodeUi.HOOK_REPO_SIZE
+HOOK_PUSH_KEY = db.RhodeCodeUi.HOOK_PUSH_KEY
 
 HG_HOOKS = frozenset(
     (HOOK_PRE_PULL, HOOK_PULL, HOOK_PRE_PUSH, HOOK_PRETX_PUSH, HOOK_PUSH,
-     HOOK_REPO_SIZE))
+     HOOK_REPO_SIZE, HOOK_PUSH_KEY))
 
 
 @pytest.mark.parametrize('disabled_hooks,expected_hooks', [
     ([], HG_HOOKS),
     (HG_HOOKS, []),
 
-    ([HOOK_PRE_PUSH, HOOK_PRETX_PUSH, HOOK_REPO_SIZE], [HOOK_PRE_PULL, HOOK_PULL, HOOK_PUSH]),
+    ([HOOK_PRE_PUSH, HOOK_PRETX_PUSH, HOOK_REPO_SIZE, HOOK_PUSH_KEY], [HOOK_PRE_PULL, HOOK_PULL, HOOK_PUSH]),
 
     # When a pull/push hook is disabled, its pre-pull/push counterpart should
     # be disabled too.
     ([HOOK_PUSH], [HOOK_PRE_PULL, HOOK_PULL, HOOK_REPO_SIZE]),
-    ([HOOK_PULL], [HOOK_PRE_PUSH, HOOK_PRETX_PUSH, HOOK_PUSH, HOOK_REPO_SIZE]),
+    ([HOOK_PULL], [HOOK_PRE_PUSH, HOOK_PRETX_PUSH, HOOK_PUSH, HOOK_REPO_SIZE,
+                   HOOK_PUSH_KEY]),
 ])
 def test_make_db_config_hg_hooks(pylonsapp, request, disabled_hooks,
                                  expected_hooks):
     disable_hooks(request, disabled_hooks)
 
     config = utils.make_db_config()
     hooks = extract_hooks(config)
 
     assert set(hooks.iterkeys()).intersection(HG_HOOKS) == set(expected_hooks)
 
 
 @pytest.mark.parametrize('disabled_hooks,expected_hooks', [
     ([], ['pull', 'push']),
     ([HOOK_PUSH], ['pull']),
     ([HOOK_PULL], ['push']),
     ([HOOK_PULL, HOOK_PUSH], []),
 ])
 def test_get_enabled_hook_classes(disabled_hooks, expected_hooks):
     hook_keys = (HOOK_PUSH, HOOK_PULL)
     ui_settings = [
         ('hooks', key, 'some value', key not in disabled_hooks)
         for key in hook_keys]
 
     result = utils.get_enabled_hook_classes(ui_settings)
     assert sorted(result) == expected_hooks
 
 
 def test_get_filesystem_repos_finds_repos(tmpdir, pylonsapp):
     _stub_git_repo(tmpdir.ensure('repo', dir=True))
     repos = list(utils.get_filesystem_repos(str(tmpdir)))
     assert repos == [('repo', ('git', tmpdir.join('repo')))]
 
 
 def test_get_filesystem_repos_skips_directories(tmpdir, pylonsapp):
     tmpdir.ensure('not-a-repo', dir=True)
     repos = list(utils.get_filesystem_repos(str(tmpdir)))
     assert repos == []
 
 
 def test_get_filesystem_repos_skips_directories_with_repos(tmpdir, pylonsapp):
     _stub_git_repo(tmpdir.ensure('subdir/repo', dir=True))
     repos = list(utils.get_filesystem_repos(str(tmpdir)))
     assert repos == []
 
 
 def test_get_filesystem_repos_finds_repos_in_subdirectories(tmpdir, pylonsapp):
     _stub_git_repo(tmpdir.ensure('subdir/repo', dir=True))
     repos = list(utils.get_filesystem_repos(str(tmpdir), recursive=True))
     assert repos == [('subdir/repo', ('git', tmpdir.join('subdir', 'repo')))]
 
 
 def test_get_filesystem_repos_skips_names_starting_with_dot(tmpdir):
     _stub_git_repo(tmpdir.ensure('.repo', dir=True))
     repos = list(utils.get_filesystem_repos(str(tmpdir)))
     assert repos == []
 
 
 def test_get_filesystem_repos_skips_files(tmpdir):
     tmpdir.ensure('test-file')
     repos = list(utils.get_filesystem_repos(str(tmpdir)))
     assert repos == []
 
 
 def test_get_filesystem_repos_skips_removed_repositories(tmpdir):
     removed_repo_name = 'rm__00000000_000000_000000__.stub'
     assert utils.REMOVED_REPO_PAT.match(removed_repo_name)
     _stub_git_repo(tmpdir.ensure(removed_repo_name, dir=True))
     repos = list(utils.get_filesystem_repos(str(tmpdir)))
     assert repos == []
 
 
 def _stub_git_repo(repo_path):
     """
     Make `repo_path` look like a Git repository.
     """
     repo_path.ensure('.git', dir=True)
 
 
 @pytest.mark.parametrize('str_class', [str, unicode], ids=['str', 'unicode'])
 def test_get_dirpaths_returns_all_paths(tmpdir, str_class):
     tmpdir.ensure('test-file')
     dirpaths = utils._get_dirpaths(str_class(tmpdir))
     assert dirpaths == ['test-file']
 
 
 def test_get_dirpaths_returns_all_paths_bytes(
         tmpdir, platform_encodes_filenames):
     if platform_encodes_filenames:
         pytest.skip("This platform seems to encode filenames.")
     tmpdir.ensure('repo-a-umlaut-\xe4')
     dirpaths = utils._get_dirpaths(str(tmpdir))
     assert dirpaths == ['repo-a-umlaut-\xe4']
 
 
 def test_get_dirpaths_skips_paths_it_cannot_decode(
         tmpdir, platform_encodes_filenames):
     if platform_encodes_filenames:
         pytest.skip("This platform seems to encode filenames.")
     path_with_latin1 = 'repo-a-umlaut-\xe4'
     tmpdir.ensure(path_with_latin1)
     dirpaths = utils._get_dirpaths(unicode(tmpdir))
     assert dirpaths == []
 
 
 @pytest.fixture(scope='session')
 def platform_encodes_filenames():
     """
     Boolean indicator if the current platform changes filename encodings.
     """
     path_with_latin1 = 'repo-a-umlaut-\xe4'
     tmpdir = py.path.local.mkdtemp()
     tmpdir.ensure(path_with_latin1)
     read_path = tmpdir.listdir()[0].basename
     tmpdir.remove()
     return path_with_latin1 != read_path
 
 
 def test_action_logger_action_size(pylonsapp, test_repo):
     action = 'x' * 1200001
     utils.action_logger(TEST_USER_ADMIN_LOGIN, action, test_repo, commit=True)
 
 
 @pytest.fixture
 def repo_groups(request):
     session = meta.Session()
     zombie_group = fixture.create_repo_group('zombie')
     parent_group = fixture.create_repo_group('parent')
     child_group = fixture.create_repo_group('parent/child')
     groups_in_db = session.query(db.RepoGroup).all()
     assert len(groups_in_db) == 3
     assert child_group.group_parent_id == parent_group.group_id
 
     @request.addfinalizer
     def cleanup():
         fixture.destroy_repo_group(zombie_group)
         fixture.destroy_repo_group(child_group)
         fixture.destroy_repo_group(parent_group)
 
     return (zombie_group, parent_group, child_group)
 
 
 def test_repo2db_mapper_groups(repo_groups):
     session = meta.Session()
     zombie_group, parent_group, child_group = repo_groups
     zombie_path = os.path.join(
         RepoGroupModel().repos_path, zombie_group.full_path)
     os.rmdir(zombie_path)
 
     # Avoid removing test repos when calling repo2db_mapper
     repo_list = {
         repo.repo_name: 'test' for repo in session.query(db.Repository).all()
     }
     utils.repo2db_mapper(repo_list, remove_obsolete=True)
 
     groups_in_db = session.query(db.RepoGroup).all()
     assert child_group in groups_in_db
     assert parent_group in groups_in_db
     assert zombie_path not in groups_in_db
 
 
 def test_repo2db_mapper_enables_largefiles(backend):
     repo = backend.create_repo()
     repo_list = {repo.repo_name: 'test'}
     with mock.patch('rhodecode.model.db.Repository.scm_instance') as scm_mock:
         with mock.patch.multiple('rhodecode.model.scm.ScmModel',
                                  install_git_hook=mock.DEFAULT,
                                  install_svn_hooks=mock.DEFAULT):
             utils.repo2db_mapper(repo_list, remove_obsolete=False)
             _, kwargs = scm_mock.call_args
             assert kwargs['config'].get('extensions', 'largefiles') == ''
 
 
 @pytest.mark.backends("git", "svn")
 def test_repo2db_mapper_installs_hooks_for_repos_in_db(backend):
     repo = backend.create_repo()
     repo_list = {repo.repo_name: 'test'}
     with mock.patch.object(ScmModel, 'install_hooks') as install_hooks_mock:
         utils.repo2db_mapper(repo_list, remove_obsolete=False)
     install_hooks_mock.assert_called_once_with(
         repo.scm_instance(), repo_type=backend.alias)
 
 
 @pytest.mark.backends("git", "svn")
 def test_repo2db_mapper_installs_hooks_for_newly_added_repos(backend):
     repo = backend.create_repo()
     RepoModel().delete(repo, fs_remove=False)
     meta.Session().commit()
     repo_list = {repo.repo_name: repo.scm_instance()}
     with mock.patch.object(ScmModel, 'install_hooks') as install_hooks_mock:
         utils.repo2db_mapper(repo_list, remove_obsolete=False)
     assert install_hooks_mock.call_count == 1
     install_hooks_args, _ = install_hooks_mock.call_args
     assert install_hooks_args[0].name == repo.repo_name
 
 
 class TestPasswordChanged(object):
     def setup(self):
         self.session = {
             'rhodecode_user': {
                 'password': '0cc175b9c0f1b6a831c399e269772661'
             }
         }
         self.auth_user = mock.Mock()
         self.auth_user.userame = 'test'
         self.auth_user.password = 'abc123'
 
     def test_returns_false_for_default_user(self):
         self.auth_user.username = db.User.DEFAULT_USER
         result = utils.password_changed(self.auth_user, self.session)
         assert result is False
 
     def test_returns_false_if_password_was_not_changed(self):
         self.session['rhodecode_user']['password'] = md5(
             self.auth_user.password)
         result = utils.password_changed(self.auth_user, self.session)
         assert result is False
 
     def test_returns_true_if_password_was_changed(self):
         result = utils.password_changed(self.auth_user, self.session)
         assert result is True
 
     def test_returns_true_if_auth_user_password_is_empty(self):
         self.auth_user.password = None
         result = utils.password_changed(self.auth_user, self.session)
         assert result is True
 
     def test_returns_true_if_session_password_is_empty(self):
         self.session['rhodecode_user'].pop('password')
         result = utils.password_changed(self.auth_user, self.session)
         assert result is True
 
 
 class TestReadOpensourceLicenses(object):
     def test_success(self):
         utils._license_cache = None
         json_data = '''
         {
             "python2.7-pytest-2.7.1": {"UNKNOWN": null},
             "python2.7-Markdown-2.6.2": {
                 "BSD-3-Clause": "http://spdx.org/licenses/BSD-3-Clause"
             }
         }
         '''
         resource_string_patch = mock.patch.object(
             utils.pkg_resources, 'resource_string', return_value=json_data)
         with resource_string_patch:
             result = utils.read_opensource_licenses()
         assert result == json.loads(json_data)
 
     def test_caching(self):
         utils._license_cache = {
             "python2.7-pytest-2.7.1": {
                 "UNKNOWN": None
             },
             "python2.7-Markdown-2.6.2": {
                 "BSD-3-Clause": "http://spdx.org/licenses/BSD-3-Clause"
             }
         }
         resource_patch = mock.patch.object(
             utils.pkg_resources, 'resource_string', side_effect=Exception)
         json_patch = mock.patch.object(
             utils.json, 'loads', side_effect=Exception)
 
         with resource_patch as resource_mock, json_patch as json_mock:
             result = utils.read_opensource_licenses()
 
         assert resource_mock.call_count == 0
         assert json_mock.call_count == 0
         assert result == utils._license_cache
 
     def test_licenses_file_contains_no_unknown_licenses(self):
         utils._license_cache = None
         result = utils.read_opensource_licenses()
         license_names = []
         for licenses in result.values():
             license_names.extend(licenses.keys())
         assert 'UNKNOWN' not in license_names
 
 
 class TestMakeDbConfig(object):
     def test_data_from_config_data_from_db_returned(self):
         test_data = [
             ('section1', 'option1', 'value1'),
             ('section2', 'option2', 'value2'),
             ('section3', 'option3', 'value3'),
         ]
         with mock.patch.object(utils, 'config_data_from_db') as config_mock:
             config_mock.return_value = test_data
             kwargs = {'clear_session': False, 'repo': 'test_repo'}
             result = utils.make_db_config(**kwargs)
         config_mock.assert_called_once_with(**kwargs)
         for section, option, expected_value in test_data:
             value = result.get(section, option)
             assert value == expected_value
 
 
 class TestConfigDataFromDb(object):
     def test_config_data_from_db_returns_active_settings(self):
         test_data = [
             UiSetting('section1', 'option1', 'value1', True),
             UiSetting('section2', 'option2', 'value2', True),
             UiSetting('section3', 'option3', 'value3', False),
         ]
         repo_name = 'test_repo'
 
         model_patch = mock.patch.object(settings, 'VcsSettingsModel')
         hooks_patch = mock.patch.object(
             utils, 'get_enabled_hook_classes',
             return_value=['pull', 'push', 'repo_size'])
         with model_patch as model_mock, hooks_patch:
             instance_mock = mock.Mock()
             model_mock.return_value = instance_mock
             instance_mock.get_ui_settings.return_value = test_data
             result = utils.config_data_from_db(
                 clear_session=False, repo=repo_name)
 
         self._assert_repo_name_passed(model_mock, repo_name)
 
         expected_result = [
             ('section1', 'option1', 'value1'),
             ('section2', 'option2', 'value2'),
         ]
         assert result == expected_result
 
     def _assert_repo_name_passed(self, model_mock, repo_name):
         assert model_mock.call_count == 1
         call_args, call_kwargs = model_mock.call_args
         assert call_kwargs['repo'] == repo_name
 
 
 class TestIsDirWritable(object):
     def test_returns_false_when_not_writable(self):
         with mock.patch('__builtin__.open', side_effect=OSError):
             assert not utils._is_dir_writable('/stub-path')
 
     def test_returns_true_when_writable(self, tmpdir):
         assert utils._is_dir_writable(str(tmpdir))
 
     def test_is_safe_against_race_conditions(self, tmpdir):
         workers = multiprocessing.Pool()
         directories = [str(tmpdir)] * 10
         workers.map(utils._is_dir_writable, directories)
 
 
 class TestGetEnabledHooks(object):
     def test_only_active_hooks_are_enabled(self):
         ui_settings = [
             UiSetting('hooks', db.RhodeCodeUi.HOOK_PUSH, 'value', True),
             UiSetting('hooks', db.RhodeCodeUi.HOOK_REPO_SIZE, 'value', True),
             UiSetting('hooks', db.RhodeCodeUi.HOOK_PULL, 'value', False)
         ]
         result = utils.get_enabled_hook_classes(ui_settings)
         assert result == ['push', 'repo_size']
 
     def test_all_hooks_are_enabled(self):
         ui_settings = [
             UiSetting('hooks', db.RhodeCodeUi.HOOK_PUSH, 'value', True),
             UiSetting('hooks', db.RhodeCodeUi.HOOK_REPO_SIZE, 'value', True),
             UiSetting('hooks', db.RhodeCodeUi.HOOK_PULL, 'value', True)
         ]
         result = utils.get_enabled_hook_classes(ui_settings)
         assert result == ['push', 'repo_size', 'pull']
 
     def test_no_enabled_hooks_when_no_hook_settings_are_found(self):
         ui_settings = []
         result = utils.get_enabled_hook_classes(ui_settings)
         assert result == []

rhodecode/tests/other/vcs_operations/__init__.py

@@ -1,147 +1,154 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Base for test suite for making push/pull operations.
 
 .. important::
 
    You must have git >= 1.8.5 for tests to work fine. With 68b939b git started
    to redirect things to stderr instead of stdout.
 """
 
 from os.path import join as jn
 from subprocess32 import Popen, PIPE
 import logging
 import os
 import tempfile
 
 from rhodecode.tests import GIT_REPO, HG_REPO
 
 DEBUG = True
 RC_LOG = os.path.join(tempfile.gettempdir(), 'rc.log')
 REPO_GROUP = 'a_repo_group'
 HG_REPO_WITH_GROUP = '%s/%s' % (REPO_GROUP, HG_REPO)
 GIT_REPO_WITH_GROUP = '%s/%s' % (REPO_GROUP, GIT_REPO)
 
 log = logging.getLogger(__name__)
 
 
 class Command(object):
 
     def __init__(self, cwd):
         self.cwd = cwd
         self.process = None
 
     def execute(self, cmd, *args):
         """
         Runs command on the system with given ``args``.
         """
 
         command = cmd + ' ' + ' '.join(args)
         if DEBUG:
             log.debug('*** CMD %s ***' % (command,))
 
         env = dict(os.environ)
         # Delete coverage variables, as they make the test fail for Mercurial
         for key in env.keys():
             if key.startswith('COV_CORE_'):
                 del env[key]
 
         self.process = Popen(command, shell=True, stdout=PIPE, stderr=PIPE,
                              cwd=self.cwd, env=env)
         stdout, stderr = self.process.communicate()
         if DEBUG:
             log.debug('STDOUT:%s' % (stdout,))
             log.debug('STDERR:%s' % (stderr,))
         return stdout, stderr
 
     def assert_returncode_success(self):
         assert self.process.returncode == 0
 
 
 def _add_files_and_push(vcs, dest, clone_url=None, **kwargs):
     """
     Generate some files, add it to DEST repo and push back
     vcs is git or hg and defines what VCS we want to make those files for
     """
     # commit some stuff into this repo
     cwd = path = jn(dest)
     added_file = jn(path, '%ssetup.py' % tempfile._RandomNameSequence().next())
     Command(cwd).execute('touch %s' % added_file)
     Command(cwd).execute('%s add %s' % (vcs, added_file))
     author_str = 'Marcin Kuźminski <[e-mail]>'
 
     git_ident = "git config user.name {} && git config user.email {}".format(
             'Marcin Kuźminski', '[e-mail]')
 
     for i in xrange(kwargs.get('files_no', 3)):
         cmd = """echo 'added_line%s' >> %s""" % (i, added_file)
         Command(cwd).execute(cmd)
         if vcs == 'hg':
             cmd = """hg commit -m 'commited new %s' -u '%s' %s """ % (
                 i, author_str, added_file
             )
         elif vcs == 'git':
             cmd = """%s && git commit -m 'commited new %s' %s""" % (
                 git_ident, i, added_file)
         Command(cwd).execute(cmd)
 
     # PUSH it back
     stdout = stderr = None
     if vcs == 'hg':
         stdout, stderr = Command(cwd).execute(
             'hg push --verbose', clone_url)
     elif vcs == 'git':
         stdout, stderr = Command(cwd).execute(
             """%s && git push --verbose %s master""" % (
                 git_ident, clone_url))
 
     return stdout, stderr
 
 
 def _check_proper_git_push(
         stdout, stderr, branch='master', should_set_default_branch=False):
     # Note: Git is writing most information to stderr intentionally
     assert 'fatal' not in stderr
     assert 'rejected' not in stderr
     assert 'Pushing to' in stderr
     assert '%s -> %s' % (branch, branch) in stderr
 
     if should_set_default_branch:
         assert "Setting default branch to %s" % branch in stderr
     else:
         assert "Setting default branch" not in stderr
 
 
+def _check_proper_hg_push(stdout, stderr, branch='default'):
+    assert 'pushing to' in stdout
+    assert 'searching for changes' in stdout
+
+    assert 'abort:' not in stderr
+
+
 def _check_proper_clone(stdout, stderr, vcs):
     if vcs == 'hg':
         assert 'requesting all changes' in stdout
         assert 'adding changesets' in stdout
         assert 'adding manifests' in stdout
         assert 'adding file changes' in stdout
 
         assert stderr == ''
 
     if vcs == 'git':
         assert '' == stdout
         assert 'Cloning into' in stderr
         assert 'abort:' not in stderr
         assert 'fatal:' not in stderr

rhodecode/tests/other/vcs_operations/conftest.py

@@ -1,267 +1,270 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 py.test config for test suite for making push/pull operations.
 
 .. important::
 
    You must have git >= 1.8.5 for tests to work fine. With 68b939b git started
    to redirect things to stderr instead of stdout.
 """
 
 import ConfigParser
 import os
 import subprocess32
 import tempfile
 import textwrap
 import pytest
 
 import rhodecode
 from rhodecode.model.db import Repository
 from rhodecode.model.meta import Session
 from rhodecode.model.settings import SettingsModel
 from rhodecode.tests import (
     GIT_REPO, HG_REPO, TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS,)
 from rhodecode.tests.fixture import Fixture
 from rhodecode.tests.utils import (
     set_anonymous_access, is_url_reachable, wait_for_url)
 
 RC_LOG = os.path.join(tempfile.gettempdir(), 'rc.log')
 REPO_GROUP = 'a_repo_group'
 HG_REPO_WITH_GROUP = '%s/%s' % (REPO_GROUP, HG_REPO)
 GIT_REPO_WITH_GROUP = '%s/%s' % (REPO_GROUP, GIT_REPO)
 
 
 def assert_no_running_instance(url):
     if is_url_reachable(url):
         print("Hint: Usually this means another instance of Enterprise "
               "is running in the background.")
         pytest.fail(
             "Port is not free at %s, cannot start web interface" % url)
 
 
 def get_host_url(pylons_config):
     """Construct the host url using the port in the test configuration."""
     config = ConfigParser.ConfigParser()
     config.read(pylons_config)
 
     return '127.0.0.1:%s' % config.get('server:main', 'port')
 
 
 class RcWebServer(object):
     """
     Represents a running RCE web server used as a test fixture.
     """
     def __init__(self, pylons_config):
         self.pylons_config = pylons_config
 
     def repo_clone_url(self, repo_name, **kwargs):
         params = {
             'user': TEST_USER_ADMIN_LOGIN,
             'passwd': TEST_USER_ADMIN_PASS,
             'host': get_host_url(self.pylons_config),
             'cloned_repo': repo_name,
         }
         params.update(**kwargs)
         _url = 'http://%(user)s:%(passwd)s@%(host)s/%(cloned_repo)s' % params
         return _url
 
+    def host_url(self):
+        return 'http://' + get_host_url(self.pylons_config)
+
 
 @pytest.fixture(scope="module")
 def rcextensions(request, pylonsapp, tmpdir_factory):
     """
     Installs a testing rcextensions pack to ensure they work as expected.
     """
     init_content = textwrap.dedent("""
         # Forward import the example rcextensions to make it
         # active for our tests.
         from rhodecode.tests.other.example_rcextensions import *
     """)
 
     # Note: rcextensions are looked up based on the path of the ini file
     root_path = tmpdir_factory.getbasetemp()
     rcextensions_path = root_path.join('rcextensions')
     init_path = rcextensions_path.join('__init__.py')
 
     if rcextensions_path.check():
         pytest.fail(
             "Path for rcextensions already exists, please clean up before "
             "test run this path: %s" % (rcextensions_path, ))
         return
 
     request.addfinalizer(rcextensions_path.remove)
     init_path.write_binary(init_content, ensure=True)
 
 
 @pytest.fixture(scope="module")
 def repos(request, pylonsapp):
     """Create a copy of each test repo in a repo group."""
     fixture = Fixture()
     repo_group = fixture.create_repo_group(REPO_GROUP)
     repo_group_id = repo_group.group_id
     fixture.create_fork(HG_REPO, HG_REPO,
                         repo_name_full=HG_REPO_WITH_GROUP,
                         repo_group=repo_group_id)
     fixture.create_fork(GIT_REPO, GIT_REPO,
                         repo_name_full=GIT_REPO_WITH_GROUP,
                         repo_group=repo_group_id)
 
     @request.addfinalizer
     def cleanup():
         fixture.destroy_repo(HG_REPO_WITH_GROUP)
         fixture.destroy_repo(GIT_REPO_WITH_GROUP)
         fixture.destroy_repo_group(repo_group_id)
 
 
 @pytest.fixture(scope="module")
 def rc_web_server_config(testini_factory):
     """
     Configuration file used for the fixture `rc_web_server`.
     """
     CUSTOM_PARAMS = [
         {'handler_console': {'level': 'DEBUG'}},
     ]
     return testini_factory(CUSTOM_PARAMS)
 
 
 @pytest.fixture(scope="module")
 def rc_web_server(
         request, pylonsapp, rc_web_server_config, repos, rcextensions):
     """
     Run the web server as a subprocess.
 
     Since we have already a running vcsserver, this is not spawned again.
     """
     env = os.environ.copy()
     env['RC_NO_TMP_PATH'] = '1'
 
     rc_log = RC_LOG
     server_out = open(rc_log, 'w')
 
     # TODO: Would be great to capture the output and err of the subprocess
     # and make it available in a section of the py.test report in case of an
     # error.
 
     host_url = 'http://' + get_host_url(rc_web_server_config)
     assert_no_running_instance(host_url)
     command = ['pserve', rc_web_server_config]
 
     print('Starting rcserver: {}'.format(host_url))
     print('Command: {}'.format(command))
     print('Logfile: {}'.format(rc_log))
 
     proc = subprocess32.Popen(
         command, bufsize=0, env=env, stdout=server_out, stderr=server_out)
 
     wait_for_url(host_url, timeout=30)
 
     @request.addfinalizer
     def stop_web_server():
         # TODO: Find out how to integrate with the reporting of py.test to
         # make this information available.
         print("\nServer log file written to %s" % (rc_log, ))
         proc.kill()
         server_out.flush()
         server_out.close()
 
     return RcWebServer(rc_web_server_config)
 
 
 @pytest.fixture(scope='class', autouse=True)
 def disable_anonymous_user_access(pylonsapp):
     set_anonymous_access(False)
 
 
 @pytest.fixture
 def disable_locking(pylonsapp):
     r = Repository.get_by_repo_name(GIT_REPO)
     Repository.unlock(r)
     r.enable_locking = False
     Session().add(r)
     Session().commit()
 
     r = Repository.get_by_repo_name(HG_REPO)
     Repository.unlock(r)
     r.enable_locking = False
     Session().add(r)
     Session().commit()
 
 
 @pytest.fixture
 def enable_auth_plugins(request, pylonsapp, csrf_token):
     """
     Return a factory object that when called, allows to control which
     authentication plugins are enabled.
     """
     def _enable_plugins(plugins_list, override=None):
         override = override or {}
         params = {
             'auth_plugins': ','.join(plugins_list),
         }
 
         # helper translate some names to others
         name_map = {
             'token': 'authtoken'
         }
 
         for module in plugins_list:
             plugin_name = module.partition('#')[-1]
             if plugin_name in name_map:
                 plugin_name = name_map[plugin_name]
             enabled_plugin = 'auth_%s_enabled' % plugin_name
             cache_ttl = 'auth_%s_cache_ttl' % plugin_name
 
             # default params that are needed for each plugin,
             # `enabled` and `cache_ttl`
             params.update({
                 enabled_plugin: True,
                 cache_ttl: 0
             })
             if override.get:
                 params.update(override.get(module, {}))
 
             validated_params = params
             for k, v in validated_params.items():
                 setting = SettingsModel().create_or_update_setting(k, v)
                 Session().add(setting)
             Session().commit()
 
     def cleanup():
         _enable_plugins(['egg:rhodecode-enterprise-ce#rhodecode'])
 
     request.addfinalizer(cleanup)
 
     return _enable_plugins
 
 
 @pytest.fixture
 def fs_repo_only(request, rhodecode_fixtures):
     def fs_repo_fabric(repo_name, repo_type):
         rhodecode_fixtures.create_repo(repo_name, repo_type=repo_type)
         rhodecode_fixtures.destroy_repo(repo_name, fs_remove=False)
 
         def cleanup():
             rhodecode_fixtures.destroy_repo(repo_name, fs_remove=True)
             rhodecode_fixtures.destroy_repo_on_filesystem(repo_name)
 
         request.addfinalizer(cleanup)
 
     return fs_repo_fabric

rhodecode/tests/other/vcs_operations/test_vcs_operations.py

@@ -1,481 +1,655 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Test suite for making push/pull operations, on specially modified INI files
 
 .. important::
 
    You must have git >= 1.8.5 for tests to work fine. With 68b939b git started
    to redirect things to stderr instead of stdout.
 """
 
 
 import os
 import time
 
 import pytest
 
 from rhodecode.lib.vcs.backends.git.repository import GitRepository
+from rhodecode.lib.vcs.backends.hg.repository import MercurialRepository
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.model.auth_token import AuthTokenModel
 from rhodecode.model.db import Repository, UserIpMap, CacheKey
 from rhodecode.model.meta import Session
 from rhodecode.model.user import UserModel
 from rhodecode.tests import (GIT_REPO, HG_REPO, TEST_USER_ADMIN_LOGIN)
 
 from rhodecode.tests.other.vcs_operations import (
-    Command, _check_proper_clone, _check_proper_git_push, _add_files_and_push,
+    Command, _check_proper_clone, _check_proper_git_push,
+    _check_proper_hg_push, _add_files_and_push,
     HG_REPO_WITH_GROUP, GIT_REPO_WITH_GROUP)
 
 
 @pytest.mark.usefixtures("disable_locking")
 class TestVCSOperations(object):
 
     def test_clone_hg_repo_by_admin(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(HG_REPO)
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         _check_proper_clone(stdout, stderr, 'hg')
 
     def test_clone_git_repo_by_admin(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(GIT_REPO)
         cmd = Command('/tmp')
         stdout, stderr = cmd.execute('git clone', clone_url, tmpdir.strpath)
         _check_proper_clone(stdout, stderr, 'git')
         cmd.assert_returncode_success()
 
     def test_clone_git_repo_by_admin_with_git_suffix(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(GIT_REPO)
         cmd = Command('/tmp')
         stdout, stderr = cmd.execute('git clone', clone_url+".git", tmpdir.strpath)
         _check_proper_clone(stdout, stderr, 'git')
         cmd.assert_returncode_success()
 
     def test_clone_hg_repo_by_id_by_admin(self, rc_web_server, tmpdir):
         repo_id = Repository.get_by_repo_name(HG_REPO).repo_id
         clone_url = rc_web_server.repo_clone_url('_%s' % repo_id)
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         _check_proper_clone(stdout, stderr, 'hg')
 
     def test_clone_git_repo_by_id_by_admin(self, rc_web_server, tmpdir):
         repo_id = Repository.get_by_repo_name(GIT_REPO).repo_id
         clone_url = rc_web_server.repo_clone_url('_%s' % repo_id)
         cmd = Command('/tmp')
         stdout, stderr = cmd.execute('git clone', clone_url, tmpdir.strpath)
         _check_proper_clone(stdout, stderr, 'git')
         cmd.assert_returncode_success()
 
     def test_clone_hg_repo_with_group_by_admin(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(HG_REPO_WITH_GROUP)
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         _check_proper_clone(stdout, stderr, 'hg')
 
     def test_clone_git_repo_with_group_by_admin(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(GIT_REPO_WITH_GROUP)
         cmd = Command('/tmp')
         stdout, stderr = cmd.execute('git clone', clone_url, tmpdir.strpath)
         _check_proper_clone(stdout, stderr, 'git')
         cmd.assert_returncode_success()
 
     def test_clone_git_repo_shallow_by_admin(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(GIT_REPO)
         cmd = Command('/tmp')
         stdout, stderr = cmd.execute(
             'git clone --depth=1', clone_url, tmpdir.strpath)
 
         assert '' == stdout
         assert 'Cloning into' in stderr
         cmd.assert_returncode_success()
 
     def test_clone_wrong_credentials_hg(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(HG_REPO, passwd='bad!')
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         assert 'abort: authorization failed' in stderr
 
     def test_clone_wrong_credentials_git(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(GIT_REPO, passwd='bad!')
         stdout, stderr = Command('/tmp').execute(
             'git clone', clone_url, tmpdir.strpath)
         assert 'fatal: Authentication failed' in stderr
 
     def test_clone_git_dir_as_hg(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(GIT_REPO)
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         assert 'HTTP Error 404: Not Found' in stderr
 
     def test_clone_hg_repo_as_git(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(HG_REPO)
         stdout, stderr = Command('/tmp').execute(
             'git clone', clone_url, tmpdir.strpath)
         assert 'not found' in stderr
 
     def test_clone_non_existing_path_hg(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url('trololo')
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         assert 'HTTP Error 404: Not Found' in stderr
 
     def test_clone_non_existing_path_git(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url('trololo')
         stdout, stderr = Command('/tmp').execute('git clone', clone_url)
         assert 'not found' in stderr
 
     def test_clone_existing_path_hg_not_in_database(
             self, rc_web_server, tmpdir, fs_repo_only):
 
         db_name = fs_repo_only('not-in-db-hg', repo_type='hg')
         clone_url = rc_web_server.repo_clone_url(db_name)
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         assert 'HTTP Error 404: Not Found' in stderr
 
     def test_clone_existing_path_git_not_in_database(
             self, rc_web_server, tmpdir, fs_repo_only):
         db_name = fs_repo_only('not-in-db-git', repo_type='git')
         clone_url = rc_web_server.repo_clone_url(db_name)
         stdout, stderr = Command('/tmp').execute(
             'git clone', clone_url, tmpdir.strpath)
         assert 'not found' in stderr
 
     def test_clone_existing_path_hg_not_in_database_different_scm(
             self, rc_web_server, tmpdir, fs_repo_only):
         db_name = fs_repo_only('not-in-db-git', repo_type='git')
         clone_url = rc_web_server.repo_clone_url(db_name)
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         assert 'HTTP Error 404: Not Found' in stderr
 
     def test_clone_existing_path_git_not_in_database_different_scm(
             self, rc_web_server, tmpdir, fs_repo_only):
         db_name = fs_repo_only('not-in-db-hg', repo_type='hg')
         clone_url = rc_web_server.repo_clone_url(db_name)
         stdout, stderr = Command('/tmp').execute(
             'git clone', clone_url, tmpdir.strpath)
         assert 'not found' in stderr
 
     def test_push_new_file_hg(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(HG_REPO)
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
 
         stdout, stderr = _add_files_and_push(
             'hg', tmpdir.strpath, clone_url=clone_url)
 
         assert 'pushing to' in stdout
         assert 'size summary' in stdout
 
     def test_push_new_file_git(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(GIT_REPO)
         stdout, stderr = Command('/tmp').execute(
             'git clone', clone_url, tmpdir.strpath)
 
         # commit some stuff into this repo
         stdout, stderr = _add_files_and_push(
             'git', tmpdir.strpath, clone_url=clone_url)
 
         _check_proper_git_push(stdout, stderr)
 
     def test_push_invalidates_cache_hg(self, rc_web_server, tmpdir):
         key = CacheKey.query().filter(CacheKey.cache_key == HG_REPO).scalar()
         if not key:
             key = CacheKey(HG_REPO, HG_REPO)
 
         key.cache_active = True
         Session().add(key)
         Session().commit()
 
         clone_url = rc_web_server.repo_clone_url(HG_REPO)
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
 
         stdout, stderr = _add_files_and_push(
             'hg', tmpdir.strpath, clone_url=clone_url, files_no=1)
 
         key = CacheKey.query().filter(CacheKey.cache_key == HG_REPO).one()
         assert key.cache_active is False
 
     def test_push_invalidates_cache_git(self, rc_web_server, tmpdir):
         key = CacheKey.query().filter(CacheKey.cache_key == GIT_REPO).scalar()
         if not key:
             key = CacheKey(GIT_REPO, GIT_REPO)
 
         key.cache_active = True
         Session().add(key)
         Session().commit()
 
         clone_url = rc_web_server.repo_clone_url(GIT_REPO)
         stdout, stderr = Command('/tmp').execute(
             'git clone', clone_url, tmpdir.strpath)
 
         # commit some stuff into this repo
         stdout, stderr = _add_files_and_push(
             'git', tmpdir.strpath, clone_url=clone_url, files_no=1)
         _check_proper_git_push(stdout, stderr)
 
         key = CacheKey.query().filter(CacheKey.cache_key == GIT_REPO).one()
 
         assert key.cache_active is False
 
     def test_push_wrong_credentials_hg(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(HG_REPO)
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
 
         push_url = rc_web_server.repo_clone_url(
             HG_REPO, user='bad', passwd='name')
         stdout, stderr = _add_files_and_push(
             'hg', tmpdir.strpath, clone_url=push_url)
 
         assert 'abort: authorization failed' in stderr
 
     def test_push_wrong_credentials_git(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(GIT_REPO)
         stdout, stderr = Command('/tmp').execute(
             'git clone', clone_url, tmpdir.strpath)
 
         push_url = rc_web_server.repo_clone_url(
             GIT_REPO, user='bad', passwd='name')
         stdout, stderr = _add_files_and_push(
             'git', tmpdir.strpath, clone_url=push_url)
 
         assert 'fatal: Authentication failed' in stderr
 
     def test_push_back_to_wrong_url_hg(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(HG_REPO)
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
 
         stdout, stderr = _add_files_and_push(
             'hg', tmpdir.strpath,
             clone_url=rc_web_server.repo_clone_url('not-existing'))
 
         assert 'HTTP Error 404: Not Found' in stderr
 
     def test_push_back_to_wrong_url_git(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(GIT_REPO)
         stdout, stderr = Command('/tmp').execute(
             'git clone', clone_url, tmpdir.strpath)
 
         stdout, stderr = _add_files_and_push(
             'git', tmpdir.strpath,
             clone_url=rc_web_server.repo_clone_url('not-existing'))
 
         assert 'not found' in stderr
 
     def test_ip_restriction_hg(self, rc_web_server, tmpdir):
         user_model = UserModel()
         try:
             user_model.add_extra_ip(TEST_USER_ADMIN_LOGIN, '10.10.10.10/32')
             Session().commit()
             time.sleep(2)
             clone_url = rc_web_server.repo_clone_url(HG_REPO)
             stdout, stderr = Command('/tmp').execute(
                 'hg clone', clone_url, tmpdir.strpath)
             assert 'abort: HTTP Error 403: Forbidden' in stderr
         finally:
             # release IP restrictions
             for ip in UserIpMap.getAll():
                 UserIpMap.delete(ip.ip_id)
             Session().commit()
 
         time.sleep(2)
 
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         _check_proper_clone(stdout, stderr, 'hg')
 
     def test_ip_restriction_git(self, rc_web_server, tmpdir):
         user_model = UserModel()
         try:
             user_model.add_extra_ip(TEST_USER_ADMIN_LOGIN, '10.10.10.10/32')
             Session().commit()
             time.sleep(2)
             clone_url = rc_web_server.repo_clone_url(GIT_REPO)
             stdout, stderr = Command('/tmp').execute(
                 'git clone', clone_url, tmpdir.strpath)
             msg = "The requested URL returned error: 403"
             assert msg in stderr
         finally:
             # release IP restrictions
             for ip in UserIpMap.getAll():
                 UserIpMap.delete(ip.ip_id)
             Session().commit()
 
         time.sleep(2)
 
         cmd = Command('/tmp')
         stdout, stderr = cmd.execute('git clone', clone_url, tmpdir.strpath)
         cmd.assert_returncode_success()
         _check_proper_clone(stdout, stderr, 'git')
 
     def test_clone_by_auth_token(
             self, rc_web_server, tmpdir, user_util, enable_auth_plugins):
         enable_auth_plugins(['egg:rhodecode-enterprise-ce#token',
                              'egg:rhodecode-enterprise-ce#rhodecode'])
 
         user = user_util.create_user()
         token = user.auth_tokens[1]
 
         clone_url = rc_web_server.repo_clone_url(
             HG_REPO, user=user.username, passwd=token)
 
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         _check_proper_clone(stdout, stderr, 'hg')
 
     def test_clone_by_auth_token_expired(
             self, rc_web_server, tmpdir, user_util, enable_auth_plugins):
         enable_auth_plugins(['egg:rhodecode-enterprise-ce#token',
                              'egg:rhodecode-enterprise-ce#rhodecode'])
 
         user = user_util.create_user()
         auth_token = AuthTokenModel().create(
             user.user_id, 'test-token', -10, AuthTokenModel.cls.ROLE_VCS)
         token = auth_token.api_key
 
         clone_url = rc_web_server.repo_clone_url(
             HG_REPO, user=user.username, passwd=token)
 
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         assert 'abort: authorization failed' in stderr
 
     def test_clone_by_auth_token_bad_role(
             self, rc_web_server, tmpdir, user_util, enable_auth_plugins):
         enable_auth_plugins(['egg:rhodecode-enterprise-ce#token',
                              'egg:rhodecode-enterprise-ce#rhodecode'])
 
         user = user_util.create_user()
         auth_token = AuthTokenModel().create(
             user.user_id, 'test-token', -1, AuthTokenModel.cls.ROLE_API)
         token = auth_token.api_key
 
         clone_url = rc_web_server.repo_clone_url(
             HG_REPO, user=user.username, passwd=token)
 
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         assert 'abort: authorization failed' in stderr
 
     def test_clone_by_auth_token_user_disabled(
             self, rc_web_server, tmpdir, user_util, enable_auth_plugins):
         enable_auth_plugins(['egg:rhodecode-enterprise-ce#token',
                              'egg:rhodecode-enterprise-ce#rhodecode'])
         user = user_util.create_user()
         user.active = False
         Session().add(user)
         Session().commit()
         token = user.auth_tokens[1]
 
         clone_url = rc_web_server.repo_clone_url(
             HG_REPO, user=user.username, passwd=token)
 
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         assert 'abort: authorization failed' in stderr
 
-
     def test_clone_by_auth_token_with_scope(
             self, rc_web_server, tmpdir, user_util, enable_auth_plugins):
         enable_auth_plugins(['egg:rhodecode-enterprise-ce#token',
                              'egg:rhodecode-enterprise-ce#rhodecode'])
         user = user_util.create_user()
         auth_token = AuthTokenModel().create(
             user.user_id, 'test-token', -1, AuthTokenModel.cls.ROLE_VCS)
         token = auth_token.api_key
 
         # manually set scope
         auth_token.repo = Repository.get_by_repo_name(HG_REPO)
         Session().add(auth_token)
         Session().commit()
 
         clone_url = rc_web_server.repo_clone_url(
             HG_REPO, user=user.username, passwd=token)
 
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         _check_proper_clone(stdout, stderr, 'hg')
 
     def test_clone_by_auth_token_with_wrong_scope(
             self, rc_web_server, tmpdir, user_util, enable_auth_plugins):
         enable_auth_plugins(['egg:rhodecode-enterprise-ce#token',
                              'egg:rhodecode-enterprise-ce#rhodecode'])
         user = user_util.create_user()
         auth_token = AuthTokenModel().create(
             user.user_id, 'test-token', -1, AuthTokenModel.cls.ROLE_VCS)
         token = auth_token.api_key
 
         # manually set scope
         auth_token.repo = Repository.get_by_repo_name(GIT_REPO)
         Session().add(auth_token)
         Session().commit()
 
         clone_url = rc_web_server.repo_clone_url(
             HG_REPO, user=user.username, passwd=token)
 
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         assert 'abort: authorization failed' in stderr
 
 
 def test_git_sets_default_branch_if_not_master(
         backend_git, tmpdir, disable_locking, rc_web_server):
     empty_repo = backend_git.create_repo()
     clone_url = rc_web_server.repo_clone_url(empty_repo.repo_name)
 
     cmd = Command(tmpdir.strpath)
     cmd.execute('git clone', clone_url)
 
     repo = GitRepository(os.path.join(tmpdir.strpath, empty_repo.repo_name))
     repo.in_memory_commit.add(FileNode('file', content=''))
     repo.in_memory_commit.commit(
         message='Commit on branch test',
         author='Automatic test',
         branch='test')
 
     repo_cmd = Command(repo.path)
     stdout, stderr = repo_cmd.execute('git push --verbose origin test')
     _check_proper_git_push(
         stdout, stderr, branch='test', should_set_default_branch=True)
 
     stdout, stderr = cmd.execute(
         'git clone', clone_url, empty_repo.repo_name + '-clone')
     _check_proper_clone(stdout, stderr, 'git')
 
     # Doing an explicit commit in order to get latest user logs on MySQL
     Session().commit()
 
 
 def test_git_fetches_from_remote_repository_with_annotated_tags(
         backend_git, disable_locking, rc_web_server):
     # Note: This is a test specific to the git backend. It checks the
     # integration of fetching from a remote repository which contains
     # annotated tags.
 
     # Dulwich shows this specific behavior only when
     # operating against a remote repository.
     source_repo = backend_git['annotated-tag']
     target_vcs_repo = backend_git.create_repo().scm_instance()
     target_vcs_repo.fetch(rc_web_server.repo_clone_url(source_repo.repo_name))
+
+
+def test_git_push_shows_pull_request_refs(backend_git, rc_web_server, tmpdir):
+    """
+    test if remote info about refs is visible
+    """
+    empty_repo = backend_git.create_repo()
+
+    clone_url = rc_web_server.repo_clone_url(empty_repo.repo_name)
+
+    cmd = Command(tmpdir.strpath)
+    cmd.execute('git clone', clone_url)
+
+    repo = GitRepository(os.path.join(tmpdir.strpath, empty_repo.repo_name))
+    repo.in_memory_commit.add(FileNode('readme.md', content='## Hello'))
+    repo.in_memory_commit.commit(
+        message='Commit on branch Master',
+        author='Automatic test',
+        branch='master')
+
+    repo_cmd = Command(repo.path)
+    stdout, stderr = repo_cmd.execute('git push --verbose origin master')
+    _check_proper_git_push(stdout, stderr, branch='master')
+
+    ref = '{}/{}/pull-request/new?branch=master'.format(
+        rc_web_server.host_url(), empty_repo.repo_name)
+    assert 'remote: RhodeCode: open pull request link: {}'.format(ref) in stderr
+    assert 'remote: RhodeCode: push completed' in stderr
+
+    # push on the same branch
+    repo = GitRepository(os.path.join(tmpdir.strpath, empty_repo.repo_name))
+    repo.in_memory_commit.add(FileNode('setup.py', content='print\n'))
+    repo.in_memory_commit.commit(
+        message='Commit2 on branch Master',
+        author='Automatic test2',
+        branch='master')
+
+    repo_cmd = Command(repo.path)
+    stdout, stderr = repo_cmd.execute('git push --verbose origin master')
+    _check_proper_git_push(stdout, stderr, branch='master')
+
+    assert 'remote: RhodeCode: open pull request link: {}'.format(ref) in stderr
+    assert 'remote: RhodeCode: push completed' in stderr
+
+    # new Branch
+    repo = GitRepository(os.path.join(tmpdir.strpath, empty_repo.repo_name))
+    repo.in_memory_commit.add(FileNode('feature1.py', content='## Hello world'))
+    repo.in_memory_commit.commit(
+        message='Commit on branch feature',
+        author='Automatic test',
+        branch='feature')
+
+    repo_cmd = Command(repo.path)
+    stdout, stderr = repo_cmd.execute('git push --verbose origin feature')
+    _check_proper_git_push(stdout, stderr, branch='feature')
+
+    ref = '{}/{}/pull-request/new?branch=feature'.format(
+        rc_web_server.host_url(), empty_repo.repo_name)
+    assert 'remote: RhodeCode: open pull request link: {}'.format(ref) in stderr
+    assert 'remote: RhodeCode: push completed' in stderr
+
+
+def test_hg_push_shows_pull_request_refs(backend_hg, rc_web_server, tmpdir):
+    empty_repo = backend_hg.create_repo()
+
+    clone_url = rc_web_server.repo_clone_url(empty_repo.repo_name)
+
+    cmd = Command(tmpdir.strpath)
+    cmd.execute('hg clone', clone_url)
+
+    repo = MercurialRepository(os.path.join(tmpdir.strpath, empty_repo.repo_name))
+    repo.in_memory_commit.add(FileNode(u'readme.md', content=u'## Hello'))
+    repo.in_memory_commit.commit(
+        message=u'Commit on branch default',
+        author=u'Automatic test',
+        branch='default')
+
+    repo_cmd = Command(repo.path)
+    repo_cmd.execute('hg checkout default')
+
+    stdout, stderr = repo_cmd.execute('hg push --verbose', clone_url)
+    _check_proper_hg_push(stdout, stderr, branch='default')
+
+    ref = '{}/{}/pull-request/new?branch=default'.format(
+        rc_web_server.host_url(), empty_repo.repo_name)
+    assert 'remote: RhodeCode: open pull request link: {}'.format(ref) in stdout
+    assert 'remote: RhodeCode: push completed' in stdout
+
+    # push on the same branch
+    repo = MercurialRepository(os.path.join(tmpdir.strpath, empty_repo.repo_name))
+    repo.in_memory_commit.add(FileNode(u'setup.py', content=u'print\n'))
+    repo.in_memory_commit.commit(
+        message=u'Commit2 on branch default',
+        author=u'Automatic test2',
+        branch=u'default')
+
+    repo_cmd = Command(repo.path)
+    repo_cmd.execute('hg checkout default')
+
+    stdout, stderr = repo_cmd.execute('hg push --verbose', clone_url)
+    _check_proper_hg_push(stdout, stderr, branch='default')
+
+    assert 'remote: RhodeCode: open pull request link: {}'.format(ref) in stdout
+    assert 'remote: RhodeCode: push completed' in stdout
+
+    # new Branch
+    repo = MercurialRepository(os.path.join(tmpdir.strpath, empty_repo.repo_name))
+    repo.in_memory_commit.add(FileNode(u'feature1.py', content=u'## Hello world'))
+    repo.in_memory_commit.commit(
+        message=u'Commit on branch feature',
+        author=u'Automatic test',
+        branch=u'feature')
+
+    repo_cmd = Command(repo.path)
+    repo_cmd.execute('hg checkout feature')
+
+    stdout, stderr = repo_cmd.execute('hg push --new-branch --verbose', clone_url)
+    _check_proper_hg_push(stdout, stderr, branch='feature')
+
+    ref = '{}/{}/pull-request/new?branch=feature'.format(
+        rc_web_server.host_url(), empty_repo.repo_name)
+    assert 'remote: RhodeCode: open pull request link: {}'.format(ref) in stdout
+    assert 'remote: RhodeCode: push completed' in stdout
+
+
+def test_hg_push_shows_pull_request_refs_book(backend_hg, rc_web_server, tmpdir):
+    empty_repo = backend_hg.create_repo()
+
+    clone_url = rc_web_server.repo_clone_url(empty_repo.repo_name)
+
+    cmd = Command(tmpdir.strpath)
+    cmd.execute('hg clone', clone_url)
+
+    repo = MercurialRepository(os.path.join(tmpdir.strpath, empty_repo.repo_name))
+    repo.in_memory_commit.add(FileNode(u'readme.md', content=u'## Hello'))
+    repo.in_memory_commit.commit(
+        message=u'Commit on branch default',
+        author=u'Automatic test',
+        branch='default')
+
+    repo_cmd = Command(repo.path)
+    repo_cmd.execute('hg checkout default')
+
+    stdout, stderr = repo_cmd.execute('hg push --verbose', clone_url)
+    _check_proper_hg_push(stdout, stderr, branch='default')
+
+    ref = '{}/{}/pull-request/new?branch=default'.format(
+        rc_web_server.host_url(), empty_repo.repo_name)
+    assert 'remote: RhodeCode: open pull request link: {}'.format(ref) in stdout
+    assert 'remote: RhodeCode: push completed' in stdout
+
+    # add bookmark
+    repo = MercurialRepository(os.path.join(tmpdir.strpath, empty_repo.repo_name))
+    repo.in_memory_commit.add(FileNode(u'setup.py', content=u'print\n'))
+    repo.in_memory_commit.commit(
+        message=u'Commit2 on branch default',
+        author=u'Automatic test2',
+        branch=u'default')
+
+    repo_cmd = Command(repo.path)
+    repo_cmd.execute('hg checkout default')
+    repo_cmd.execute('hg bookmark feature2')
+    stdout, stderr = repo_cmd.execute('hg push -B feature2 --verbose', clone_url)
+    _check_proper_hg_push(stdout, stderr, branch='default')
+
+    ref = '{}/{}/pull-request/new?branch=default'.format(
+        rc_web_server.host_url(), empty_repo.repo_name)
+    assert 'remote: RhodeCode: open pull request link: {}'.format(ref) in stdout
+    ref = '{}/{}/pull-request/new?bookmark=feature2'.format(
+        rc_web_server.host_url(), empty_repo.repo_name)
+    assert 'remote: RhodeCode: open pull request link: {}'.format(ref) in stdout
+    assert 'remote: RhodeCode: push completed' in stdout
+    assert 'exporting bookmark feature2' in stdout