simplevcs: handle a case of damaged filesystem repo without 500 exception.

rhodecode/lib/middleware/simplevcs.py

@@ -1,609 +1,610 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 SimpleVCS middleware for handling protocol request (push/clone etc.)
 It's implemented with basic auth function
 """
 
 import os
 import re
 import logging
 import importlib
 from functools import wraps
 
 import time
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 # TODO(marcink): check if we should use webob.exc here ?
 from pyramid.httpexceptions import (
     HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
 
 import rhodecode
 from rhodecode.authentication.base import (
     authenticate, get_perms_cache_manager, VCS_TYPE)
 from rhodecode.lib import caches
 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
 from rhodecode.lib.base import (
     BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
 from rhodecode.lib.exceptions import (
     HTTPLockedRC, HTTPRequirementError, UserCreationError,
     NotAllowedToCreateUserError)
 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
 from rhodecode.lib.middleware import appenlight
 from rhodecode.lib.middleware.utils import scm_app_http
 from rhodecode.lib.utils import is_valid_repo, SLUG_RE
 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool, safe_unicode
 from rhodecode.lib.vcs.conf import settings as vcs_settings
 from rhodecode.lib.vcs.backends import base
 from rhodecode.model import meta
 from rhodecode.model.db import User, Repository, PullRequest
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.pull_request import PullRequestModel
 from rhodecode.model.settings import SettingsModel, VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
 
 def initialize_generator(factory):
     """
     Initializes the returned generator by draining its first element.
 
     This can be used to give a generator an initializer, which is the code
     up to the first yield statement. This decorator enforces that the first
     produced element has the value ``"__init__"`` to make its special
     purpose very explicit in the using code.
     """
 
     @wraps(factory)
     def wrapper(*args, **kwargs):
         gen = factory(*args, **kwargs)
         try:
             init = gen.next()
         except StopIteration:
             raise ValueError('Generator must yield at least one element.')
         if init != "__init__":
             raise ValueError('First yielded element must be "__init__".')
         return gen
     return wrapper
 
 
 class SimpleVCS(object):
     """Common functionality for SCM HTTP handlers."""
 
     SCM = 'unknown'
 
     acl_repo_name = None
     url_repo_name = None
     vcs_repo_name = None
 
     # We have to handle requests to shadow repositories different than requests
     # to normal repositories. Therefore we have to distinguish them. To do this
     # we use this regex which will match only on URLs pointing to shadow
     # repositories.
     shadow_repo_re = re.compile(
         '(?P<groups>(?:{slug_pat}/)*)'  # repo groups
         '(?P<target>{slug_pat})/'       # target repo
         'pull-request/(?P<pr_id>\d+)/'  # pull request
         'repository$'                   # shadow repo
         .format(slug_pat=SLUG_RE.pattern))
 
     def __init__(self, config, registry):
         self.registry = registry
         self.config = config
         # re-populated by specialized middleware
         self.repo_vcs_config = base.Config()
         self.rhodecode_settings = SettingsModel().get_all_settings(cache=True)
 
         registry.rhodecode_settings = self.rhodecode_settings
         # authenticate this VCS request using authfunc
         auth_ret_code_detection = \
             str2bool(self.config.get('auth_ret_code_detection', False))
         self.authenticate = BasicAuth(
             '', authenticate, registry, config.get('auth_ret_code'),
             auth_ret_code_detection)
         self.ip_addr = '0.0.0.0'
 
     @property
     def base_path(self):
         settings_path = self.repo_vcs_config.get(*VcsSettingsModel.PATH_SETTING)
         if not settings_path:
             # try, maybe we passed in explicitly as config option
             settings_path = self.config.get('base_path')
         return settings_path
 
     def set_repo_names(self, environ):
         """
         This will populate the attributes acl_repo_name, url_repo_name,
         vcs_repo_name and is_shadow_repo. In case of requests to normal (non
         shadow) repositories all names are equal. In case of requests to a
         shadow repository the acl-name points to the target repo of the pull
         request and the vcs-name points to the shadow repo file system path.
         The url-name is always the URL used by the vcs client program.
 
         Example in case of a shadow repo:
             acl_repo_name = RepoGroup/MyRepo
             url_repo_name = RepoGroup/MyRepo/pull-request/3/repository
             vcs_repo_name = /repo/base/path/RepoGroup/.__shadow_MyRepo_pr-3'
         """
         # First we set the repo name from URL for all attributes. This is the
         # default if handling normal (non shadow) repo requests.
         self.url_repo_name = self._get_repository_name(environ)
         self.acl_repo_name = self.vcs_repo_name = self.url_repo_name
         self.is_shadow_repo = False
 
         # Check if this is a request to a shadow repository.
         match = self.shadow_repo_re.match(self.url_repo_name)
         if match:
             match_dict = match.groupdict()
 
             # Build acl repo name from regex match.
             acl_repo_name = safe_unicode('{groups}{target}'.format(
                 groups=match_dict['groups'] or '',
                 target=match_dict['target']))
 
             # Retrieve pull request instance by ID from regex match.
             pull_request = PullRequest.get(match_dict['pr_id'])
 
             # Only proceed if we got a pull request and if acl repo name from
             # URL equals the target repo name of the pull request.
             if pull_request and (acl_repo_name ==
                                  pull_request.target_repo.repo_name):
                 # Get file system path to shadow repository.
                 workspace_id = PullRequestModel()._workspace_id(pull_request)
                 target_vcs = pull_request.target_repo.scm_instance()
                 vcs_repo_name = target_vcs._get_shadow_repository_path(
                     workspace_id)
 
                 # Store names for later usage.
                 self.vcs_repo_name = vcs_repo_name
                 self.acl_repo_name = acl_repo_name
                 self.is_shadow_repo = True
 
         log.debug('Setting all VCS repository names: %s', {
             'acl_repo_name': self.acl_repo_name,
             'url_repo_name': self.url_repo_name,
             'vcs_repo_name': self.vcs_repo_name,
         })
 
     @property
     def scm_app(self):
         custom_implementation = self.config['vcs.scm_app_implementation']
         if custom_implementation == 'http':
             log.info('Using HTTP implementation of scm app.')
             scm_app_impl = scm_app_http
         else:
             log.info('Using custom implementation of scm_app: "{}"'.format(
                 custom_implementation))
             scm_app_impl = importlib.import_module(custom_implementation)
         return scm_app_impl
 
     def _get_by_id(self, repo_name):
         """
         Gets a special pattern _<ID> from clone url and tries to replace it
         with a repository_name for support of _<ID> non changeable urls
         """
 
         data = repo_name.split('/')
         if len(data) >= 2:
             from rhodecode.model.repo import RepoModel
             by_id_match = RepoModel().get_repo_by_id(repo_name)
             if by_id_match:
                 data[1] = by_id_match.repo_name
 
         return safe_str('/'.join(data))
 
     def _invalidate_cache(self, repo_name):
         """
         Set's cache for this repository for invalidation on next access
 
         :param repo_name: full repo name, also a cache key
         """
         ScmModel().mark_for_invalidation(repo_name)
 
     def is_valid_and_existing_repo(self, repo_name, base_path, scm_type):
         db_repo = Repository.get_by_repo_name(repo_name)
         if not db_repo:
             log.debug('Repository `%s` not found inside the database.',
                       repo_name)
             return False
 
         if db_repo.repo_type != scm_type:
             log.warning(
                 'Repository `%s` have incorrect scm_type, expected %s got %s',
                 repo_name, db_repo.repo_type, scm_type)
             return False
 
-        return is_valid_repo(repo_name, base_path, explicit_scm=scm_type)
+        return is_valid_repo(repo_name, base_path,
+                             explicit_scm=scm_type, expect_scm=scm_type)
 
     def valid_and_active_user(self, user):
         """
         Checks if that user is not empty, and if it's actually object it checks
         if he's active.
 
         :param user: user object or None
         :return: boolean
         """
         if user is None:
             return False
 
         elif user.active:
             return True
 
         return False
 
     @property
     def is_shadow_repo_dir(self):
         return os.path.isdir(self.vcs_repo_name)
 
     def _check_permission(self, action, user, repo_name, ip_addr=None,
                           plugin_id='', plugin_cache_active=False, cache_ttl=0):
         """
         Checks permissions using action (push/pull) user and repository
         name. If plugin_cache and ttl is set it will use the plugin which
         authenticated the user to store the cached permissions result for N
         amount of seconds as in cache_ttl
 
         :param action: push or pull action
         :param user: user instance
         :param repo_name: repository name
         """
 
         # get instance of cache manager configured for a namespace
         cache_manager = get_perms_cache_manager(custom_ttl=cache_ttl)
         log.debug('AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)',
                   plugin_id, plugin_cache_active, cache_ttl)
 
         # for environ based password can be empty, but then the validation is
         # on the server that fills in the env data needed for authentication
         _perm_calc_hash = caches.compute_key_from_params(
             plugin_id, action, user.user_id, repo_name, ip_addr)
 
         # _authenticate is a wrapper for .auth() method of plugin.
         # it checks if .auth() sends proper data.
         # For RhodeCodeExternalAuthPlugin it also maps users to
         # Database and maps the attributes returned from .auth()
         # to RhodeCode database. If this function returns data
         # then auth is correct.
         start = time.time()
         log.debug('Running plugin `%s` permissions check', plugin_id)
 
         def perm_func():
             """
             This function is used internally in Cache of Beaker to calculate
             Results
             """
             log.debug('auth: calculating permission access now...')
             # check IP
             inherit = user.inherit_default_permissions
             ip_allowed = AuthUser.check_ip_allowed(
                 user.user_id, ip_addr, inherit_from_default=inherit)
             if ip_allowed:
                 log.info('Access for IP:%s allowed', ip_addr)
             else:
                 return False
 
             if action == 'push':
                 perms = ('repository.write', 'repository.admin')
                 if not HasPermissionAnyMiddleware(*perms)(user, repo_name):
                     return False
 
             else:
                 # any other action need at least read permission
                 perms = (
                     'repository.read', 'repository.write', 'repository.admin')
                 if not HasPermissionAnyMiddleware(*perms)(user, repo_name):
                     return False
 
             return True
 
         if plugin_cache_active:
             log.debug('Trying to fetch cached perms by %s', _perm_calc_hash[:6])
             perm_result = cache_manager.get(
                 _perm_calc_hash, createfunc=perm_func)
         else:
             perm_result = perm_func()
 
         auth_time = time.time() - start
         log.debug('Permissions for plugin `%s` completed in %.3fs, '
                   'expiration time of fetched cache %.1fs.',
                   plugin_id, auth_time, cache_ttl)
 
         return perm_result
 
     def _check_ssl(self, environ, start_response):
         """
         Checks the SSL check flag and returns False if SSL is not present
         and required True otherwise
         """
         org_proto = environ['wsgi._org_proto']
         # check if we have SSL required  ! if not it's a bad request !
         require_ssl = str2bool(self.repo_vcs_config.get('web', 'push_ssl'))
         if require_ssl and org_proto == 'http':
             log.debug('proto is %s and SSL is required BAD REQUEST !',
                       org_proto)
             return False
         return True
 
     def __call__(self, environ, start_response):
         try:
             return self._handle_request(environ, start_response)
         except Exception:
             log.exception("Exception while handling request")
             appenlight.track_exception(environ)
             return HTTPInternalServerError()(environ, start_response)
         finally:
             meta.Session.remove()
 
     def _handle_request(self, environ, start_response):
 
         if not self._check_ssl(environ, start_response):
             reason = ('SSL required, while RhodeCode was unable '
                       'to detect this as SSL request')
             log.debug('User not allowed to proceed, %s', reason)
             return HTTPNotAcceptable(reason)(environ, start_response)
 
         if not self.url_repo_name:
             log.warning('Repository name is empty: %s', self.url_repo_name)
             # failed to get repo name, we fail now
             return HTTPNotFound()(environ, start_response)
         log.debug('Extracted repo name is %s', self.url_repo_name)
 
         ip_addr = get_ip_addr(environ)
         user_agent = get_user_agent(environ)
         username = None
 
         # skip passing error to error controller
         environ['pylons.status_code_redirect'] = True
 
         # ======================================================================
         # GET ACTION PULL or PUSH
         # ======================================================================
         action = self._get_action(environ)
 
         # ======================================================================
         # Check if this is a request to a shadow repository of a pull request.
         # In this case only pull action is allowed.
         # ======================================================================
         if self.is_shadow_repo and action != 'pull':
             reason = 'Only pull action is allowed for shadow repositories.'
             log.debug('User not allowed to proceed, %s', reason)
             return HTTPNotAcceptable(reason)(environ, start_response)
 
         # Check if the shadow repo actually exists, in case someone refers
         # to it, and it has been deleted because of successful merge.
         if self.is_shadow_repo and not self.is_shadow_repo_dir:
             log.debug('Shadow repo detected, and shadow repo dir `%s` is missing',
                       self.is_shadow_repo_dir)
             return HTTPNotFound()(environ, start_response)
 
         # ======================================================================
         # CHECK ANONYMOUS PERMISSION
         # ======================================================================
         if action in ['pull', 'push']:
             anonymous_user = User.get_default_user()
             username = anonymous_user.username
             if anonymous_user.active:
                 # ONLY check permissions if the user is activated
                 anonymous_perm = self._check_permission(
                     action, anonymous_user, self.acl_repo_name, ip_addr)
             else:
                 anonymous_perm = False
 
             if not anonymous_user.active or not anonymous_perm:
                 if not anonymous_user.active:
                     log.debug('Anonymous access is disabled, running '
                               'authentication')
 
                 if not anonymous_perm:
                     log.debug('Not enough credentials to access this '
                               'repository as anonymous user')
 
                 username = None
                 # ==============================================================
                 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
                 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
                 # ==============================================================
 
                 # try to auth based on environ, container auth methods
                 log.debug('Running PRE-AUTH for container based authentication')
                 pre_auth = authenticate(
                     '', '', environ, VCS_TYPE, registry=self.registry,
                     acl_repo_name=self.acl_repo_name)
                 if pre_auth and pre_auth.get('username'):
                     username = pre_auth['username']
                 log.debug('PRE-AUTH got %s as username', username)
                 if pre_auth:
                     log.debug('PRE-AUTH successful from %s',
                               pre_auth.get('auth_data', {}).get('_plugin'))
 
                 # If not authenticated by the container, running basic auth
                 # before inject the calling repo_name for special scope checks
                 self.authenticate.acl_repo_name = self.acl_repo_name
 
                 plugin_cache_active, cache_ttl = False, 0
                 plugin = None
                 if not username:
                     self.authenticate.realm = self.authenticate.get_rc_realm()
 
                     try:
                         auth_result = self.authenticate(environ)
                     except (UserCreationError, NotAllowedToCreateUserError) as e:
                         log.error(e)
                         reason = safe_str(e)
                         return HTTPNotAcceptable(reason)(environ, start_response)
 
                     if isinstance(auth_result, dict):
                         AUTH_TYPE.update(environ, 'basic')
                         REMOTE_USER.update(environ, auth_result['username'])
                         username = auth_result['username']
                         plugin = auth_result.get('auth_data', {}).get('_plugin')
                         log.info(
                             'MAIN-AUTH successful for user `%s` from %s plugin',
                             username, plugin)
 
                         plugin_cache_active, cache_ttl = auth_result.get(
                             'auth_data', {}).get('_ttl_cache') or (False, 0)
                     else:
                         return auth_result.wsgi_application(
                             environ, start_response)
 
 
                 # ==============================================================
                 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
                 # ==============================================================
                 user = User.get_by_username(username)
                 if not self.valid_and_active_user(user):
                     return HTTPForbidden()(environ, start_response)
                 username = user.username
                 user.update_lastactivity()
                 meta.Session().commit()
 
                 # check user attributes for password change flag
                 user_obj = user
                 if user_obj and user_obj.username != User.DEFAULT_USER and \
                         user_obj.user_data.get('force_password_change'):
                     reason = 'password change required'
                     log.debug('User not allowed to authenticate, %s', reason)
                     return HTTPNotAcceptable(reason)(environ, start_response)
 
                 # check permissions for this repository
                 perm = self._check_permission(
                     action, user, self.acl_repo_name, ip_addr,
                     plugin, plugin_cache_active, cache_ttl)
                 if not perm:
                     return HTTPForbidden()(environ, start_response)
 
         # extras are injected into UI object and later available
         # in hooks executed by RhodeCode
         check_locking = _should_check_locking(environ.get('QUERY_STRING'))
         extras = vcs_operation_context(
             environ, repo_name=self.acl_repo_name, username=username,
             action=action, scm=self.SCM, check_locking=check_locking,
             is_shadow_repo=self.is_shadow_repo
         )
 
         # ======================================================================
         # REQUEST HANDLING
         # ======================================================================
         repo_path = os.path.join(
             safe_str(self.base_path), safe_str(self.vcs_repo_name))
         log.debug('Repository path is %s', repo_path)
 
         fix_PATH()
 
         log.info(
             '%s action on %s repo "%s" by "%s" from %s %s',
             action, self.SCM, safe_str(self.url_repo_name),
             safe_str(username), ip_addr, user_agent)
 
         return self._generate_vcs_response(
             environ, start_response, repo_path, extras, action)
 
     @initialize_generator
     def _generate_vcs_response(
             self, environ, start_response, repo_path, extras, action):
         """
         Returns a generator for the response content.
 
         This method is implemented as a generator, so that it can trigger
         the cache validation after all content sent back to the client. It
         also handles the locking exceptions which will be triggered when
         the first chunk is produced by the underlying WSGI application.
         """
         callback_daemon, extras = self._prepare_callback_daemon(extras)
         config = self._create_config(extras, self.acl_repo_name)
         log.debug('HOOKS extras is %s', extras)
         app = self._create_wsgi_app(repo_path, self.url_repo_name, config)
 
         try:
             with callback_daemon:
                 try:
                     response = app(environ, start_response)
                 finally:
                     # This statement works together with the decorator
                     # "initialize_generator" above. The decorator ensures that
                     # we hit the first yield statement before the generator is
                     # returned back to the WSGI server. This is needed to
                     # ensure that the call to "app" above triggers the
                     # needed callback to "start_response" before the
                     # generator is actually used.
                     yield "__init__"
 
                 for chunk in response:
                     yield chunk
         except Exception as exc:
             # TODO: martinb: Exceptions are only raised in case of the Pyro4
             # backend. Refactor this except block after dropping Pyro4 support.
             # TODO: johbo: Improve "translating" back the exception.
             if getattr(exc, '_vcs_kind', None) == 'repo_locked':
                 exc = HTTPLockedRC(*exc.args)
                 _code = rhodecode.CONFIG.get('lock_ret_code')
                 log.debug('Repository LOCKED ret code %s!', (_code,))
             elif getattr(exc, '_vcs_kind', None) == 'requirement':
                 log.debug(
                     'Repository requires features unknown to this Mercurial')
                 exc = HTTPRequirementError(*exc.args)
             else:
                 raise
 
             for chunk in exc(environ, start_response):
                 yield chunk
         finally:
             # invalidate cache on push
             try:
                 if action == 'push':
                     self._invalidate_cache(self.url_repo_name)
             finally:
                 meta.Session.remove()
 
     def _get_repository_name(self, environ):
         """Get repository name out of the environmnent
 
         :param environ: WSGI environment
         """
         raise NotImplementedError()
 
     def _get_action(self, environ):
         """Map request commands into a pull or push command.
 
         :param environ: WSGI environment
         """
         raise NotImplementedError()
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
         """Return the WSGI app that will finally handle the request."""
         raise NotImplementedError()
 
     def _create_config(self, extras, repo_name):
         """Create a safe config representation."""
         raise NotImplementedError()
 
     def _prepare_callback_daemon(self, extras):
         return prepare_callback_daemon(
             extras, protocol=vcs_settings.HOOKS_PROTOCOL,
             use_direct_calls=vcs_settings.HOOKS_DIRECT_CALLS)
 
 
 def _should_check_locking(query_string):
     # this is kind of hacky, but due to how mercurial handles client-server
     # server see all operation on commit; bookmarks, phases and
     # obsolescence marker in different transaction, we don't want to check
     # locking on those
     return query_string not in ['cmd=listkeys']

rhodecode/lib/utils.py

@@ -1,802 +1,802 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Utilities library for RhodeCode
 """
 
 import datetime
 import decorator
 import json
 import logging
 import os
 import re
 import shutil
 import tempfile
 import traceback
 import tarfile
 import warnings
 import hashlib
 from os.path import join as jn
 
 import paste
 import pkg_resources
 from webhelpers.text import collapse, remove_formatting, strip_tags
 from mako import exceptions
 from pyramid.threadlocal import get_current_registry
 from pyramid.request import Request
 
 from rhodecode.lib.fakemod import create_module
 from rhodecode.lib.vcs.backends.base import Config
 from rhodecode.lib.vcs.exceptions import VCSError
 from rhodecode.lib.vcs.utils.helpers import get_scm, get_scm_backend
 from rhodecode.lib.utils2 import (
     safe_str, safe_unicode, get_current_rhodecode_user, md5)
 from rhodecode.model import meta
 from rhodecode.model.db import (
     Repository, User, RhodeCodeUi, UserLog, RepoGroup, UserGroup)
 from rhodecode.model.meta import Session
 
 
 log = logging.getLogger(__name__)
 
 REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}__.*')
 
 # String which contains characters that are not allowed in slug names for
 # repositories or repository groups. It is properly escaped to use it in
 # regular expressions.
 SLUG_BAD_CHARS = re.escape('`?=[]\;\'"<>,/~!@#$%^&*()+{}|:')
 
 # Regex that matches forbidden characters in repo/group slugs.
 SLUG_BAD_CHAR_RE = re.compile('[{}]'.format(SLUG_BAD_CHARS))
 
 # Regex that matches allowed characters in repo/group slugs.
 SLUG_GOOD_CHAR_RE = re.compile('[^{}]'.format(SLUG_BAD_CHARS))
 
 # Regex that matches whole repo/group slugs.
 SLUG_RE = re.compile('[^{}]+'.format(SLUG_BAD_CHARS))
 
 _license_cache = None
 
 
 def repo_name_slug(value):
     """
     Return slug of name of repository
     This function is called on each creation/modification
     of repository to prevent bad names in repo
     """
     replacement_char = '-'
 
     slug = remove_formatting(value)
     slug = SLUG_BAD_CHAR_RE.sub('', slug)
     slug = re.sub('[\s]+', '-', slug)
     slug = collapse(slug, replacement_char)
     return slug
 
 
 #==============================================================================
 # PERM DECORATOR HELPERS FOR EXTRACTING NAMES FOR PERM CHECKS
 #==============================================================================
 def get_repo_slug(request):
     _repo = ''
 
     if hasattr(request, 'db_repo'):
         # if our requests has set db reference use it for name, this
         # translates the example.com/_<id> into proper repo names
         _repo = request.db_repo.repo_name
     elif getattr(request, 'matchdict', None):
         # pyramid
         _repo = request.matchdict.get('repo_name')
 
     if _repo:
         _repo = _repo.rstrip('/')
     return _repo
 
 
 def get_repo_group_slug(request):
     _group = ''
     if hasattr(request, 'db_repo_group'):
         # if our requests has set db reference use it for name, this
         # translates the example.com/_<id> into proper repo group names
         _group = request.db_repo_group.group_name
     elif getattr(request, 'matchdict', None):
         # pyramid
         _group = request.matchdict.get('repo_group_name')
 
 
     if _group:
         _group = _group.rstrip('/')
     return _group
 
 
 def get_user_group_slug(request):
     _user_group = ''
 
     if hasattr(request, 'db_user_group'):
         _user_group = request.db_user_group.users_group_name
     elif getattr(request, 'matchdict', None):
         # pyramid
         _user_group = request.matchdict.get('user_group_id')
 
         try:
             _user_group = UserGroup.get(_user_group)
             if _user_group:
                 _user_group = _user_group.users_group_name
         except Exception:
             log.exception('Failed to get user group by id')
             # catch all failures here
             return None
 
     return _user_group
 
 
 def get_filesystem_repos(path, recursive=False, skip_removed_repos=True):
     """
     Scans given path for repos and return (name,(type,path)) tuple
 
     :param path: path to scan for repositories
     :param recursive: recursive search and return names with subdirs in front
     """
 
     # remove ending slash for better results
     path = path.rstrip(os.sep)
     log.debug('now scanning in %s location recursive:%s...', path, recursive)
 
     def _get_repos(p):
         dirpaths = _get_dirpaths(p)
         if not _is_dir_writable(p):
             log.warning('repo path without write access: %s', p)
 
         for dirpath in dirpaths:
             if os.path.isfile(os.path.join(p, dirpath)):
                 continue
             cur_path = os.path.join(p, dirpath)
 
             # skip removed repos
             if skip_removed_repos and REMOVED_REPO_PAT.match(dirpath):
                 continue
 
             #skip .<somethin> dirs
             if dirpath.startswith('.'):
                 continue
 
             try:
                 scm_info = get_scm(cur_path)
                 yield scm_info[1].split(path, 1)[-1].lstrip(os.sep), scm_info
             except VCSError:
                 if not recursive:
                     continue
                 #check if this dir containts other repos for recursive scan
                 rec_path = os.path.join(p, dirpath)
                 if os.path.isdir(rec_path):
                     for inner_scm in _get_repos(rec_path):
                         yield inner_scm
 
     return _get_repos(path)
 
 
 def _get_dirpaths(p):
     try:
         # OS-independable way of checking if we have at least read-only
         # access or not.
         dirpaths = os.listdir(p)
     except OSError:
         log.warning('ignoring repo path without read access: %s', p)
         return []
 
     # os.listpath has a tweak: If a unicode is passed into it, then it tries to
     # decode paths and suddenly returns unicode objects itself. The items it
     # cannot decode are returned as strings and cause issues.
     #
     # Those paths are ignored here until a solid solution for path handling has
     # been built.
     expected_type = type(p)
 
     def _has_correct_type(item):
         if type(item) is not expected_type:
             log.error(
                 u"Ignoring path %s since it cannot be decoded into unicode.",
                 # Using "repr" to make sure that we see the byte value in case
                 # of support.
                 repr(item))
             return False
         return True
 
     dirpaths = [item for item in dirpaths if _has_correct_type(item)]
 
     return dirpaths
 
 
 def _is_dir_writable(path):
     """
     Probe if `path` is writable.
 
     Due to trouble on Cygwin / Windows, this is actually probing if it is
     possible to create a file inside of `path`, stat does not produce reliable
     results in this case.
     """
     try:
         with tempfile.TemporaryFile(dir=path):
             pass
     except OSError:
         return False
     return True
 
 
 def is_valid_repo(repo_name, base_path, expect_scm=None, explicit_scm=None):
     """
     Returns True if given path is a valid repository False otherwise.
     If expect_scm param is given also, compare if given scm is the same
     as expected from scm parameter. If explicit_scm is given don't try to
     detect the scm, just use the given one to check if repo is valid
 
     :param repo_name:
     :param base_path:
     :param expect_scm:
     :param explicit_scm:
 
     :return True: if given path is a valid repository
     """
     full_path = os.path.join(safe_str(base_path), safe_str(repo_name))
     log.debug('Checking if `%s` is a valid path for repository. '
               'Explicit type: %s', repo_name, explicit_scm)
 
     try:
         if explicit_scm:
-            detected_scms = [get_scm_backend(explicit_scm)]
+            detected_scms = [get_scm_backend(explicit_scm)(full_path).alias]
         else:
             detected_scms = get_scm(full_path)
 
         if expect_scm:
             return detected_scms[0] == expect_scm
         log.debug('path: %s is an vcs object:%s', full_path, detected_scms)
         return True
     except VCSError:
         log.debug('path: %s is not a valid repo !', full_path)
         return False
 
 
 def is_valid_repo_group(repo_group_name, base_path, skip_path_check=False):
     """
     Returns True if given path is a repository group, False otherwise
 
     :param repo_name:
     :param base_path:
     """
     full_path = os.path.join(safe_str(base_path), safe_str(repo_group_name))
     log.debug('Checking if `%s` is a valid path for repository group',
               repo_group_name)
 
     # check if it's not a repo
     if is_valid_repo(repo_group_name, base_path):
         log.debug('Repo called %s exist, it is not a valid '
                   'repo group' % repo_group_name)
         return False
 
     try:
         # we need to check bare git repos at higher level
         # since we might match branches/hooks/info/objects or possible
         # other things inside bare git repo
         scm_ = get_scm(os.path.dirname(full_path))
         log.debug('path: %s is a vcs object:%s, not valid '
                   'repo group' % (full_path, scm_))
         return False
     except VCSError:
         pass
 
     # check if it's a valid path
     if skip_path_check or os.path.isdir(full_path):
         log.debug('path: %s is a valid repo group !', full_path)
         return True
 
     log.debug('path: %s is not a valid repo group !', full_path)
     return False
 
 
 def ask_ok(prompt, retries=4, complaint='[y]es or [n]o please!'):
     while True:
         ok = raw_input(prompt)
         if ok.lower() in ('y', 'ye', 'yes'):
             return True
         if ok.lower() in ('n', 'no', 'nop', 'nope'):
             return False
         retries = retries - 1
         if retries < 0:
             raise IOError
         print(complaint)
 
 # propagated from mercurial documentation
 ui_sections = [
     'alias', 'auth',
     'decode/encode', 'defaults',
     'diff', 'email',
     'extensions', 'format',
     'merge-patterns', 'merge-tools',
     'hooks', 'http_proxy',
     'smtp', 'patch',
     'paths', 'profiling',
     'server', 'trusted',
     'ui', 'web', ]
 
 
 def config_data_from_db(clear_session=True, repo=None):
     """
     Read the configuration data from the database and return configuration
     tuples.
     """
     from rhodecode.model.settings import VcsSettingsModel
 
     config = []
 
     sa = meta.Session()
     settings_model = VcsSettingsModel(repo=repo, sa=sa)
 
     ui_settings = settings_model.get_ui_settings()
 
     for setting in ui_settings:
         if setting.active:
             log.debug(
                 'settings ui from db: [%s] %s=%s',
                 setting.section, setting.key, setting.value)
             config.append((
                 safe_str(setting.section), safe_str(setting.key),
                 safe_str(setting.value)))
         if setting.key == 'push_ssl':
             # force set push_ssl requirement to False, rhodecode
             # handles that
             config.append((
                 safe_str(setting.section), safe_str(setting.key), False))
     if clear_session:
         meta.Session.remove()
 
     # TODO: mikhail: probably it makes no sense to re-read hooks information.
     # It's already there and activated/deactivated
     skip_entries = []
     enabled_hook_classes = get_enabled_hook_classes(ui_settings)
     if 'pull' not in enabled_hook_classes:
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PULL))
     if 'push' not in enabled_hook_classes:
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PUSH))
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRETX_PUSH))
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PUSH_KEY))
 
     config = [entry for entry in config if entry[:2] not in skip_entries]
 
     return config
 
 
 def make_db_config(clear_session=True, repo=None):
     """
     Create a :class:`Config` instance based on the values in the database.
     """
     config = Config()
     config_data = config_data_from_db(clear_session=clear_session, repo=repo)
     for section, option, value in config_data:
         config.set(section, option, value)
     return config
 
 
 def get_enabled_hook_classes(ui_settings):
     """
     Return the enabled hook classes.
 
     :param ui_settings: List of ui_settings as returned
         by :meth:`VcsSettingsModel.get_ui_settings`
 
     :return: a list with the enabled hook classes. The order is not guaranteed.
     :rtype: list
     """
     enabled_hooks = []
     active_hook_keys = [
         key for section, key, value, active in ui_settings
         if section == 'hooks' and active]
 
     hook_names = {
         RhodeCodeUi.HOOK_PUSH: 'push',
         RhodeCodeUi.HOOK_PULL: 'pull',
         RhodeCodeUi.HOOK_REPO_SIZE: 'repo_size'
     }
 
     for key in active_hook_keys:
         hook = hook_names.get(key)
         if hook:
             enabled_hooks.append(hook)
 
     return enabled_hooks
 
 
 def set_rhodecode_config(config):
     """
     Updates pyramid config with new settings from database
 
     :param config:
     """
     from rhodecode.model.settings import SettingsModel
     app_settings = SettingsModel().get_all_settings()
 
     for k, v in app_settings.items():
         config[k] = v
 
 
 def get_rhodecode_realm():
     """
     Return the rhodecode realm from database.
     """
     from rhodecode.model.settings import SettingsModel
     realm = SettingsModel().get_setting_by_name('realm')
     return safe_str(realm.app_settings_value)
 
 
 def get_rhodecode_base_path():
     """
     Returns the base path. The base path is the filesystem path which points
     to the repository store.
     """
     from rhodecode.model.settings import SettingsModel
     paths_ui = SettingsModel().get_ui_by_section_and_key('paths', '/')
     return safe_str(paths_ui.ui_value)
 
 
 def map_groups(path):
     """
     Given a full path to a repository, create all nested groups that this
     repo is inside. This function creates parent-child relationships between
     groups and creates default perms for all new groups.
 
     :param paths: full path to repository
     """
     from rhodecode.model.repo_group import RepoGroupModel
     sa = meta.Session()
     groups = path.split(Repository.NAME_SEP)
     parent = None
     group = None
 
     # last element is repo in nested groups structure
     groups = groups[:-1]
     rgm = RepoGroupModel(sa)
     owner = User.get_first_super_admin()
     for lvl, group_name in enumerate(groups):
         group_name = '/'.join(groups[:lvl] + [group_name])
         group = RepoGroup.get_by_group_name(group_name)
         desc = '%s group' % group_name
 
         # skip folders that are now removed repos
         if REMOVED_REPO_PAT.match(group_name):
             break
 
         if group is None:
             log.debug('creating group level: %s group_name: %s',
                        lvl, group_name)
             group = RepoGroup(group_name, parent)
             group.group_description = desc
             group.user = owner
             sa.add(group)
             perm_obj = rgm._create_default_perms(group)
             sa.add(perm_obj)
             sa.flush()
 
         parent = group
     return group
 
 
 def repo2db_mapper(initial_repo_list, remove_obsolete=False):
     """
     maps all repos given in initial_repo_list, non existing repositories
     are created, if remove_obsolete is True it also checks for db entries
     that are not in initial_repo_list and removes them.
 
     :param initial_repo_list: list of repositories found by scanning methods
     :param remove_obsolete: check for obsolete entries in database
     """
     from rhodecode.model.repo import RepoModel
     from rhodecode.model.scm import ScmModel
     from rhodecode.model.repo_group import RepoGroupModel
     from rhodecode.model.settings import SettingsModel
 
     sa = meta.Session()
     repo_model = RepoModel()
     user = User.get_first_super_admin()
     added = []
 
     # creation defaults
     defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
     enable_statistics = defs.get('repo_enable_statistics')
     enable_locking = defs.get('repo_enable_locking')
     enable_downloads = defs.get('repo_enable_downloads')
     private = defs.get('repo_private')
 
     for name, repo in initial_repo_list.items():
         group = map_groups(name)
         unicode_name = safe_unicode(name)
         db_repo = repo_model.get_by_repo_name(unicode_name)
         # found repo that is on filesystem not in RhodeCode database
         if not db_repo:
             log.info('repository %s not found, creating now', name)
             added.append(name)
             desc = (repo.description
                     if repo.description != 'unknown'
                     else '%s repository' % name)
 
             db_repo = repo_model._create_repo(
                 repo_name=name,
                 repo_type=repo.alias,
                 description=desc,
                 repo_group=getattr(group, 'group_id', None),
                 owner=user,
                 enable_locking=enable_locking,
                 enable_downloads=enable_downloads,
                 enable_statistics=enable_statistics,
                 private=private,
                 state=Repository.STATE_CREATED
             )
             sa.commit()
             # we added that repo just now, and make sure we updated server info
             if db_repo.repo_type == 'git':
                 git_repo = db_repo.scm_instance()
                 # update repository server-info
                 log.debug('Running update server info')
                 git_repo._update_server_info()
 
             db_repo.update_commit_cache()
 
         config = db_repo._config
         config.set('extensions', 'largefiles', '')
         ScmModel().install_hooks(
             db_repo.scm_instance(config=config),
             repo_type=db_repo.repo_type)
 
     removed = []
     if remove_obsolete:
         # remove from database those repositories that are not in the filesystem
         for repo in sa.query(Repository).all():
             if repo.repo_name not in initial_repo_list.keys():
                 log.debug("Removing non-existing repository found in db `%s`",
                           repo.repo_name)
                 try:
                     RepoModel(sa).delete(repo, forks='detach', fs_remove=False)
                     sa.commit()
                     removed.append(repo.repo_name)
                 except Exception:
                     # don't hold further removals on error
                     log.error(traceback.format_exc())
                     sa.rollback()
 
         def splitter(full_repo_name):
             _parts = full_repo_name.rsplit(RepoGroup.url_sep(), 1)
             gr_name = None
             if len(_parts) == 2:
                 gr_name = _parts[0]
             return gr_name
 
         initial_repo_group_list = [splitter(x) for x in
                                    initial_repo_list.keys() if splitter(x)]
 
         # remove from database those repository groups that are not in the
         # filesystem due to parent child relationships we need to delete them
         # in a specific order of most nested first
         all_groups = [x.group_name for x in sa.query(RepoGroup).all()]
         nested_sort = lambda gr: len(gr.split('/'))
         for group_name in sorted(all_groups, key=nested_sort, reverse=True):
             if group_name not in initial_repo_group_list:
                 repo_group = RepoGroup.get_by_group_name(group_name)
                 if (repo_group.children.all() or
                     not RepoGroupModel().check_exist_filesystem(
                         group_name=group_name, exc_on_failure=False)):
                     continue
 
                 log.info(
                     'Removing non-existing repository group found in db `%s`',
                     group_name)
                 try:
                     RepoGroupModel(sa).delete(group_name, fs_remove=False)
                     sa.commit()
                     removed.append(group_name)
                 except Exception:
                     # don't hold further removals on error
                     log.exception(
                         'Unable to remove repository group `%s`',
                         group_name)
                     sa.rollback()
                     raise
 
     return added, removed
 
 
 def get_default_cache_settings(settings):
     cache_settings = {}
     for key in settings.keys():
         for prefix in ['beaker.cache.', 'cache.']:
             if key.startswith(prefix):
                 name = key.split(prefix)[1].strip()
                 cache_settings[name] = settings[key].strip()
     return cache_settings
 
 
 # set cache regions for beaker so celery can utilise it
 def add_cache(settings):
     from rhodecode.lib import caches
     cache_settings = {'regions': None}
     # main cache settings used as default ...
     cache_settings.update(get_default_cache_settings(settings))
 
     if cache_settings['regions']:
         for region in cache_settings['regions'].split(','):
             region = region.strip()
             region_settings = {}
             for key, value in cache_settings.items():
                 if key.startswith(region):
                     region_settings[key.split('.')[1]] = value
 
             caches.configure_cache_region(
                 region, region_settings, cache_settings)
 
 
 def load_rcextensions(root_path):
     import rhodecode
     from rhodecode.config import conf
 
     path = os.path.join(root_path, 'rcextensions', '__init__.py')
     if os.path.isfile(path):
         rcext = create_module('rc', path)
         EXT = rhodecode.EXTENSIONS = rcext
         log.debug('Found rcextensions now loading %s...', rcext)
 
         # Additional mappings that are not present in the pygments lexers
         conf.LANGUAGES_EXTENSIONS_MAP.update(getattr(EXT, 'EXTRA_MAPPINGS', {}))
 
         # auto check if the module is not missing any data, set to default if is
         # this will help autoupdate new feature of rcext module
         #from rhodecode.config import rcextensions
         #for k in dir(rcextensions):
         #    if not k.startswith('_') and not hasattr(EXT, k):
         #        setattr(EXT, k, getattr(rcextensions, k))
 
 
 def get_custom_lexer(extension):
     """
     returns a custom lexer if it is defined in rcextensions module, or None
     if there's no custom lexer defined
     """
     import rhodecode
     from pygments import lexers
 
     # custom override made by RhodeCode
     if extension in ['mako']:
         return lexers.get_lexer_by_name('html+mako')
 
     # check if we didn't define this extension as other lexer
     extensions = rhodecode.EXTENSIONS and getattr(rhodecode.EXTENSIONS, 'EXTRA_LEXERS', None)
     if extensions and extension in rhodecode.EXTENSIONS.EXTRA_LEXERS:
         _lexer_name = rhodecode.EXTENSIONS.EXTRA_LEXERS[extension]
         return lexers.get_lexer_by_name(_lexer_name)
 
 
 #==============================================================================
 # TEST FUNCTIONS AND CREATORS
 #==============================================================================
 def create_test_index(repo_location, config):
     """
     Makes default test index.
     """
     import rc_testdata
 
     rc_testdata.extract_search_index(
         'vcs_search_index', os.path.dirname(config['search.location']))
 
 
 def create_test_directory(test_path):
     """
     Create test directory if it doesn't exist.
     """
     if not os.path.isdir(test_path):
         log.debug('Creating testdir %s', test_path)
         os.makedirs(test_path)
 
 
 def create_test_database(test_path, config):
     """
     Makes a fresh database.
     """
     from rhodecode.lib.db_manage import DbManage
 
     # PART ONE create db
     dbconf = config['sqlalchemy.db1.url']
     log.debug('making test db %s', dbconf)
 
     dbmanage = DbManage(log_sql=False, dbconf=dbconf, root=config['here'],
                         tests=True, cli_args={'force_ask': True})
     dbmanage.create_tables(override=True)
     dbmanage.set_db_version()
     # for tests dynamically set new root paths based on generated content
     dbmanage.create_settings(dbmanage.config_prompt(test_path))
     dbmanage.create_default_user()
     dbmanage.create_test_admin_and_users()
     dbmanage.create_permissions()
     dbmanage.populate_default_permissions()
     Session().commit()
 
 
 def create_test_repositories(test_path, config):
     """
     Creates test repositories in the temporary directory. Repositories are
     extracted from archives within the rc_testdata package.
     """
     import rc_testdata
     from rhodecode.tests import HG_REPO, GIT_REPO, SVN_REPO
 
     log.debug('making test vcs repositories')
 
     idx_path = config['search.location']
     data_path = config['cache_dir']
 
     # clean index and data
     if idx_path and os.path.exists(idx_path):
         log.debug('remove %s', idx_path)
         shutil.rmtree(idx_path)
 
     if data_path and os.path.exists(data_path):
         log.debug('remove %s', data_path)
         shutil.rmtree(data_path)
 
     rc_testdata.extract_hg_dump('vcs_test_hg', jn(test_path, HG_REPO))
     rc_testdata.extract_git_dump('vcs_test_git', jn(test_path, GIT_REPO))
 
     # Note: Subversion is in the process of being integrated with the system,
     # until we have a properly packed version of the test svn repository, this
     # tries to copy over the repo from a package "rc_testdata"
     svn_repo_path = rc_testdata.get_svn_repo_archive()
     with tarfile.open(svn_repo_path) as tar:
         tar.extractall(jn(test_path, SVN_REPO))
 
 
 def password_changed(auth_user, session):
     # Never report password change in case of default user or anonymous user.
     if auth_user.username == User.DEFAULT_USER or auth_user.user_id is None:
         return False
 
     password_hash = md5(auth_user.password) if auth_user.password else None
     rhodecode_user = session.get('rhodecode_user', {})
     session_password_hash = rhodecode_user.get('password', '')
     return password_hash != session_password_hash
 
 
 def read_opensource_licenses():
     global _license_cache
 
     if not _license_cache:
         licenses = pkg_resources.resource_string(
             'rhodecode', 'config/licenses.json')
         _license_cache = json.loads(licenses)
 
     return _license_cache
 
 
 def generate_platform_uuid():
     """
     Generates platform UUID based on it's name
     """
     import platform
 
     try:
         uuid_list = [platform.platform()]
         return hashlib.sha256(':'.join(uuid_list)).hexdigest()
     except Exception as e:
         log.error('Failed to generate host uuid: %s' % e)
         return 'UNDEFINED'

rhodecode/tests/other/vcs_operations/test_vcs_operations.py

@@ -1,655 +1,678 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Test suite for making push/pull operations, on specially modified INI files
 
 .. important::
 
    You must have git >= 1.8.5 for tests to work fine. With 68b939b git started
    to redirect things to stderr instead of stdout.
 """
 
 
 import os
 import time
 
 import pytest
 
 from rhodecode.lib.vcs.backends.git.repository import GitRepository
 from rhodecode.lib.vcs.backends.hg.repository import MercurialRepository
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.model.auth_token import AuthTokenModel
 from rhodecode.model.db import Repository, UserIpMap, CacheKey
 from rhodecode.model.meta import Session
+from rhodecode.model.repo import RepoModel
 from rhodecode.model.user import UserModel
 from rhodecode.tests import (GIT_REPO, HG_REPO, TEST_USER_ADMIN_LOGIN)
 
 from rhodecode.tests.other.vcs_operations import (
     Command, _check_proper_clone, _check_proper_git_push,
     _check_proper_hg_push, _add_files_and_push,
     HG_REPO_WITH_GROUP, GIT_REPO_WITH_GROUP)
 
 
 @pytest.mark.usefixtures("disable_locking", "disable_anonymous_user")
 class TestVCSOperations(object):
 
     def test_clone_hg_repo_by_admin(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(HG_REPO)
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         _check_proper_clone(stdout, stderr, 'hg')
 
     def test_clone_git_repo_by_admin(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(GIT_REPO)
         cmd = Command('/tmp')
         stdout, stderr = cmd.execute('git clone', clone_url, tmpdir.strpath)
         _check_proper_clone(stdout, stderr, 'git')
         cmd.assert_returncode_success()
 
     def test_clone_git_repo_by_admin_with_git_suffix(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(GIT_REPO)
         cmd = Command('/tmp')
         stdout, stderr = cmd.execute('git clone', clone_url+".git", tmpdir.strpath)
         _check_proper_clone(stdout, stderr, 'git')
         cmd.assert_returncode_success()
 
     def test_clone_hg_repo_by_id_by_admin(self, rc_web_server, tmpdir):
         repo_id = Repository.get_by_repo_name(HG_REPO).repo_id
         clone_url = rc_web_server.repo_clone_url('_%s' % repo_id)
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         _check_proper_clone(stdout, stderr, 'hg')
 
     def test_clone_git_repo_by_id_by_admin(self, rc_web_server, tmpdir):
         repo_id = Repository.get_by_repo_name(GIT_REPO).repo_id
         clone_url = rc_web_server.repo_clone_url('_%s' % repo_id)
         cmd = Command('/tmp')
         stdout, stderr = cmd.execute('git clone', clone_url, tmpdir.strpath)
         _check_proper_clone(stdout, stderr, 'git')
         cmd.assert_returncode_success()
 
     def test_clone_hg_repo_with_group_by_admin(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(HG_REPO_WITH_GROUP)
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         _check_proper_clone(stdout, stderr, 'hg')
 
     def test_clone_git_repo_with_group_by_admin(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(GIT_REPO_WITH_GROUP)
         cmd = Command('/tmp')
         stdout, stderr = cmd.execute('git clone', clone_url, tmpdir.strpath)
         _check_proper_clone(stdout, stderr, 'git')
         cmd.assert_returncode_success()
 
     def test_clone_git_repo_shallow_by_admin(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(GIT_REPO)
         cmd = Command('/tmp')
         stdout, stderr = cmd.execute(
             'git clone --depth=1', clone_url, tmpdir.strpath)
 
         assert '' == stdout
         assert 'Cloning into' in stderr
         cmd.assert_returncode_success()
 
     def test_clone_wrong_credentials_hg(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(HG_REPO, passwd='bad!')
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         assert 'abort: authorization failed' in stderr
 
     def test_clone_wrong_credentials_git(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(GIT_REPO, passwd='bad!')
         stdout, stderr = Command('/tmp').execute(
             'git clone', clone_url, tmpdir.strpath)
         assert 'fatal: Authentication failed' in stderr
 
     def test_clone_git_dir_as_hg(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(GIT_REPO)
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         assert 'HTTP Error 404: Not Found' in stderr
 
     def test_clone_hg_repo_as_git(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(HG_REPO)
         stdout, stderr = Command('/tmp').execute(
             'git clone', clone_url, tmpdir.strpath)
         assert 'not found' in stderr
 
     def test_clone_non_existing_path_hg(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url('trololo')
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         assert 'HTTP Error 404: Not Found' in stderr
 
     def test_clone_non_existing_path_git(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url('trololo')
         stdout, stderr = Command('/tmp').execute('git clone', clone_url)
         assert 'not found' in stderr
 
     def test_clone_existing_path_hg_not_in_database(
             self, rc_web_server, tmpdir, fs_repo_only):
 
         db_name = fs_repo_only('not-in-db-hg', repo_type='hg')
         clone_url = rc_web_server.repo_clone_url(db_name)
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         assert 'HTTP Error 404: Not Found' in stderr
 
     def test_clone_existing_path_git_not_in_database(
             self, rc_web_server, tmpdir, fs_repo_only):
         db_name = fs_repo_only('not-in-db-git', repo_type='git')
         clone_url = rc_web_server.repo_clone_url(db_name)
         stdout, stderr = Command('/tmp').execute(
             'git clone', clone_url, tmpdir.strpath)
         assert 'not found' in stderr
 
     def test_clone_existing_path_hg_not_in_database_different_scm(
             self, rc_web_server, tmpdir, fs_repo_only):
         db_name = fs_repo_only('not-in-db-git', repo_type='git')
         clone_url = rc_web_server.repo_clone_url(db_name)
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         assert 'HTTP Error 404: Not Found' in stderr
 
     def test_clone_existing_path_git_not_in_database_different_scm(
             self, rc_web_server, tmpdir, fs_repo_only):
         db_name = fs_repo_only('not-in-db-hg', repo_type='hg')
         clone_url = rc_web_server.repo_clone_url(db_name)
         stdout, stderr = Command('/tmp').execute(
             'git clone', clone_url, tmpdir.strpath)
         assert 'not found' in stderr
 
+    def test_clone_non_existing_store_path_hg(self, rc_web_server, tmpdir, user_util):
+        repo = user_util.create_repo()
+        clone_url = rc_web_server.repo_clone_url(repo.repo_name)
+
+        # Damage repo by removing it's folder
+        RepoModel()._delete_filesystem_repo(repo)
+
+        stdout, stderr = Command('/tmp').execute(
+            'hg clone', clone_url, tmpdir.strpath)
+        assert 'HTTP Error 404: Not Found' in stderr
+
+    def test_clone_non_existing_store_path_git(self, rc_web_server, tmpdir, user_util):
+        repo = user_util.create_repo(repo_type='git')
+        clone_url = rc_web_server.repo_clone_url(repo.repo_name)
+
+        # Damage repo by removing it's folder
+        RepoModel()._delete_filesystem_repo(repo)
+
+        stdout, stderr = Command('/tmp').execute(
+            'git clone', clone_url, tmpdir.strpath)
+        assert 'not found' in stderr
+
     def test_push_new_file_hg(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(HG_REPO)
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
 
         stdout, stderr = _add_files_and_push(
             'hg', tmpdir.strpath, clone_url=clone_url)
 
         assert 'pushing to' in stdout
         assert 'size summary' in stdout
 
     def test_push_new_file_git(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(GIT_REPO)
         stdout, stderr = Command('/tmp').execute(
             'git clone', clone_url, tmpdir.strpath)
 
         # commit some stuff into this repo
         stdout, stderr = _add_files_and_push(
             'git', tmpdir.strpath, clone_url=clone_url)
 
         _check_proper_git_push(stdout, stderr)
 
     def test_push_invalidates_cache_hg(self, rc_web_server, tmpdir):
         key = CacheKey.query().filter(CacheKey.cache_key == HG_REPO).scalar()
         if not key:
             key = CacheKey(HG_REPO, HG_REPO)
 
         key.cache_active = True
         Session().add(key)
         Session().commit()
 
         clone_url = rc_web_server.repo_clone_url(HG_REPO)
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
 
         stdout, stderr = _add_files_and_push(
             'hg', tmpdir.strpath, clone_url=clone_url, files_no=1)
 
         key = CacheKey.query().filter(CacheKey.cache_key == HG_REPO).one()
         assert key.cache_active is False
 
     def test_push_invalidates_cache_git(self, rc_web_server, tmpdir):
         key = CacheKey.query().filter(CacheKey.cache_key == GIT_REPO).scalar()
         if not key:
             key = CacheKey(GIT_REPO, GIT_REPO)
 
         key.cache_active = True
         Session().add(key)
         Session().commit()
 
         clone_url = rc_web_server.repo_clone_url(GIT_REPO)
         stdout, stderr = Command('/tmp').execute(
             'git clone', clone_url, tmpdir.strpath)
 
         # commit some stuff into this repo
         stdout, stderr = _add_files_and_push(
             'git', tmpdir.strpath, clone_url=clone_url, files_no=1)
         _check_proper_git_push(stdout, stderr)
 
         key = CacheKey.query().filter(CacheKey.cache_key == GIT_REPO).one()
 
         assert key.cache_active is False
 
     def test_push_wrong_credentials_hg(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(HG_REPO)
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
 
         push_url = rc_web_server.repo_clone_url(
             HG_REPO, user='bad', passwd='name')
         stdout, stderr = _add_files_and_push(
             'hg', tmpdir.strpath, clone_url=push_url)
 
         assert 'abort: authorization failed' in stderr
 
     def test_push_wrong_credentials_git(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(GIT_REPO)
         stdout, stderr = Command('/tmp').execute(
             'git clone', clone_url, tmpdir.strpath)
 
         push_url = rc_web_server.repo_clone_url(
             GIT_REPO, user='bad', passwd='name')
         stdout, stderr = _add_files_and_push(
             'git', tmpdir.strpath, clone_url=push_url)
 
         assert 'fatal: Authentication failed' in stderr
 
     def test_push_back_to_wrong_url_hg(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(HG_REPO)
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
 
         stdout, stderr = _add_files_and_push(
             'hg', tmpdir.strpath,
             clone_url=rc_web_server.repo_clone_url('not-existing'))
 
         assert 'HTTP Error 404: Not Found' in stderr
 
     def test_push_back_to_wrong_url_git(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(GIT_REPO)
         stdout, stderr = Command('/tmp').execute(
             'git clone', clone_url, tmpdir.strpath)
 
         stdout, stderr = _add_files_and_push(
             'git', tmpdir.strpath,
             clone_url=rc_web_server.repo_clone_url('not-existing'))
 
         assert 'not found' in stderr
 
     def test_ip_restriction_hg(self, rc_web_server, tmpdir):
         user_model = UserModel()
         try:
             user_model.add_extra_ip(TEST_USER_ADMIN_LOGIN, '10.10.10.10/32')
             Session().commit()
             time.sleep(2)
             clone_url = rc_web_server.repo_clone_url(HG_REPO)
             stdout, stderr = Command('/tmp').execute(
                 'hg clone', clone_url, tmpdir.strpath)
             assert 'abort: HTTP Error 403: Forbidden' in stderr
         finally:
             # release IP restrictions
             for ip in UserIpMap.getAll():
                 UserIpMap.delete(ip.ip_id)
             Session().commit()
 
         time.sleep(2)
 
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         _check_proper_clone(stdout, stderr, 'hg')
 
     def test_ip_restriction_git(self, rc_web_server, tmpdir):
         user_model = UserModel()
         try:
             user_model.add_extra_ip(TEST_USER_ADMIN_LOGIN, '10.10.10.10/32')
             Session().commit()
             time.sleep(2)
             clone_url = rc_web_server.repo_clone_url(GIT_REPO)
             stdout, stderr = Command('/tmp').execute(
                 'git clone', clone_url, tmpdir.strpath)
             msg = "The requested URL returned error: 403"
             assert msg in stderr
         finally:
             # release IP restrictions
             for ip in UserIpMap.getAll():
                 UserIpMap.delete(ip.ip_id)
             Session().commit()
 
         time.sleep(2)
 
         cmd = Command('/tmp')
         stdout, stderr = cmd.execute('git clone', clone_url, tmpdir.strpath)
         cmd.assert_returncode_success()
         _check_proper_clone(stdout, stderr, 'git')
 
     def test_clone_by_auth_token(
             self, rc_web_server, tmpdir, user_util, enable_auth_plugins):
         enable_auth_plugins(['egg:rhodecode-enterprise-ce#token',
                              'egg:rhodecode-enterprise-ce#rhodecode'])
 
         user = user_util.create_user()
         token = user.auth_tokens[1]
 
         clone_url = rc_web_server.repo_clone_url(
             HG_REPO, user=user.username, passwd=token)
 
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         _check_proper_clone(stdout, stderr, 'hg')
 
     def test_clone_by_auth_token_expired(
             self, rc_web_server, tmpdir, user_util, enable_auth_plugins):
         enable_auth_plugins(['egg:rhodecode-enterprise-ce#token',
                              'egg:rhodecode-enterprise-ce#rhodecode'])
 
         user = user_util.create_user()
         auth_token = AuthTokenModel().create(
             user.user_id, 'test-token', -10, AuthTokenModel.cls.ROLE_VCS)
         token = auth_token.api_key
 
         clone_url = rc_web_server.repo_clone_url(
             HG_REPO, user=user.username, passwd=token)
 
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         assert 'abort: authorization failed' in stderr
 
     def test_clone_by_auth_token_bad_role(
             self, rc_web_server, tmpdir, user_util, enable_auth_plugins):
         enable_auth_plugins(['egg:rhodecode-enterprise-ce#token',
                              'egg:rhodecode-enterprise-ce#rhodecode'])
 
         user = user_util.create_user()
         auth_token = AuthTokenModel().create(
             user.user_id, 'test-token', -1, AuthTokenModel.cls.ROLE_API)
         token = auth_token.api_key
 
         clone_url = rc_web_server.repo_clone_url(
             HG_REPO, user=user.username, passwd=token)
 
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         assert 'abort: authorization failed' in stderr
 
     def test_clone_by_auth_token_user_disabled(
             self, rc_web_server, tmpdir, user_util, enable_auth_plugins):
         enable_auth_plugins(['egg:rhodecode-enterprise-ce#token',
                              'egg:rhodecode-enterprise-ce#rhodecode'])
         user = user_util.create_user()
         user.active = False
         Session().add(user)
         Session().commit()
         token = user.auth_tokens[1]
 
         clone_url = rc_web_server.repo_clone_url(
             HG_REPO, user=user.username, passwd=token)
 
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         assert 'abort: authorization failed' in stderr
 
     def test_clone_by_auth_token_with_scope(
             self, rc_web_server, tmpdir, user_util, enable_auth_plugins):
         enable_auth_plugins(['egg:rhodecode-enterprise-ce#token',
                              'egg:rhodecode-enterprise-ce#rhodecode'])
         user = user_util.create_user()
         auth_token = AuthTokenModel().create(
             user.user_id, 'test-token', -1, AuthTokenModel.cls.ROLE_VCS)
         token = auth_token.api_key
 
         # manually set scope
         auth_token.repo = Repository.get_by_repo_name(HG_REPO)
         Session().add(auth_token)
         Session().commit()
 
         clone_url = rc_web_server.repo_clone_url(
             HG_REPO, user=user.username, passwd=token)
 
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         _check_proper_clone(stdout, stderr, 'hg')
 
     def test_clone_by_auth_token_with_wrong_scope(
             self, rc_web_server, tmpdir, user_util, enable_auth_plugins):
         enable_auth_plugins(['egg:rhodecode-enterprise-ce#token',
                              'egg:rhodecode-enterprise-ce#rhodecode'])
         user = user_util.create_user()
         auth_token = AuthTokenModel().create(
             user.user_id, 'test-token', -1, AuthTokenModel.cls.ROLE_VCS)
         token = auth_token.api_key
 
         # manually set scope
         auth_token.repo = Repository.get_by_repo_name(GIT_REPO)
         Session().add(auth_token)
         Session().commit()
 
         clone_url = rc_web_server.repo_clone_url(
             HG_REPO, user=user.username, passwd=token)
 
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         assert 'abort: authorization failed' in stderr
 
 
 def test_git_sets_default_branch_if_not_master(
         backend_git, tmpdir, disable_locking, rc_web_server):
     empty_repo = backend_git.create_repo()
     clone_url = rc_web_server.repo_clone_url(empty_repo.repo_name)
 
     cmd = Command(tmpdir.strpath)
     cmd.execute('git clone', clone_url)
 
     repo = GitRepository(os.path.join(tmpdir.strpath, empty_repo.repo_name))
     repo.in_memory_commit.add(FileNode('file', content=''))
     repo.in_memory_commit.commit(
         message='Commit on branch test',
         author='Automatic test',
         branch='test')
 
     repo_cmd = Command(repo.path)
     stdout, stderr = repo_cmd.execute('git push --verbose origin test')
     _check_proper_git_push(
         stdout, stderr, branch='test', should_set_default_branch=True)
 
     stdout, stderr = cmd.execute(
         'git clone', clone_url, empty_repo.repo_name + '-clone')
     _check_proper_clone(stdout, stderr, 'git')
 
     # Doing an explicit commit in order to get latest user logs on MySQL
     Session().commit()
 
 
 def test_git_fetches_from_remote_repository_with_annotated_tags(
         backend_git, disable_locking, rc_web_server):
     # Note: This is a test specific to the git backend. It checks the
     # integration of fetching from a remote repository which contains
     # annotated tags.
 
     # Dulwich shows this specific behavior only when
     # operating against a remote repository.
     source_repo = backend_git['annotated-tag']
     target_vcs_repo = backend_git.create_repo().scm_instance()
     target_vcs_repo.fetch(rc_web_server.repo_clone_url(source_repo.repo_name))
 
 
 def test_git_push_shows_pull_request_refs(backend_git, rc_web_server, tmpdir):
     """
     test if remote info about refs is visible
     """
     empty_repo = backend_git.create_repo()
 
     clone_url = rc_web_server.repo_clone_url(empty_repo.repo_name)
 
     cmd = Command(tmpdir.strpath)
     cmd.execute('git clone', clone_url)
 
     repo = GitRepository(os.path.join(tmpdir.strpath, empty_repo.repo_name))
     repo.in_memory_commit.add(FileNode('readme.md', content='## Hello'))
     repo.in_memory_commit.commit(
         message='Commit on branch Master',
         author='Automatic test',
         branch='master')
 
     repo_cmd = Command(repo.path)
     stdout, stderr = repo_cmd.execute('git push --verbose origin master')
     _check_proper_git_push(stdout, stderr, branch='master')
 
     ref = '{}/{}/pull-request/new?branch=master'.format(
         rc_web_server.host_url(), empty_repo.repo_name)
     assert 'remote: RhodeCode: open pull request link: {}'.format(ref) in stderr
     assert 'remote: RhodeCode: push completed' in stderr
 
     # push on the same branch
     repo = GitRepository(os.path.join(tmpdir.strpath, empty_repo.repo_name))
     repo.in_memory_commit.add(FileNode('setup.py', content='print\n'))
     repo.in_memory_commit.commit(
         message='Commit2 on branch Master',
         author='Automatic test2',
         branch='master')
 
     repo_cmd = Command(repo.path)
     stdout, stderr = repo_cmd.execute('git push --verbose origin master')
     _check_proper_git_push(stdout, stderr, branch='master')
 
     assert 'remote: RhodeCode: open pull request link: {}'.format(ref) in stderr
     assert 'remote: RhodeCode: push completed' in stderr
 
     # new Branch
     repo = GitRepository(os.path.join(tmpdir.strpath, empty_repo.repo_name))
     repo.in_memory_commit.add(FileNode('feature1.py', content='## Hello world'))
     repo.in_memory_commit.commit(
         message='Commit on branch feature',
         author='Automatic test',
         branch='feature')
 
     repo_cmd = Command(repo.path)
     stdout, stderr = repo_cmd.execute('git push --verbose origin feature')
     _check_proper_git_push(stdout, stderr, branch='feature')
 
     ref = '{}/{}/pull-request/new?branch=feature'.format(
         rc_web_server.host_url(), empty_repo.repo_name)
     assert 'remote: RhodeCode: open pull request link: {}'.format(ref) in stderr
     assert 'remote: RhodeCode: push completed' in stderr
 
 
 def test_hg_push_shows_pull_request_refs(backend_hg, rc_web_server, tmpdir):
     empty_repo = backend_hg.create_repo()
 
     clone_url = rc_web_server.repo_clone_url(empty_repo.repo_name)
 
     cmd = Command(tmpdir.strpath)
     cmd.execute('hg clone', clone_url)
 
     repo = MercurialRepository(os.path.join(tmpdir.strpath, empty_repo.repo_name))
     repo.in_memory_commit.add(FileNode(u'readme.md', content=u'## Hello'))
     repo.in_memory_commit.commit(
         message=u'Commit on branch default',
         author=u'Automatic test',
         branch='default')
 
     repo_cmd = Command(repo.path)
     repo_cmd.execute('hg checkout default')
 
     stdout, stderr = repo_cmd.execute('hg push --verbose', clone_url)
     _check_proper_hg_push(stdout, stderr, branch='default')
 
     ref = '{}/{}/pull-request/new?branch=default'.format(
         rc_web_server.host_url(), empty_repo.repo_name)
     assert 'remote: RhodeCode: open pull request link: {}'.format(ref) in stdout
     assert 'remote: RhodeCode: push completed' in stdout
 
     # push on the same branch
     repo = MercurialRepository(os.path.join(tmpdir.strpath, empty_repo.repo_name))
     repo.in_memory_commit.add(FileNode(u'setup.py', content=u'print\n'))
     repo.in_memory_commit.commit(
         message=u'Commit2 on branch default',
         author=u'Automatic test2',
         branch=u'default')
 
     repo_cmd = Command(repo.path)
     repo_cmd.execute('hg checkout default')
 
     stdout, stderr = repo_cmd.execute('hg push --verbose', clone_url)
     _check_proper_hg_push(stdout, stderr, branch='default')
 
     assert 'remote: RhodeCode: open pull request link: {}'.format(ref) in stdout
     assert 'remote: RhodeCode: push completed' in stdout
 
     # new Branch
     repo = MercurialRepository(os.path.join(tmpdir.strpath, empty_repo.repo_name))
     repo.in_memory_commit.add(FileNode(u'feature1.py', content=u'## Hello world'))
     repo.in_memory_commit.commit(
         message=u'Commit on branch feature',
         author=u'Automatic test',
         branch=u'feature')
 
     repo_cmd = Command(repo.path)
     repo_cmd.execute('hg checkout feature')
 
     stdout, stderr = repo_cmd.execute('hg push --new-branch --verbose', clone_url)
     _check_proper_hg_push(stdout, stderr, branch='feature')
 
     ref = '{}/{}/pull-request/new?branch=feature'.format(
         rc_web_server.host_url(), empty_repo.repo_name)
     assert 'remote: RhodeCode: open pull request link: {}'.format(ref) in stdout
     assert 'remote: RhodeCode: push completed' in stdout
 
 
 def test_hg_push_shows_pull_request_refs_book(backend_hg, rc_web_server, tmpdir):
     empty_repo = backend_hg.create_repo()
 
     clone_url = rc_web_server.repo_clone_url(empty_repo.repo_name)
 
     cmd = Command(tmpdir.strpath)
     cmd.execute('hg clone', clone_url)
 
     repo = MercurialRepository(os.path.join(tmpdir.strpath, empty_repo.repo_name))
     repo.in_memory_commit.add(FileNode(u'readme.md', content=u'## Hello'))
     repo.in_memory_commit.commit(
         message=u'Commit on branch default',
         author=u'Automatic test',
         branch='default')
 
     repo_cmd = Command(repo.path)
     repo_cmd.execute('hg checkout default')
 
     stdout, stderr = repo_cmd.execute('hg push --verbose', clone_url)
     _check_proper_hg_push(stdout, stderr, branch='default')
 
     ref = '{}/{}/pull-request/new?branch=default'.format(
         rc_web_server.host_url(), empty_repo.repo_name)
     assert 'remote: RhodeCode: open pull request link: {}'.format(ref) in stdout
     assert 'remote: RhodeCode: push completed' in stdout
 
     # add bookmark
     repo = MercurialRepository(os.path.join(tmpdir.strpath, empty_repo.repo_name))
     repo.in_memory_commit.add(FileNode(u'setup.py', content=u'print\n'))
     repo.in_memory_commit.commit(
         message=u'Commit2 on branch default',
         author=u'Automatic test2',
         branch=u'default')
 
     repo_cmd = Command(repo.path)
     repo_cmd.execute('hg checkout default')
     repo_cmd.execute('hg bookmark feature2')
     stdout, stderr = repo_cmd.execute('hg push -B feature2 --verbose', clone_url)
     _check_proper_hg_push(stdout, stderr, branch='default')
 
     ref = '{}/{}/pull-request/new?branch=default'.format(
         rc_web_server.host_url(), empty_repo.repo_name)
     assert 'remote: RhodeCode: open pull request link: {}'.format(ref) in stdout
     ref = '{}/{}/pull-request/new?bookmark=feature2'.format(
         rc_web_server.host_url(), empty_repo.repo_name)
     assert 'remote: RhodeCode: open pull request link: {}'.format(ref) in stdout
     assert 'remote: RhodeCode: push completed' in stdout
     assert 'exporting bookmark feature2' in stdout