git-lfs: fixed bug #5399 git-lfs application failed to generate HTTPS urls properly.

rhodecode/lib/middleware/simplegit.py

@@ -1,155 +1,156 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 SimpleGit middleware for handling git protocol request (push/clone etc.)
 It's implemented with basic auth function
 """
 import os
 import re
 import logging
 import urlparse
 
 import rhodecode
 from rhodecode.lib import utils
 from rhodecode.lib import utils2
 from rhodecode.lib.middleware import simplevcs
 
 log = logging.getLogger(__name__)
 
 
 GIT_PROTO_PAT = re.compile(
     r'^/(.+)/(info/refs|info/lfs/(.+)|git-upload-pack|git-receive-pack)')
 GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))')
 
 
 def default_lfs_store():
     """
     Default lfs store location, it's consistent with Mercurials large file
     store which is in .cache/largefiles
     """
     from rhodecode.lib.vcs.backends.git import lfs_store
     user_home = os.path.expanduser("~")
     return lfs_store(user_home)
 
 
 class SimpleGit(simplevcs.SimpleVCS):
 
     SCM = 'git'
 
     def _get_repository_name(self, environ):
         """
         Gets repository name out of PATH_INFO header
 
         :param environ: environ where PATH_INFO is stored
         """
         repo_name = GIT_PROTO_PAT.match(environ['PATH_INFO']).group(1)
         # for GIT LFS, and bare format strip .git suffix from names
         if repo_name.endswith('.git'):
             repo_name = repo_name[:-4]
         return repo_name
 
     def _get_lfs_action(self, path, request_method):
         """
         return an action based on LFS requests type.
         Those routes are handled inside vcsserver app.
 
         batch           -> POST to /info/lfs/objects/batch  => PUSH/PULL
                         batch is based on the `operation.
                         that could be download or upload, but those are only
                         instructions to fetch so we return pull always
 
         download        -> GET  to /info/lfs/{oid}          => PULL
         upload          -> PUT  to /info/lfs/{oid}          => PUSH
 
         verification    -> POST to /info/lfs/verify         => PULL
 
         """
 
         match_obj = GIT_LFS_PROTO_PAT.match(path)
         _parts = match_obj.groups()
         repo_name, path, operation = _parts
         log.debug(
             'LFS: detecting operation based on following '
             'data: %s, req_method:%s', _parts, request_method)
 
         if operation == 'verify':
             return 'pull'
         elif operation == 'objects/batch':
             # batch sends back instructions for API to dl/upl we report it
             # as pull
             if request_method == 'POST':
                 return 'pull'
 
         elif operation:
             # probably a OID, upload  is PUT, download a GET
             if request_method == 'GET':
                 return 'pull'
             else:
                 return 'push'
 
         # if default not found require push, as action
         return 'push'
 
     _ACTION_MAPPING = {
         'git-receive-pack': 'push',
         'git-upload-pack': 'pull',
     }
 
     def _get_action(self, environ):
         """
         Maps git request commands into a pull or push command.
         In case of unknown/unexpected data, it returns 'pull' to be safe.
 
         :param environ:
         """
         path = environ['PATH_INFO']
 
         if path.endswith('/info/refs'):
             query = urlparse.parse_qs(environ['QUERY_STRING'])
             service_cmd = query.get('service', [''])[0]
             return self._ACTION_MAPPING.get(service_cmd, 'pull')
 
         elif GIT_LFS_PROTO_PAT.match(environ['PATH_INFO']):
             return self._get_lfs_action(
                 environ['PATH_INFO'], environ['REQUEST_METHOD'])
 
         elif path.endswith('/git-receive-pack'):
             return 'push'
         elif path.endswith('/git-upload-pack'):
             return 'pull'
 
         return 'pull'
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
         return self.scm_app.create_git_wsgi_app(
             repo_path, repo_name, config)
 
-    def _create_config(self, extras, repo_name):
+    def _create_config(self, extras, repo_name, scheme='http'):
         extras['git_update_server_info'] = utils2.str2bool(
             rhodecode.CONFIG.get('git_update_server_info'))
 
         config = utils.make_db_config(repo=repo_name)
         custom_store = config.get('vcs_git_lfs', 'store_location')
 
         extras['git_lfs_enabled'] = utils2.str2bool(
             config.get('vcs_git_lfs', 'enabled'))
         extras['git_lfs_store_path'] = custom_store or default_lfs_store()
+        extras['git_lfs_http_scheme'] = scheme
         return extras

rhodecode/lib/middleware/simplehg.py

@@ -1,160 +1,160 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 SimpleHG middleware for handling mercurial protocol request
 (push/clone etc.). It's implemented with basic auth function
 """
 
 import logging
 import urlparse
 import urllib
 
 from rhodecode.lib import utils
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.middleware import simplevcs
 
 log = logging.getLogger(__name__)
 
 
 class SimpleHg(simplevcs.SimpleVCS):
 
     SCM = 'hg'
 
     def _get_repository_name(self, environ):
         """
         Gets repository name out of PATH_INFO header
 
         :param environ: environ where PATH_INFO is stored
         """
         repo_name = environ['PATH_INFO']
         if repo_name and repo_name.startswith('/'):
             # remove only the first leading /
             repo_name = repo_name[1:]
         return repo_name.rstrip('/')
 
     _ACTION_MAPPING = {
         'changegroup': 'pull',
         'changegroupsubset': 'pull',
         'getbundle': 'pull',
         'stream_out': 'pull',
         'listkeys': 'pull',
         'between': 'pull',
         'branchmap': 'pull',
         'branches': 'pull',
         'clonebundles': 'pull',
         'capabilities': 'pull',
         'debugwireargs': 'pull',
         'heads': 'pull',
         'lookup': 'pull',
         'hello': 'pull',
         'known': 'pull',
 
         # largefiles
         'putlfile': 'push',
         'getlfile': 'pull',
         'statlfile': 'pull',
         'lheads': 'pull',
 
         # evolve
         'evoext_obshashrange_v1': 'pull',
         'evoext_obshash': 'pull',
         'evoext_obshash1': 'pull',
 
         'unbundle': 'push',
         'pushkey': 'push',
     }
 
     @classmethod
     def _get_xarg_headers(cls, environ):
         i = 1
         chunks = []  # gather chunks stored in multiple 'hgarg_N'
         while True:
             head = environ.get('HTTP_X_HGARG_{}'.format(i))
             if not head:
                 break
             i += 1
             chunks.append(urllib.unquote_plus(head))
         full_arg = ''.join(chunks)
         pref = 'cmds='
         if full_arg.startswith(pref):
             # strip the cmds= header defining our batch commands
             full_arg = full_arg[len(pref):]
         cmds = full_arg.split(';')
         return cmds
 
     @classmethod
     def _get_batch_cmd(cls, environ):
         """
         Handle batch command send commands. Those are ';' separated commands
         sent by batch command that server needs to execute. We need to extract
         those, and map them to our ACTION_MAPPING to get all push/pull commands
         specified in the batch
         """
         default = 'push'
         batch_cmds = []
         try:
             cmds = cls._get_xarg_headers(environ)
             for pair in cmds:
                 parts = pair.split(' ', 1)
                 if len(parts) != 2:
                     continue
                 # entry should be in a format `key ARGS`
                 cmd, args = parts
                 action = cls._ACTION_MAPPING.get(cmd, default)
                 batch_cmds.append(action)
         except Exception:
             log.exception('Failed to extract batch commands operations')
 
         # in case we failed, (e.g malformed data) assume it's PUSH sub-command
         # for safety
         return batch_cmds or [default]
 
     def _get_action(self, environ):
         """
         Maps mercurial request commands into a pull or push command.
         In case of unknown/unexpected data, it returns 'push' to be safe.
 
         :param environ:
         """
         default = 'push'
         query = urlparse.parse_qs(environ['QUERY_STRING'],
                                   keep_blank_values=True)
 
         if 'cmd' in query:
             cmd = query['cmd'][0]
             if cmd == 'batch':
                 cmds = self._get_batch_cmd(environ)
                 if 'push' in cmds:
                     return 'push'
                 else:
                     return 'pull'
             return self._ACTION_MAPPING.get(cmd, default)
 
         return default
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
         return self.scm_app.create_hg_wsgi_app(repo_path, repo_name, config)
 
-    def _create_config(self, extras, repo_name):
+    def _create_config(self, extras, repo_name, scheme='http'):
         config = utils.make_db_config(repo=repo_name)
         config.set('rhodecode', 'RC_SCM_DATA', json.dumps(extras))
 
         return config.serialize()

rhodecode/lib/middleware/simplesvn.py

@@ -1,228 +1,228 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import base64
 import logging
 import urllib
 import urlparse
 
 import requests
 from pyramid.httpexceptions import HTTPNotAcceptable
 
 from rhodecode.lib import rc_cache
 from rhodecode.lib.middleware import simplevcs
 from rhodecode.lib.utils import is_valid_repo
 from rhodecode.lib.utils2 import str2bool, safe_int
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.hooks_daemon import store_txn_id_data
 
 
 log = logging.getLogger(__name__)
 
 
 class SimpleSvnApp(object):
     IGNORED_HEADERS = [
         'connection', 'keep-alive', 'content-encoding',
         'transfer-encoding', 'content-length']
     rc_extras = {}
 
     def __init__(self, config):
         self.config = config
 
     def __call__(self, environ, start_response):
         request_headers = self._get_request_headers(environ)
         data = environ['wsgi.input']
         req_method = environ['REQUEST_METHOD']
         has_content_length = 'CONTENT_LENGTH' in environ
         path_info = self._get_url(
             self.config.get('subversion_http_server_url', ''), environ['PATH_INFO'])
         transfer_encoding = environ.get('HTTP_TRANSFER_ENCODING', '')
         log.debug('Handling: %s method via `%s`', req_method, path_info)
 
         # stream control flag, based on request and content type...
         stream = False
 
         if req_method in ['MKCOL'] or has_content_length:
             data_processed = False
             # read chunk to check if we have txn-with-props
             initial_data = data.read(1024)
             if initial_data.startswith('(create-txn-with-props'):
                 data = initial_data + data.read()
                 # store on-the-fly our rc_extra using svn revision properties
                 # those can be read later on in hooks executed so we have a way
                 # to pass in the data into svn hooks
                 rc_data = base64.urlsafe_b64encode(json.dumps(self.rc_extras))
                 rc_data_len = len(rc_data)
                 # header defines data length, and serialized data
                 skel = ' rc-scm-extras {} {}'.format(rc_data_len, rc_data)
                 data = data[:-2] + skel + '))'
                 data_processed = True
 
             if not data_processed:
                 # NOTE(johbo): Avoid that we end up with sending the request in chunked
                 # transfer encoding (mainly on Gunicorn). If we know the content
                 # length, then we should transfer the payload in one request.
                 data = initial_data + data.read()
 
         if req_method in ['GET', 'PUT'] or transfer_encoding == 'chunked':
             # NOTE(marcink): when getting/uploading files we want to STREAM content
             # back to the client/proxy instead of buffering it here...
             stream = True
 
         stream = stream
         log.debug('Calling SVN PROXY at `%s`, using method:%s. Stream: %s',
                   path_info, req_method, stream)
         try:
             response = requests.request(
                 req_method, path_info,
                 data=data, headers=request_headers, stream=stream)
         except requests.ConnectionError:
             log.exception('ConnectionError occurred for endpoint %s', path_info)
             raise
 
         if response.status_code not in [200, 401]:
             text = '\n{}'.format(response.text) if response.text else ''
             if response.status_code >= 500:
                 log.error('Got SVN response:%s with text:`%s`', response, text)
             else:
                 log.debug('Got SVN response:%s with text:`%s`', response, text)
         else:
             log.debug('got response code: %s', response.status_code)
 
         response_headers = self._get_response_headers(response.headers)
 
         if response.headers.get('SVN-Txn-name'):
             svn_tx_id = response.headers.get('SVN-Txn-name')
             txn_id = rc_cache.utils.compute_key_from_params(
                 self.config['repository'], svn_tx_id)
             port = safe_int(self.rc_extras['hooks_uri'].split(':')[-1])
             store_txn_id_data(txn_id, {'port': port})
 
         start_response(
             '{} {}'.format(response.status_code, response.reason),
             response_headers)
         return response.iter_content(chunk_size=1024)
 
     def _get_url(self, svn_http_server, path):
         svn_http_server_url = (svn_http_server or '').rstrip('/')
         url_path = urlparse.urljoin(svn_http_server_url + '/', (path or '').lstrip('/'))
         url_path = urllib.quote(url_path, safe="/:=~+!$,;'")
         return url_path
 
     def _get_request_headers(self, environ):
         headers = {}
 
         for key in environ:
             if not key.startswith('HTTP_'):
                 continue
             new_key = key.split('_')
             new_key = [k.capitalize() for k in new_key[1:]]
             new_key = '-'.join(new_key)
             headers[new_key] = environ[key]
 
         if 'CONTENT_TYPE' in environ:
             headers['Content-Type'] = environ['CONTENT_TYPE']
 
         if 'CONTENT_LENGTH' in environ:
             headers['Content-Length'] = environ['CONTENT_LENGTH']
 
         return headers
 
     def _get_response_headers(self, headers):
         headers = [
             (h, headers[h])
             for h in headers
             if h.lower() not in self.IGNORED_HEADERS
         ]
 
         return headers
 
 
 class DisabledSimpleSvnApp(object):
     def __init__(self, config):
         self.config = config
 
     def __call__(self, environ, start_response):
         reason = 'Cannot handle SVN call because: SVN HTTP Proxy is not enabled'
         log.warning(reason)
         return HTTPNotAcceptable(reason)(environ, start_response)
 
 
 class SimpleSvn(simplevcs.SimpleVCS):
 
     SCM = 'svn'
     READ_ONLY_COMMANDS = ('OPTIONS', 'PROPFIND', 'GET', 'REPORT')
     DEFAULT_HTTP_SERVER = 'http://localhost:8090'
 
     def _get_repository_name(self, environ):
         """
         Gets repository name out of PATH_INFO header
 
         :param environ: environ where PATH_INFO is stored
         """
         path = environ['PATH_INFO'].split('!')
         repo_name = path[0].strip('/')
 
         # SVN includes the whole path in it's requests, including
         # subdirectories inside the repo. Therefore we have to search for
         # the repo root directory.
         if not is_valid_repo(
                 repo_name, self.base_path, explicit_scm=self.SCM):
             current_path = ''
             for component in repo_name.split('/'):
                 current_path += component
                 if is_valid_repo(
                         current_path, self.base_path, explicit_scm=self.SCM):
                     return current_path
                 current_path += '/'
 
         return repo_name
 
     def _get_action(self, environ):
         return (
             'pull'
             if environ['REQUEST_METHOD'] in self.READ_ONLY_COMMANDS
             else 'push')
 
     def _should_use_callback_daemon(self, extras, environ, action):
         # only MERGE command triggers hooks, so we don't want to start
         # hooks server too many times. POST however starts the svn transaction
         # so we also need to run the init of callback daemon of POST
         if environ['REQUEST_METHOD'] in ['MERGE', 'POST']:
             return True
         return False
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
         if self._is_svn_enabled():
             return SimpleSvnApp(config)
         # we don't have http proxy enabled return dummy request handler
         return DisabledSimpleSvnApp(config)
 
     def _is_svn_enabled(self):
         conf = self.repo_vcs_config
         return str2bool(conf.get('vcs_svn_proxy', 'http_requests_enabled'))
 
-    def _create_config(self, extras, repo_name):
+    def _create_config(self, extras, repo_name, scheme='http'):
         conf = self.repo_vcs_config
         server_url = conf.get('vcs_svn_proxy', 'http_server_url')
         server_url = server_url or self.DEFAULT_HTTP_SERVER
 
         extras['subversion_http_server_url'] = server_url
         return extras

rhodecode/lib/middleware/simplevcs.py

@@ -1,669 +1,678 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 SimpleVCS middleware for handling protocol request (push/clone etc.)
 It's implemented with basic auth function
 """
 
 import os
 import re
 import logging
 import importlib
 from functools import wraps
 from StringIO import StringIO
 from lxml import etree
 
 import time
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 
 from pyramid.httpexceptions import (
     HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 import rhodecode
 from rhodecode.authentication.base import authenticate, VCS_TYPE, loadplugin
 from rhodecode.lib import rc_cache
 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
 from rhodecode.lib.base import (
     BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
 from rhodecode.lib.exceptions import (UserCreationError, NotAllowedToCreateUserError)
 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
 from rhodecode.lib.middleware import appenlight
 from rhodecode.lib.middleware.utils import scm_app_http
 from rhodecode.lib.utils import is_valid_repo, SLUG_RE
 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool, safe_unicode
 from rhodecode.lib.vcs.conf import settings as vcs_settings
 from rhodecode.lib.vcs.backends import base
 
 from rhodecode.model import meta
 from rhodecode.model.db import User, Repository, PullRequest
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.pull_request import PullRequestModel
 from rhodecode.model.settings import SettingsModel, VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
 
 def extract_svn_txn_id(acl_repo_name, data):
     """
     Helper method for extraction of svn txn_id from submitted XML data during
     POST operations
     """
     try:
         root = etree.fromstring(data)
         pat = re.compile(r'/txn/(?P<txn_id>.*)')
         for el in root:
             if el.tag == '{DAV:}source':
                 for sub_el in el:
                     if sub_el.tag == '{DAV:}href':
                         match = pat.search(sub_el.text)
                         if match:
                             svn_tx_id = match.groupdict()['txn_id']
                             txn_id = rc_cache.utils.compute_key_from_params(
                                 acl_repo_name, svn_tx_id)
                             return txn_id
     except Exception:
         log.exception('Failed to extract txn_id')
 
 
 def initialize_generator(factory):
     """
     Initializes the returned generator by draining its first element.
 
     This can be used to give a generator an initializer, which is the code
     up to the first yield statement. This decorator enforces that the first
     produced element has the value ``"__init__"`` to make its special
     purpose very explicit in the using code.
     """
 
     @wraps(factory)
     def wrapper(*args, **kwargs):
         gen = factory(*args, **kwargs)
         try:
             init = gen.next()
         except StopIteration:
             raise ValueError('Generator must yield at least one element.')
         if init != "__init__":
             raise ValueError('First yielded element must be "__init__".')
         return gen
     return wrapper
 
 
 class SimpleVCS(object):
     """Common functionality for SCM HTTP handlers."""
 
     SCM = 'unknown'
 
     acl_repo_name = None
     url_repo_name = None
     vcs_repo_name = None
     rc_extras = {}
 
     # We have to handle requests to shadow repositories different than requests
     # to normal repositories. Therefore we have to distinguish them. To do this
     # we use this regex which will match only on URLs pointing to shadow
     # repositories.
     shadow_repo_re = re.compile(
         '(?P<groups>(?:{slug_pat}/)*)'  # repo groups
         '(?P<target>{slug_pat})/'       # target repo
         'pull-request/(?P<pr_id>\d+)/'  # pull request
         'repository$'                   # shadow repo
         .format(slug_pat=SLUG_RE.pattern))
 
     def __init__(self, config, registry):
         self.registry = registry
         self.config = config
         # re-populated by specialized middleware
         self.repo_vcs_config = base.Config()
         self.rhodecode_settings = SettingsModel().get_all_settings(cache=True)
 
         registry.rhodecode_settings = self.rhodecode_settings
         # authenticate this VCS request using authfunc
         auth_ret_code_detection = \
             str2bool(self.config.get('auth_ret_code_detection', False))
         self.authenticate = BasicAuth(
             '', authenticate, registry, config.get('auth_ret_code'),
             auth_ret_code_detection)
         self.ip_addr = '0.0.0.0'
 
     @LazyProperty
     def global_vcs_config(self):
         try:
             return VcsSettingsModel().get_ui_settings_as_config_obj()
         except Exception:
             return base.Config()
 
     @property
     def base_path(self):
         settings_path = self.repo_vcs_config.get(*VcsSettingsModel.PATH_SETTING)
 
         if not settings_path:
             settings_path = self.global_vcs_config.get(*VcsSettingsModel.PATH_SETTING)
 
         if not settings_path:
             # try, maybe we passed in explicitly as config option
             settings_path = self.config.get('base_path')
 
         if not settings_path:
             raise ValueError('FATAL: base_path is empty')
         return settings_path
 
     def set_repo_names(self, environ):
         """
         This will populate the attributes acl_repo_name, url_repo_name,
         vcs_repo_name and is_shadow_repo. In case of requests to normal (non
         shadow) repositories all names are equal. In case of requests to a
         shadow repository the acl-name points to the target repo of the pull
         request and the vcs-name points to the shadow repo file system path.
         The url-name is always the URL used by the vcs client program.
 
         Example in case of a shadow repo:
             acl_repo_name = RepoGroup/MyRepo
             url_repo_name = RepoGroup/MyRepo/pull-request/3/repository
             vcs_repo_name = /repo/base/path/RepoGroup/.__shadow_MyRepo_pr-3'
         """
         # First we set the repo name from URL for all attributes. This is the
         # default if handling normal (non shadow) repo requests.
         self.url_repo_name = self._get_repository_name(environ)
         self.acl_repo_name = self.vcs_repo_name = self.url_repo_name
         self.is_shadow_repo = False
 
         # Check if this is a request to a shadow repository.
         match = self.shadow_repo_re.match(self.url_repo_name)
         if match:
             match_dict = match.groupdict()
 
             # Build acl repo name from regex match.
             acl_repo_name = safe_unicode('{groups}{target}'.format(
                 groups=match_dict['groups'] or '',
                 target=match_dict['target']))
 
             # Retrieve pull request instance by ID from regex match.
             pull_request = PullRequest.get(match_dict['pr_id'])
 
             # Only proceed if we got a pull request and if acl repo name from
             # URL equals the target repo name of the pull request.
             if pull_request and \
                     (acl_repo_name == pull_request.target_repo.repo_name):
                 repo_id = pull_request.target_repo.repo_id
                 # Get file system path to shadow repository.
                 workspace_id = PullRequestModel()._workspace_id(pull_request)
                 target_vcs = pull_request.target_repo.scm_instance()
                 vcs_repo_name = target_vcs._get_shadow_repository_path(
                     repo_id, workspace_id)
 
                 # Store names for later usage.
                 self.vcs_repo_name = vcs_repo_name
                 self.acl_repo_name = acl_repo_name
                 self.is_shadow_repo = True
 
         log.debug('Setting all VCS repository names: %s', {
             'acl_repo_name': self.acl_repo_name,
             'url_repo_name': self.url_repo_name,
             'vcs_repo_name': self.vcs_repo_name,
         })
 
     @property
     def scm_app(self):
         custom_implementation = self.config['vcs.scm_app_implementation']
         if custom_implementation == 'http':
             log.info('Using HTTP implementation of scm app.')
             scm_app_impl = scm_app_http
         else:
             log.info('Using custom implementation of scm_app: "{}"'.format(
                 custom_implementation))
             scm_app_impl = importlib.import_module(custom_implementation)
         return scm_app_impl
 
     def _get_by_id(self, repo_name):
         """
         Gets a special pattern _<ID> from clone url and tries to replace it
         with a repository_name for support of _<ID> non changeable urls
         """
 
         data = repo_name.split('/')
         if len(data) >= 2:
             from rhodecode.model.repo import RepoModel
             by_id_match = RepoModel().get_repo_by_id(repo_name)
             if by_id_match:
                 data[1] = by_id_match.repo_name
 
         return safe_str('/'.join(data))
 
     def _invalidate_cache(self, repo_name):
         """
         Set's cache for this repository for invalidation on next access
 
         :param repo_name: full repo name, also a cache key
         """
         ScmModel().mark_for_invalidation(repo_name)
 
     def is_valid_and_existing_repo(self, repo_name, base_path, scm_type):
         db_repo = Repository.get_by_repo_name(repo_name)
         if not db_repo:
             log.debug('Repository `%s` not found inside the database.',
                       repo_name)
             return False
 
         if db_repo.repo_type != scm_type:
             log.warning(
                 'Repository `%s` have incorrect scm_type, expected %s got %s',
                 repo_name, db_repo.repo_type, scm_type)
             return False
 
         config = db_repo._config
         config.set('extensions', 'largefiles', '')
         return is_valid_repo(
             repo_name, base_path,
             explicit_scm=scm_type, expect_scm=scm_type, config=config)
 
     def valid_and_active_user(self, user):
         """
         Checks if that user is not empty, and if it's actually object it checks
         if he's active.
 
         :param user: user object or None
         :return: boolean
         """
         if user is None:
             return False
 
         elif user.active:
             return True
 
         return False
 
     @property
     def is_shadow_repo_dir(self):
         return os.path.isdir(self.vcs_repo_name)
 
     def _check_permission(self, action, user, auth_user, repo_name, ip_addr=None,
                           plugin_id='', plugin_cache_active=False, cache_ttl=0):
         """
         Checks permissions using action (push/pull) user and repository
         name. If plugin_cache and ttl is set it will use the plugin which
         authenticated the user to store the cached permissions result for N
         amount of seconds as in cache_ttl
 
         :param action: push or pull action
         :param user: user instance
         :param repo_name: repository name
         """
 
         log.debug('AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)',
                   plugin_id, plugin_cache_active, cache_ttl)
 
         user_id = user.user_id
         cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
         region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid,
                                                expiration_time=cache_ttl,
                                                condition=plugin_cache_active)
         def compute_perm_vcs(
                 cache_name, plugin_id, action, user_id, repo_name, ip_addr):
 
             log.debug('auth: calculating permission access now...')
             # check IP
             inherit = user.inherit_default_permissions
             ip_allowed = AuthUser.check_ip_allowed(
                 user_id, ip_addr, inherit_from_default=inherit)
             if ip_allowed:
                 log.info('Access for IP:%s allowed', ip_addr)
             else:
                 return False
 
             if action == 'push':
                 perms = ('repository.write', 'repository.admin')
                 if not HasPermissionAnyMiddleware(*perms)(auth_user, repo_name):
                     return False
 
             else:
                 # any other action need at least read permission
                 perms = (
                     'repository.read', 'repository.write', 'repository.admin')
                 if not HasPermissionAnyMiddleware(*perms)(auth_user, repo_name):
                     return False
 
             return True
 
         start = time.time()
         log.debug('Running plugin `%s` permissions check', plugin_id)
 
         # for environ based auth, password can be empty, but then the validation is
         # on the server that fills in the env data needed for authentication
         perm_result = compute_perm_vcs(
             'vcs_permissions', plugin_id, action, user.user_id, repo_name, ip_addr)
 
         auth_time = time.time() - start
         log.debug('Permissions for plugin `%s` completed in %.3fs, '
                   'expiration time of fetched cache %.1fs.',
                   plugin_id, auth_time, cache_ttl)
 
         return perm_result
 
+    def _get_http_scheme(self, environ):
+        try:
+            return environ['wsgi.url_scheme']
+        except Exception:
+            log.exception('Failed to read http scheme')
+            return 'http'
+
     def _check_ssl(self, environ, start_response):
         """
         Checks the SSL check flag and returns False if SSL is not present
         and required True otherwise
         """
         org_proto = environ['wsgi._org_proto']
         # check if we have SSL required  ! if not it's a bad request !
         require_ssl = str2bool(self.repo_vcs_config.get('web', 'push_ssl'))
         if require_ssl and org_proto == 'http':
             log.debug(
                 'Bad request: detected protocol is `%s` and '
                 'SSL/HTTPS is required.', org_proto)
             return False
         return True
 
     def _get_default_cache_ttl(self):
         # take AUTH_CACHE_TTL from the `rhodecode` auth plugin
         plugin = loadplugin('egg:rhodecode-enterprise-ce#rhodecode')
         plugin_settings = plugin.get_settings()
         plugin_cache_active, cache_ttl = plugin.get_ttl_cache(
             plugin_settings) or (False, 0)
         return plugin_cache_active, cache_ttl
 
     def __call__(self, environ, start_response):
         try:
             return self._handle_request(environ, start_response)
         except Exception:
             log.exception("Exception while handling request")
             appenlight.track_exception(environ)
             return HTTPInternalServerError()(environ, start_response)
         finally:
             meta.Session.remove()
 
     def _handle_request(self, environ, start_response):
         if not self._check_ssl(environ, start_response):
             reason = ('SSL required, while RhodeCode was unable '
                       'to detect this as SSL request')
             log.debug('User not allowed to proceed, %s', reason)
             return HTTPNotAcceptable(reason)(environ, start_response)
 
         if not self.url_repo_name:
             log.warning('Repository name is empty: %s', self.url_repo_name)
             # failed to get repo name, we fail now
             return HTTPNotFound()(environ, start_response)
         log.debug('Extracted repo name is %s', self.url_repo_name)
 
         ip_addr = get_ip_addr(environ)
         user_agent = get_user_agent(environ)
         username = None
 
         # skip passing error to error controller
         environ['pylons.status_code_redirect'] = True
 
         # ======================================================================
         # GET ACTION PULL or PUSH
         # ======================================================================
         action = self._get_action(environ)
 
         # ======================================================================
         # Check if this is a request to a shadow repository of a pull request.
         # In this case only pull action is allowed.
         # ======================================================================
         if self.is_shadow_repo and action != 'pull':
             reason = 'Only pull action is allowed for shadow repositories.'
             log.debug('User not allowed to proceed, %s', reason)
             return HTTPNotAcceptable(reason)(environ, start_response)
 
         # Check if the shadow repo actually exists, in case someone refers
         # to it, and it has been deleted because of successful merge.
         if self.is_shadow_repo and not self.is_shadow_repo_dir:
             log.debug(
                 'Shadow repo detected, and shadow repo dir `%s` is missing',
                 self.is_shadow_repo_dir)
             return HTTPNotFound()(environ, start_response)
 
         # ======================================================================
         # CHECK ANONYMOUS PERMISSION
         # ======================================================================
         detect_force_push = False
         check_branch_perms = False
         if action in ['pull', 'push']:
             user_obj = anonymous_user = User.get_default_user()
             auth_user = user_obj.AuthUser()
             username = anonymous_user.username
             if anonymous_user.active:
                 plugin_cache_active, cache_ttl = self._get_default_cache_ttl()
                 # ONLY check permissions if the user is activated
                 anonymous_perm = self._check_permission(
                     action, anonymous_user, auth_user, self.acl_repo_name, ip_addr,
                     plugin_id='anonymous_access',
                     plugin_cache_active=plugin_cache_active,
                     cache_ttl=cache_ttl,
                 )
             else:
                 anonymous_perm = False
 
             if not anonymous_user.active or not anonymous_perm:
                 if not anonymous_user.active:
                     log.debug('Anonymous access is disabled, running '
                               'authentication')
 
                 if not anonymous_perm:
                     log.debug('Not enough credentials to access this '
                               'repository as anonymous user')
 
                 username = None
                 # ==============================================================
                 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
                 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
                 # ==============================================================
 
                 # try to auth based on environ, container auth methods
                 log.debug('Running PRE-AUTH for container based authentication')
                 pre_auth = authenticate(
                     '', '', environ, VCS_TYPE, registry=self.registry,
                     acl_repo_name=self.acl_repo_name)
                 if pre_auth and pre_auth.get('username'):
                     username = pre_auth['username']
                 log.debug('PRE-AUTH got %s as username', username)
                 if pre_auth:
                     log.debug('PRE-AUTH successful from %s',
                               pre_auth.get('auth_data', {}).get('_plugin'))
 
                 # If not authenticated by the container, running basic auth
                 # before inject the calling repo_name for special scope checks
                 self.authenticate.acl_repo_name = self.acl_repo_name
 
                 plugin_cache_active, cache_ttl = False, 0
                 plugin = None
                 if not username:
                     self.authenticate.realm = self.authenticate.get_rc_realm()
 
                     try:
                         auth_result = self.authenticate(environ)
                     except (UserCreationError, NotAllowedToCreateUserError) as e:
                         log.error(e)
                         reason = safe_str(e)
                         return HTTPNotAcceptable(reason)(environ, start_response)
 
                     if isinstance(auth_result, dict):
                         AUTH_TYPE.update(environ, 'basic')
                         REMOTE_USER.update(environ, auth_result['username'])
                         username = auth_result['username']
                         plugin = auth_result.get('auth_data', {}).get('_plugin')
                         log.info(
                             'MAIN-AUTH successful for user `%s` from %s plugin',
                             username, plugin)
 
                         plugin_cache_active, cache_ttl = auth_result.get(
                             'auth_data', {}).get('_ttl_cache') or (False, 0)
                     else:
                         return auth_result.wsgi_application(environ, start_response)
 
                 # ==============================================================
                 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
                 # ==============================================================
                 user = User.get_by_username(username)
                 if not self.valid_and_active_user(user):
                     return HTTPForbidden()(environ, start_response)
                 username = user.username
                 user_id = user.user_id
 
                 # check user attributes for password change flag
                 user_obj = user
                 auth_user = user_obj.AuthUser()
                 if user_obj and user_obj.username != User.DEFAULT_USER and \
                         user_obj.user_data.get('force_password_change'):
                     reason = 'password change required'
                     log.debug('User not allowed to authenticate, %s', reason)
                     return HTTPNotAcceptable(reason)(environ, start_response)
 
                 # check permissions for this repository
                 perm = self._check_permission(
                     action, user, auth_user, self.acl_repo_name, ip_addr,
                     plugin, plugin_cache_active, cache_ttl)
                 if not perm:
                     return HTTPForbidden()(environ, start_response)
                 environ['rc_auth_user_id'] = user_id
 
             if action == 'push':
                 perms = auth_user.get_branch_permissions(self.acl_repo_name)
                 if perms:
                     check_branch_perms = True
                     detect_force_push = True
 
         # extras are injected into UI object and later available
         # in hooks executed by RhodeCode
         check_locking = _should_check_locking(environ.get('QUERY_STRING'))
 
         extras = vcs_operation_context(
             environ, repo_name=self.acl_repo_name, username=username,
             action=action, scm=self.SCM, check_locking=check_locking,
             is_shadow_repo=self.is_shadow_repo, check_branch_perms=check_branch_perms,
             detect_force_push=detect_force_push
         )
 
         # ======================================================================
         # REQUEST HANDLING
         # ======================================================================
         repo_path = os.path.join(
             safe_str(self.base_path), safe_str(self.vcs_repo_name))
         log.debug('Repository path is %s', repo_path)
 
         fix_PATH()
 
         log.info(
             '%s action on %s repo "%s" by "%s" from %s %s',
             action, self.SCM, safe_str(self.url_repo_name),
             safe_str(username), ip_addr, user_agent)
 
         return self._generate_vcs_response(
             environ, start_response, repo_path, extras, action)
 
     @initialize_generator
     def _generate_vcs_response(
             self, environ, start_response, repo_path, extras, action):
         """
         Returns a generator for the response content.
 
         This method is implemented as a generator, so that it can trigger
         the cache validation after all content sent back to the client. It
         also handles the locking exceptions which will be triggered when
         the first chunk is produced by the underlying WSGI application.
         """
         txn_id = ''
         if 'CONTENT_LENGTH' in environ and environ['REQUEST_METHOD'] == 'MERGE':
             # case for SVN, we want to re-use the callback daemon port
             # so we use the txn_id, for this we peek the body, and still save
             # it as wsgi.input
             data = environ['wsgi.input'].read()
             environ['wsgi.input'] = StringIO(data)
             txn_id = extract_svn_txn_id(self.acl_repo_name, data)
 
         callback_daemon, extras = self._prepare_callback_daemon(
             extras, environ, action, txn_id=txn_id)
         log.debug('HOOKS extras is %s', extras)
 
-        config = self._create_config(extras, self.acl_repo_name)
+        http_scheme = self._get_http_scheme(environ)
+
+        config = self._create_config(extras, self.acl_repo_name, scheme=http_scheme)
         app = self._create_wsgi_app(repo_path, self.url_repo_name, config)
         with callback_daemon:
             app.rc_extras = extras
 
             try:
                 response = app(environ, start_response)
             finally:
                 # This statement works together with the decorator
                 # "initialize_generator" above. The decorator ensures that
                 # we hit the first yield statement before the generator is
                 # returned back to the WSGI server. This is needed to
                 # ensure that the call to "app" above triggers the
                 # needed callback to "start_response" before the
                 # generator is actually used.
                 yield "__init__"
 
             # iter content
             for chunk in response:
                 yield chunk
 
             try:
                 # invalidate cache on push
                 if action == 'push':
                     self._invalidate_cache(self.url_repo_name)
             finally:
                 meta.Session.remove()
 
     def _get_repository_name(self, environ):
         """Get repository name out of the environmnent
 
         :param environ: WSGI environment
         """
         raise NotImplementedError()
 
     def _get_action(self, environ):
         """Map request commands into a pull or push command.
 
         :param environ: WSGI environment
         """
         raise NotImplementedError()
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
         """Return the WSGI app that will finally handle the request."""
         raise NotImplementedError()
 
-    def _create_config(self, extras, repo_name):
+    def _create_config(self, extras, repo_name, scheme='http'):
         """Create a safe config representation."""
         raise NotImplementedError()
 
     def _should_use_callback_daemon(self, extras, environ, action):
         return True
 
     def _prepare_callback_daemon(self, extras, environ, action, txn_id=None):
         direct_calls = vcs_settings.HOOKS_DIRECT_CALLS
         if not self._should_use_callback_daemon(extras, environ, action):
             # disable callback daemon for actions that don't require it
             direct_calls = True
 
         return prepare_callback_daemon(
             extras, protocol=vcs_settings.HOOKS_PROTOCOL,
             host=vcs_settings.HOOKS_HOST, use_direct_calls=direct_calls, txn_id=txn_id)
 
 
 def _should_check_locking(query_string):
     # this is kind of hacky, but due to how mercurial handles client-server
     # server see all operation on commit; bookmarks, phases and
     # obsolescence marker in different transaction, we don't want to check
     # locking on those
     return query_string not in ['cmd=listkeys']

rhodecode/tests/lib/middleware/test_simplegit.py

@@ -1,139 +1,140 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import pytest
 import urlparse
 import mock
 import simplejson as json
 
 from rhodecode.lib.vcs.backends.base import Config
 from rhodecode.tests.lib.middleware import mock_scm_app
 import rhodecode.lib.middleware.simplegit as simplegit
 
 
 def get_environ(url, request_method):
     """Construct a minimum WSGI environ based on the URL."""
     parsed_url = urlparse.urlparse(url)
     environ = {
         'PATH_INFO': parsed_url.path,
         'QUERY_STRING': parsed_url.query,
         'REQUEST_METHOD': request_method,
     }
 
     return environ
 
 
 @pytest.mark.parametrize(
     'url, expected_action, request_method',
     [
         ('/foo/bar/info/refs?service=git-upload-pack', 'pull', 'GET'),
         ('/foo/bar/info/refs?service=git-receive-pack', 'push', 'GET'),
         ('/foo/bar/git-upload-pack', 'pull', 'GET'),
         ('/foo/bar/git-receive-pack', 'push', 'GET'),
         # Edge case: missing data for info/refs
         ('/foo/info/refs?service=', 'pull', 'GET'),
         ('/foo/info/refs', 'pull', 'GET'),
         # Edge case: git command comes with service argument
         ('/foo/git-upload-pack?service=git-receive-pack', 'pull', 'GET'),
         ('/foo/git-receive-pack?service=git-upload-pack', 'push', 'GET'),
         # Edge case: repo name conflicts with git commands
         ('/git-receive-pack/git-upload-pack', 'pull', 'GET'),
         ('/git-receive-pack/git-receive-pack', 'push', 'GET'),
         ('/git-upload-pack/git-upload-pack', 'pull', 'GET'),
         ('/git-upload-pack/git-receive-pack', 'push', 'GET'),
         ('/foo/git-receive-pack', 'push', 'GET'),
         # Edge case: not a smart protocol url
         ('/foo/bar', 'pull', 'GET'),
         # GIT LFS cases, batch
         ('/foo/bar/info/lfs/objects/batch', 'push', 'GET'),
         ('/foo/bar/info/lfs/objects/batch', 'pull', 'POST'),
         # GIT LFS oid, dl/upl
         ('/foo/bar/info/lfs/abcdeabcde', 'pull', 'GET'),
         ('/foo/bar/info/lfs/abcdeabcde', 'push', 'PUT'),
         ('/foo/bar/info/lfs/abcdeabcde', 'push', 'POST'),
         # Edge case: repo name conflicts with git commands
         ('/info/lfs/info/lfs/objects/batch', 'push', 'GET'),
         ('/info/lfs/info/lfs/objects/batch', 'pull', 'POST'),
 
     ])
 def test_get_action(url, expected_action, request_method, baseapp, request_stub):
     app = simplegit.SimpleGit(config={'auth_ret_code': '', 'base_path': ''},
                               registry=request_stub.registry)
     assert expected_action == app._get_action(get_environ(url, request_method))
 
 
 @pytest.mark.parametrize(
     'url, expected_repo_name, request_method',
     [
         ('/foo/info/refs?service=git-upload-pack', 'foo', 'GET'),
         ('/foo/bar/info/refs?service=git-receive-pack', 'foo/bar', 'GET'),
         ('/foo/git-upload-pack', 'foo', 'GET'),
         ('/foo/git-receive-pack', 'foo', 'GET'),
         ('/foo/bar/git-upload-pack', 'foo/bar', 'GET'),
         ('/foo/bar/git-receive-pack', 'foo/bar', 'GET'),
 
         # GIT LFS cases, batch
         ('/foo/bar/info/lfs/objects/batch', 'foo/bar', 'GET'),
         ('/example-git/info/lfs/objects/batch', 'example-git', 'POST'),
         # GIT LFS oid, dl/upl
         ('/foo/info/lfs/abcdeabcde', 'foo', 'GET'),
         ('/foo/bar/info/lfs/abcdeabcde', 'foo/bar', 'PUT'),
         ('/my-git-repo/info/lfs/abcdeabcde', 'my-git-repo', 'POST'),
         # Edge case: repo name conflicts with git commands
         ('/info/lfs/info/lfs/objects/batch', 'info/lfs', 'GET'),
         ('/info/lfs/info/lfs/objects/batch', 'info/lfs', 'POST'),
 
     ])
 def test_get_repository_name(url, expected_repo_name, request_method, baseapp, request_stub):
     app = simplegit.SimpleGit(config={'auth_ret_code': '', 'base_path': ''},
                               registry=request_stub.registry)
     assert expected_repo_name == app._get_repository_name(
         get_environ(url, request_method))
 
 
 def test_get_config(user_util, baseapp, request_stub):
     repo = user_util.create_repo(repo_type='git')
     app = simplegit.SimpleGit(config={'auth_ret_code': '', 'base_path': ''},
                               registry=request_stub.registry)
     extras = {'foo': 'FOO', 'bar': 'BAR'}
 
     # We copy the extras as the method below will change the contents.
     git_config = app._create_config(dict(extras), repo_name=repo.repo_name)
 
     expected_config = dict(extras)
     expected_config.update({
         'git_update_server_info': False,
         'git_lfs_enabled': False,
-        'git_lfs_store_path': git_config['git_lfs_store_path']
+        'git_lfs_store_path': git_config['git_lfs_store_path'],
+        'git_lfs_http_scheme': 'http'
     })
 
     assert git_config == expected_config
 
 
 def test_create_wsgi_app_uses_scm_app_from_simplevcs(baseapp, request_stub):
     config = {
         'auth_ret_code': '',
         'base_path': '',
         'vcs.scm_app_implementation':
             'rhodecode.tests.lib.middleware.mock_scm_app',
     }
     app = simplegit.SimpleGit(config=config, registry=request_stub.registry)
     wsgi_app = app._create_wsgi_app('/tmp/test', 'test_repo', {})
     assert wsgi_app is mock_scm_app.mock_git_wsgi

rhodecode/tests/lib/middleware/test_simplevcs.py

@@ -1,475 +1,475 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import base64
 
 import mock
 import pytest
 
 from rhodecode.lib.utils2 import AttributeDict
 from rhodecode.tests.utils import CustomTestApp
 
 from rhodecode.lib.caching_query import FromCache
 from rhodecode.lib.hooks_daemon import DummyHooksCallbackDaemon
 from rhodecode.lib.middleware import simplevcs
 from rhodecode.lib.middleware.https_fixup import HttpsFixup
 from rhodecode.lib.middleware.utils import scm_app_http
 from rhodecode.model.db import User, _hash_key
 from rhodecode.model.meta import Session
 from rhodecode.tests import (
     HG_REPO, TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS)
 from rhodecode.tests.lib.middleware import mock_scm_app
 
 
 class StubVCSController(simplevcs.SimpleVCS):
 
     SCM = 'hg'
     stub_response_body = tuple()
 
     def __init__(self, *args, **kwargs):
         super(StubVCSController, self).__init__(*args, **kwargs)
         self._action = 'pull'
         self._is_shadow_repo_dir = True
         self._name = HG_REPO
         self.set_repo_names(None)
 
     @property
     def is_shadow_repo_dir(self):
         return self._is_shadow_repo_dir
 
     def _get_repository_name(self, environ):
         return self._name
 
     def _get_action(self, environ):
         return self._action
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
         def fake_app(environ, start_response):
             headers = [
                 ('Http-Accept', 'application/mercurial')
             ]
             start_response('200 OK', headers)
             return self.stub_response_body
         return fake_app
 
-    def _create_config(self, extras, repo_name):
+    def _create_config(self, extras, repo_name, scheme='http'):
         return None
 
 
 @pytest.fixture
 def vcscontroller(baseapp, config_stub, request_stub):
     config_stub.testing_securitypolicy()
     config_stub.include('rhodecode.authentication')
     config_stub.include('rhodecode.authentication.plugins.auth_rhodecode')
     config_stub.include('rhodecode.authentication.plugins.auth_token')
 
     controller = StubVCSController(
         baseapp.config.get_settings(), request_stub.registry)
     app = HttpsFixup(controller, baseapp.config.get_settings())
     app = CustomTestApp(app)
 
     _remove_default_user_from_query_cache()
 
     # Sanity checks that things are set up correctly
     app.get('/' + HG_REPO, status=200)
 
     app.controller = controller
     return app
 
 
 def _remove_default_user_from_query_cache():
     user = User.get_default_user(cache=True)
     query = Session().query(User).filter(User.username == user.username)
     query = query.options(
         FromCache("sql_cache_short", "get_user_%s" % _hash_key(user.username)))
     query.invalidate()
     Session().expire(user)
 
 
 def test_handles_exceptions_during_permissions_checks(
         vcscontroller, disable_anonymous_user):
     user_and_pass = '%s:%s' % (TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS)
     auth_password = base64.encodestring(user_and_pass).strip()
     extra_environ = {
         'AUTH_TYPE': 'Basic',
         'HTTP_AUTHORIZATION': 'Basic %s' % auth_password,
         'REMOTE_USER': TEST_USER_ADMIN_LOGIN,
     }
 
     # Verify that things are hooked up correctly
     vcscontroller.get('/', status=200, extra_environ=extra_environ)
 
     # Simulate trouble during permission checks
     with mock.patch('rhodecode.model.db.User.get_by_username',
                     side_effect=Exception) as get_user:
         # Verify that a correct 500 is returned and check that the expected
         # code path was hit.
         vcscontroller.get('/', status=500, extra_environ=extra_environ)
         assert get_user.called
 
 
 def test_returns_forbidden_if_no_anonymous_access(
         vcscontroller, disable_anonymous_user):
     vcscontroller.get('/', status=401)
 
 
 class StubFailVCSController(simplevcs.SimpleVCS):
     def _handle_request(self, environ, start_response):
         raise Exception("BOOM")
 
 
 @pytest.fixture(scope='module')
 def fail_controller(baseapp):
     controller = StubFailVCSController(
         baseapp.config.get_settings(), baseapp.config)
     controller = HttpsFixup(controller, baseapp.config.get_settings())
     controller = CustomTestApp(controller)
     return controller
 
 
 def test_handles_exceptions_as_internal_server_error(fail_controller):
     fail_controller.get('/', status=500)
 
 
 def test_provides_traceback_for_appenlight(fail_controller):
     response = fail_controller.get(
         '/', status=500, extra_environ={'appenlight.client': 'fake'})
     assert 'appenlight.__traceback' in response.request.environ
 
 
 def test_provides_utils_scm_app_as_scm_app_by_default(baseapp, request_stub):
     controller = StubVCSController(baseapp.config.get_settings(), request_stub.registry)
     assert controller.scm_app is scm_app_http
 
 
 def test_allows_to_override_scm_app_via_config(baseapp, request_stub):
     config = baseapp.config.get_settings().copy()
     config['vcs.scm_app_implementation'] = (
         'rhodecode.tests.lib.middleware.mock_scm_app')
     controller = StubVCSController(config, request_stub.registry)
     assert controller.scm_app is mock_scm_app
 
 
 @pytest.mark.parametrize('query_string, expected', [
     ('cmd=stub_command', True),
     ('cmd=listkeys', False),
 ])
 def test_should_check_locking(query_string, expected):
     result = simplevcs._should_check_locking(query_string)
     assert result == expected
 
 
 class TestShadowRepoRegularExpression(object):
     pr_segment = 'pull-request'
     shadow_segment = 'repository'
 
     @pytest.mark.parametrize('url, expected', [
         # repo with/without groups
         ('My-Repo/{pr_segment}/1/{shadow_segment}', True),
         ('Group/My-Repo/{pr_segment}/2/{shadow_segment}', True),
         ('Group/Sub-Group/My-Repo/{pr_segment}/3/{shadow_segment}', True),
         ('Group/Sub-Group1/Sub-Group2/My-Repo/{pr_segment}/3/{shadow_segment}', True),
 
         # pull request ID
         ('MyRepo/{pr_segment}/1/{shadow_segment}', True),
         ('MyRepo/{pr_segment}/1234567890/{shadow_segment}', True),
         ('MyRepo/{pr_segment}/-1/{shadow_segment}', False),
         ('MyRepo/{pr_segment}/invalid/{shadow_segment}', False),
 
         # unicode
         (u'Sp€çîál-Repö/{pr_segment}/1/{shadow_segment}', True),
         (u'Sp€çîál-Gröüp/Sp€çîál-Repö/{pr_segment}/1/{shadow_segment}', True),
 
         # trailing/leading slash
         ('/My-Repo/{pr_segment}/1/{shadow_segment}', False),
         ('My-Repo/{pr_segment}/1/{shadow_segment}/', False),
         ('/My-Repo/{pr_segment}/1/{shadow_segment}/', False),
 
         # misc
         ('My-Repo/{pr_segment}/1/{shadow_segment}/extra', False),
         ('My-Repo/{pr_segment}/1/{shadow_segment}extra', False),
     ])
     def test_shadow_repo_regular_expression(self, url, expected):
         from rhodecode.lib.middleware.simplevcs import SimpleVCS
         url = url.format(
             pr_segment=self.pr_segment,
             shadow_segment=self.shadow_segment)
         match_obj = SimpleVCS.shadow_repo_re.match(url)
         assert (match_obj is not None) == expected
 
 
 @pytest.mark.backends('git', 'hg')
 class TestShadowRepoExposure(object):
 
     def test_pull_on_shadow_repo_propagates_to_wsgi_app(
             self, baseapp, request_stub):
         """
         Check that a pull action to a shadow repo is propagated to the
         underlying wsgi app.
         """
         controller = StubVCSController(
             baseapp.config.get_settings(), request_stub.registry)
         controller._check_ssl = mock.Mock()
         controller.is_shadow_repo = True
         controller._action = 'pull'
         controller._is_shadow_repo_dir = True
         controller.stub_response_body = 'dummy body value'
         controller._get_default_cache_ttl = mock.Mock(
             return_value=(False, 0))
 
         environ_stub = {
             'HTTP_HOST': 'test.example.com',
             'HTTP_ACCEPT': 'application/mercurial',
             'REQUEST_METHOD': 'GET',
             'wsgi.url_scheme': 'http',
         }
 
         response = controller(environ_stub, mock.Mock())
         response_body = ''.join(response)
 
         # Assert that we got the response from the wsgi app.
         assert response_body == controller.stub_response_body
 
     def test_pull_on_shadow_repo_that_is_missing(self, baseapp, request_stub):
         """
         Check that a pull action to a shadow repo is propagated to the
         underlying wsgi app.
         """
         controller = StubVCSController(
             baseapp.config.get_settings(), request_stub.registry)
         controller._check_ssl = mock.Mock()
         controller.is_shadow_repo = True
         controller._action = 'pull'
         controller._is_shadow_repo_dir = False
         controller.stub_response_body = 'dummy body value'
         environ_stub = {
             'HTTP_HOST': 'test.example.com',
             'HTTP_ACCEPT': 'application/mercurial',
             'REQUEST_METHOD': 'GET',
             'wsgi.url_scheme': 'http',
         }
 
         response = controller(environ_stub, mock.Mock())
         response_body = ''.join(response)
 
         # Assert that we got the response from the wsgi app.
         assert '404 Not Found' in response_body
 
     def test_push_on_shadow_repo_raises(self, baseapp, request_stub):
         """
         Check that a push action to a shadow repo is aborted.
         """
         controller = StubVCSController(
             baseapp.config.get_settings(), request_stub.registry)
         controller._check_ssl = mock.Mock()
         controller.is_shadow_repo = True
         controller._action = 'push'
         controller.stub_response_body = 'dummy body value'
         environ_stub = {
             'HTTP_HOST': 'test.example.com',
             'HTTP_ACCEPT': 'application/mercurial',
             'REQUEST_METHOD': 'GET',
             'wsgi.url_scheme': 'http',
         }
 
         response = controller(environ_stub, mock.Mock())
         response_body = ''.join(response)
 
         assert response_body != controller.stub_response_body
         # Assert that a 406 error is returned.
         assert '406 Not Acceptable' in response_body
 
     def test_set_repo_names_no_shadow(self, baseapp, request_stub):
         """
         Check that the set_repo_names method sets all names to the one returned
         by the _get_repository_name method on a request to a non shadow repo.
         """
         environ_stub = {}
         controller = StubVCSController(
             baseapp.config.get_settings(), request_stub.registry)
         controller._name = 'RepoGroup/MyRepo'
         controller.set_repo_names(environ_stub)
         assert not controller.is_shadow_repo
         assert (controller.url_repo_name ==
                 controller.acl_repo_name ==
                 controller.vcs_repo_name ==
                 controller._get_repository_name(environ_stub))
 
     def test_set_repo_names_with_shadow(
             self, baseapp, pr_util, config_stub, request_stub):
         """
         Check that the set_repo_names method sets correct names on a request
         to a shadow repo.
         """
         from rhodecode.model.pull_request import PullRequestModel
 
         pull_request = pr_util.create_pull_request()
         shadow_url = '{target}/{pr_segment}/{pr_id}/{shadow_segment}'.format(
             target=pull_request.target_repo.repo_name,
             pr_id=pull_request.pull_request_id,
             pr_segment=TestShadowRepoRegularExpression.pr_segment,
             shadow_segment=TestShadowRepoRegularExpression.shadow_segment)
         controller = StubVCSController(
             baseapp.config.get_settings(), request_stub.registry)
         controller._name = shadow_url
         controller.set_repo_names({})
 
         # Get file system path to shadow repo for assertions.
         workspace_id = PullRequestModel()._workspace_id(pull_request)
         target_vcs = pull_request.target_repo.scm_instance()
         vcs_repo_name = target_vcs._get_shadow_repository_path(
             pull_request.target_repo.repo_id, workspace_id)
 
         assert controller.vcs_repo_name == vcs_repo_name
         assert controller.url_repo_name == shadow_url
         assert controller.acl_repo_name == pull_request.target_repo.repo_name
         assert controller.is_shadow_repo
 
     def test_set_repo_names_with_shadow_but_missing_pr(
             self, baseapp, pr_util, config_stub, request_stub):
         """
         Checks that the set_repo_names method enforces matching target repos
         and pull request IDs.
         """
         pull_request = pr_util.create_pull_request()
         shadow_url = '{target}/{pr_segment}/{pr_id}/{shadow_segment}'.format(
             target=pull_request.target_repo.repo_name,
             pr_id=999999999,
             pr_segment=TestShadowRepoRegularExpression.pr_segment,
             shadow_segment=TestShadowRepoRegularExpression.shadow_segment)
         controller = StubVCSController(
             baseapp.config.get_settings(), request_stub.registry)
         controller._name = shadow_url
         controller.set_repo_names({})
 
         assert not controller.is_shadow_repo
         assert (controller.url_repo_name ==
                 controller.acl_repo_name ==
                 controller.vcs_repo_name)
 
 
 @pytest.mark.usefixtures('baseapp')
 class TestGenerateVcsResponse(object):
 
     def test_ensures_that_start_response_is_called_early_enough(self):
         self.call_controller_with_response_body(iter(['a', 'b']))
         assert self.start_response.called
 
     def test_invalidates_cache_after_body_is_consumed(self):
         result = self.call_controller_with_response_body(iter(['a', 'b']))
         assert not self.was_cache_invalidated()
         # Consume the result
         list(result)
         assert self.was_cache_invalidated()
 
     def test_raises_unknown_exceptions(self):
         result = self.call_controller_with_response_body(
             self.raise_result_iter(vcs_kind='unknown'))
         with pytest.raises(Exception):
             list(result)
 
     def test_prepare_callback_daemon_is_called(self):
         def side_effect(extras, environ, action, txn_id=None):
             return DummyHooksCallbackDaemon(), extras
 
         prepare_patcher = mock.patch.object(
             StubVCSController, '_prepare_callback_daemon')
         with prepare_patcher as prepare_mock:
             prepare_mock.side_effect = side_effect
             self.call_controller_with_response_body(iter(['a', 'b']))
         assert prepare_mock.called
         assert prepare_mock.call_count == 1
 
     def call_controller_with_response_body(self, response_body):
         settings = {
             'base_path': 'fake_base_path',
             'vcs.hooks.protocol': 'http',
             'vcs.hooks.direct_calls': False,
         }
         registry = AttributeDict()
         controller = StubVCSController(settings, registry)
         controller._invalidate_cache = mock.Mock()
         controller.stub_response_body = response_body
         self.start_response = mock.Mock()
         result = controller._generate_vcs_response(
             environ={}, start_response=self.start_response,
             repo_path='fake_repo_path',
             extras={}, action='push')
         self.controller = controller
         return result
 
     def raise_result_iter(self, vcs_kind='repo_locked'):
         """
         Simulates an exception due to a vcs raised exception if kind vcs_kind
         """
         raise self.vcs_exception(vcs_kind=vcs_kind)
         yield "never_reached"
 
     def vcs_exception(self, vcs_kind='repo_locked'):
         locked_exception = Exception('TEST_MESSAGE')
         locked_exception._vcs_kind = vcs_kind
         return locked_exception
 
     def was_cache_invalidated(self):
         return self.controller._invalidate_cache.called
 
 
 class TestInitializeGenerator(object):
 
     def test_drains_first_element(self):
         gen = self.factory(['__init__', 1, 2])
         result = list(gen)
         assert result == [1, 2]
 
     @pytest.mark.parametrize('values', [
         [],
         [1, 2],
     ])
     def test_raises_value_error(self, values):
         with pytest.raises(ValueError):
             self.factory(values)
 
     @simplevcs.initialize_generator
     def factory(self, iterable):
         for elem in iterable:
             yield elem
 
 
 class TestPrepareHooksDaemon(object):
     def test_calls_imported_prepare_callback_daemon(self, app_settings, request_stub):
         expected_extras = {'extra1': 'value1'}
         daemon = DummyHooksCallbackDaemon()
 
         controller = StubVCSController(app_settings, request_stub.registry)
         prepare_patcher = mock.patch.object(
             simplevcs, 'prepare_callback_daemon',
             return_value=(daemon, expected_extras))
         with prepare_patcher as prepare_mock:
             callback_daemon, extras = controller._prepare_callback_daemon(
                 expected_extras.copy(), {}, 'push')
         prepare_mock.assert_called_once_with(
             expected_extras,
             protocol=app_settings['vcs.hooks.protocol'],
             host=app_settings['vcs.hooks.host'],
             txn_id=None,
             use_direct_calls=app_settings['vcs.hooks.direct_calls'])
 
         assert callback_daemon == daemon
         assert extras == extras