chore(refactor): renamed rc_json to ext_json for ce compat

vcsserver/git_lfs/app.py

@@ -1,296 +1,296 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import re
 import logging
 
 from pyramid.config import Configurator
 from pyramid.response import Response, FileIter
 from pyramid.httpexceptions import (
     HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden,
     HTTPUnprocessableEntity)
 
-from vcsserver.lib.rc_json import json
+from vcsserver.lib.ext_json import json
 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
 from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator
 from vcsserver.str_utils import safe_int
 
 log = logging.getLogger(__name__)
 
 
 GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs'  # +json ?
 GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))')
 
 
 def write_response_error(http_exception, text=None):
     content_type = GIT_LFS_CONTENT_TYPE + '+json'
     _exception = http_exception(content_type=content_type)
     _exception.content_type = content_type
     if text:
         _exception.body = json.dumps({'message': text})
     log.debug('LFS: writing response of type %s to client with text:%s',
               http_exception, text)
     return _exception
 
 
 class AuthHeaderRequired:
     """
     Decorator to check if request has proper auth-header
     """
 
     def __call__(self, func):
         return get_cython_compat_decorator(self.__wrapper, func)
 
     def __wrapper(self, func, *fargs, **fkwargs):
         request = fargs[1]
         auth = request.authorization
         if not auth:
             return write_response_error(HTTPForbidden)
         return func(*fargs[1:], **fkwargs)
 
 
 # views
 
 def lfs_objects(request):
     # indicate not supported, V1 API
     log.warning('LFS: v1 api not supported, reporting it back to client')
     return write_response_error(HTTPNotImplemented, 'LFS: v1 api not supported')
 
 
 @AuthHeaderRequired()
 def lfs_objects_batch(request):
     """
     The client sends the following information to the Batch endpoint to transfer some objects:
 
         operation - Should be download or upload.
         transfers - An optional Array of String identifiers for transfer
             adapters that the client has configured. If omitted, the basic
             transfer adapter MUST be assumed by the server.
         objects - An Array of objects to download.
         oid - String OID of the LFS object.
         size - Integer byte size of the LFS object. Must be at least zero.
     """
     request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
     auth = request.authorization
     repo = request.matchdict.get('repo')
     data = request.json
     operation = data.get('operation')
     http_scheme = request.registry.git_lfs_http_scheme
 
     if operation not in ('download', 'upload'):
         log.debug('LFS: unsupported operation:%s', operation)
         return write_response_error(
             HTTPBadRequest, f'unsupported operation mode: `{operation}`')
 
     if 'objects' not in data:
         log.debug('LFS: missing objects data')
         return write_response_error(
             HTTPBadRequest, 'missing objects data')
 
     log.debug('LFS: handling operation of type: %s', operation)
 
     objects = []
     for o in data['objects']:
         try:
             oid = o['oid']
             obj_size = o['size']
         except KeyError:
             log.exception('LFS, failed to extract data')
             return write_response_error(
                 HTTPBadRequest, 'unsupported data in objects')
 
         obj_data = {'oid': oid}
         if http_scheme == 'http':
             # Note(marcink): when using http, we might have a custom port
             # so we skip setting it to http, url dispatch then wont generate a port in URL
             # for development we need this
             http_scheme = None
 
         obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid,
                                      _scheme=http_scheme)
         obj_verify_href = request.route_url('lfs_objects_verify', repo=repo,
                                             _scheme=http_scheme)
         store = LFSOidStore(
             oid, repo, store_location=request.registry.git_lfs_store_path)
         handler = OidHandler(
             store, repo, auth, oid, obj_size, obj_data,
             obj_href, obj_verify_href)
 
         # this verifies also OIDs
         actions, errors = handler.exec_operation(operation)
         if errors:
             log.warning('LFS: got following errors: %s', errors)
             obj_data['errors'] = errors
 
         if actions:
             obj_data['actions'] = actions
 
         obj_data['size'] = obj_size
         obj_data['authenticated'] = True
         objects.append(obj_data)
 
     result = {'objects': objects, 'transfer': 'basic'}
     log.debug('LFS Response %s', safe_result(result))
 
     return result
 
 
 def lfs_objects_oid_upload(request):
     request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
     repo = request.matchdict.get('repo')
     oid = request.matchdict.get('oid')
     store = LFSOidStore(
         oid, repo, store_location=request.registry.git_lfs_store_path)
     engine = store.get_engine(mode='wb')
     log.debug('LFS: starting chunked write of LFS oid: %s to storage', oid)
 
     body = request.environ['wsgi.input']
 
     with engine as f:
         blksize = 64 * 1024  # 64kb
         while True:
             # read in chunks as stream comes in from Gunicorn
             # this is a specific Gunicorn support function.
             # might work differently on waitress
             chunk = body.read(blksize)
             if not chunk:
                 break
             f.write(chunk)
 
     return {'upload': 'ok'}
 
 
 def lfs_objects_oid_download(request):
     repo = request.matchdict.get('repo')
     oid = request.matchdict.get('oid')
 
     store = LFSOidStore(
         oid, repo, store_location=request.registry.git_lfs_store_path)
     if not store.has_oid():
         log.debug('LFS: oid %s does not exists in store', oid)
         return write_response_error(
             HTTPNotFound, f'requested file with oid `{oid}` not found in store')
 
     # TODO(marcink): support range header ?
     # Range: bytes=0-, `bytes=(\d+)\-.*`
 
     f = open(store.oid_path, 'rb')
     response = Response(
         content_type='application/octet-stream', app_iter=FileIter(f))
     response.headers.add('X-RC-LFS-Response-Oid', str(oid))
     return response
 
 
 def lfs_objects_verify(request):
     request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
     repo = request.matchdict.get('repo')
 
     data = request.json
     oid = data.get('oid')
     size = safe_int(data.get('size'))
 
     if not (oid and size):
         return write_response_error(
             HTTPBadRequest, 'missing oid and size in request data')
 
     store = LFSOidStore(
         oid, repo, store_location=request.registry.git_lfs_store_path)
     if not store.has_oid():
         log.debug('LFS: oid %s does not exists in store', oid)
         return write_response_error(
             HTTPNotFound, f'oid `{oid}` does not exists in store')
 
     store_size = store.size_oid()
     if store_size != size:
         msg = 'requested file size mismatch store size:{} requested:{}'.format(
             store_size, size)
         return write_response_error(
             HTTPUnprocessableEntity, msg)
 
     return {'message': {'size': 'ok', 'in_store': 'ok'}}
 
 
 def lfs_objects_lock(request):
     return write_response_error(
         HTTPNotImplemented, 'GIT LFS locking api not supported')
 
 
 def not_found(request):
     return write_response_error(
         HTTPNotFound, 'request path not found')
 
 
 def lfs_disabled(request):
     return write_response_error(
         HTTPNotImplemented, 'GIT LFS disabled for this repo')
 
 
 def git_lfs_app(config):
 
     # v1 API deprecation endpoint
     config.add_route('lfs_objects',
                      '/{repo:.*?[^/]}/info/lfs/objects')
     config.add_view(lfs_objects, route_name='lfs_objects',
                     request_method='POST', renderer='json')
 
     # locking API
     config.add_route('lfs_objects_lock',
                      '/{repo:.*?[^/]}/info/lfs/locks')
     config.add_view(lfs_objects_lock, route_name='lfs_objects_lock',
                     request_method=('POST', 'GET'), renderer='json')
 
     config.add_route('lfs_objects_lock_verify',
                      '/{repo:.*?[^/]}/info/lfs/locks/verify')
     config.add_view(lfs_objects_lock, route_name='lfs_objects_lock_verify',
                     request_method=('POST', 'GET'), renderer='json')
 
     # batch API
     config.add_route('lfs_objects_batch',
                      '/{repo:.*?[^/]}/info/lfs/objects/batch')
     config.add_view(lfs_objects_batch, route_name='lfs_objects_batch',
                     request_method='POST', renderer='json')
 
     # oid upload/download API
     config.add_route('lfs_objects_oid',
                      '/{repo:.*?[^/]}/info/lfs/objects/{oid}')
     config.add_view(lfs_objects_oid_upload, route_name='lfs_objects_oid',
                     request_method='PUT', renderer='json')
     config.add_view(lfs_objects_oid_download, route_name='lfs_objects_oid',
                     request_method='GET', renderer='json')
 
     # verification API
     config.add_route('lfs_objects_verify',
                      '/{repo:.*?[^/]}/info/lfs/verify')
     config.add_view(lfs_objects_verify, route_name='lfs_objects_verify',
                     request_method='POST', renderer='json')
 
     # not found handler for API
     config.add_notfound_view(not_found, renderer='json')
 
 
 def create_app(git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme):
     config = Configurator()
     if git_lfs_enabled:
         config.include(git_lfs_app)
         config.registry.git_lfs_store_path = git_lfs_store_path
         config.registry.git_lfs_http_scheme = git_lfs_http_scheme
     else:
         # not found handler for API, reporting disabled LFS support
         config.add_notfound_view(lfs_disabled, renderer='json')
 
     app = config.make_wsgi_app()
     return app

vcsserver/git_lfs/tests/test_lfs_app.py

@@ -1,274 +1,274 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import os
 import pytest
 from webtest.app import TestApp as WebObTestApp
 
-from vcsserver.lib.rc_json import json
+from vcsserver.lib.ext_json import json
 from vcsserver.str_utils import safe_bytes
 from vcsserver.git_lfs.app import create_app
 from vcsserver.git_lfs.lib import LFSOidStore
 
 
 @pytest.fixture(scope='function')
 def git_lfs_app(tmpdir):
     custom_app = WebObTestApp(create_app(
         git_lfs_enabled=True, git_lfs_store_path=str(tmpdir),
         git_lfs_http_scheme='http'))
     custom_app._store = str(tmpdir)
     return custom_app
 
 
 @pytest.fixture(scope='function')
 def git_lfs_https_app(tmpdir):
     custom_app = WebObTestApp(create_app(
         git_lfs_enabled=True, git_lfs_store_path=str(tmpdir),
         git_lfs_http_scheme='https'))
     custom_app._store = str(tmpdir)
     return custom_app
 
 
 @pytest.fixture()
 def http_auth():
     return {'HTTP_AUTHORIZATION': "Basic XXXXX"}
 
 
 class TestLFSApplication:
 
     def test_app_wrong_path(self, git_lfs_app):
         git_lfs_app.get('/repo/info/lfs/xxx', status=404)
 
     def test_app_deprecated_endpoint(self, git_lfs_app):
         response = git_lfs_app.post('/repo/info/lfs/objects', status=501)
         assert response.status_code == 501
         assert json.loads(response.text) == {'message': 'LFS: v1 api not supported'}
 
     def test_app_lock_verify_api_not_available(self, git_lfs_app):
         response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501)
         assert response.status_code == 501
         assert json.loads(response.text) == {
             'message': 'GIT LFS locking api not supported'}
 
     def test_app_lock_api_not_available(self, git_lfs_app):
         response = git_lfs_app.post('/repo/info/lfs/locks', status=501)
         assert response.status_code == 501
         assert json.loads(response.text) == {
             'message': 'GIT LFS locking api not supported'}
 
     def test_app_batch_api_missing_auth(self, git_lfs_app):
         git_lfs_app.post_json(
             '/repo/info/lfs/objects/batch', params={}, status=403)
 
     def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth):
         response = git_lfs_app.post_json(
             '/repo/info/lfs/objects/batch', params={}, status=400,
             extra_environ=http_auth)
         assert json.loads(response.text) == {
             'message': 'unsupported operation mode: `None`'}
 
     def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth):
         response = git_lfs_app.post_json(
             '/repo/info/lfs/objects/batch', params={'operation': 'download'},
             status=400, extra_environ=http_auth)
         assert json.loads(response.text) == {
             'message': 'missing objects data'}
 
     def test_app_batch_api_unsupported_data_in_objects(
             self, git_lfs_app, http_auth):
         params = {'operation': 'download',
                   'objects': [{}]}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/objects/batch', params=params, status=400,
             extra_environ=http_auth)
         assert json.loads(response.text) == {
             'message': 'unsupported data in objects'}
 
     def test_app_batch_api_download_missing_object(
             self, git_lfs_app, http_auth):
         params = {'operation': 'download',
                   'objects': [{'oid': '123', 'size': '1024'}]}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/objects/batch', params=params,
             extra_environ=http_auth)
 
         expected_objects = [
             {'authenticated': True,
              'errors': {'error': {
                  'code': 404,
                  'message': 'object: 123 does not exist in store'}},
              'oid': '123',
              'size': '1024'}
         ]
         assert json.loads(response.text) == {
             'objects': expected_objects, 'transfer': 'basic'}
 
     def test_app_batch_api_download(self, git_lfs_app, http_auth):
         oid = '456'
         oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
         if not os.path.isdir(os.path.dirname(oid_path)):
             os.makedirs(os.path.dirname(oid_path))
         with open(oid_path, 'wb') as f:
             f.write(safe_bytes('OID_CONTENT'))
 
         params = {'operation': 'download',
                   'objects': [{'oid': oid, 'size': '1024'}]}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/objects/batch', params=params,
             extra_environ=http_auth)
 
         expected_objects = [
             {'authenticated': True,
              'actions': {
                  'download': {
                      'header': {'Authorization': 'Basic XXXXX'},
                      'href': 'http://localhost/repo/info/lfs/objects/456'},
              },
              'oid': '456',
              'size': '1024'}
         ]
         assert json.loads(response.text) == {
             'objects': expected_objects, 'transfer': 'basic'}
 
     def test_app_batch_api_upload(self, git_lfs_app, http_auth):
         params = {'operation': 'upload',
                   'objects': [{'oid': '123', 'size': '1024'}]}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/objects/batch', params=params,
             extra_environ=http_auth)
         expected_objects = [
             {'authenticated': True,
              'actions': {
                  'upload': {
                      'header': {'Authorization': 'Basic XXXXX',
                                  'Transfer-Encoding': 'chunked'},
                      'href': 'http://localhost/repo/info/lfs/objects/123'},
                  'verify': {
                      'header': {'Authorization': 'Basic XXXXX'},
                      'href': 'http://localhost/repo/info/lfs/verify'}
              },
              'oid': '123',
              'size': '1024'}
         ]
         assert json.loads(response.text) == {
             'objects': expected_objects, 'transfer': 'basic'}
 
     def test_app_batch_api_upload_for_https(self, git_lfs_https_app, http_auth):
         params = {'operation': 'upload',
                   'objects': [{'oid': '123', 'size': '1024'}]}
         response = git_lfs_https_app.post_json(
             '/repo/info/lfs/objects/batch', params=params,
             extra_environ=http_auth)
         expected_objects = [
             {'authenticated': True,
              'actions': {
                  'upload': {
                      'header': {'Authorization': 'Basic XXXXX',
                                  'Transfer-Encoding': 'chunked'},
                      'href': 'https://localhost/repo/info/lfs/objects/123'},
                  'verify': {
                      'header': {'Authorization': 'Basic XXXXX'},
                      'href': 'https://localhost/repo/info/lfs/verify'}
              },
              'oid': '123',
              'size': '1024'}
         ]
         assert json.loads(response.text) == {
             'objects': expected_objects, 'transfer': 'basic'}
 
     def test_app_verify_api_missing_data(self, git_lfs_app):
         params = {'oid': 'missing'}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/verify', params=params,
             status=400)
 
         assert json.loads(response.text) == {
             'message': 'missing oid and size in request data'}
 
     def test_app_verify_api_missing_obj(self, git_lfs_app):
         params = {'oid': 'missing', 'size': '1024'}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/verify', params=params,
             status=404)
 
         assert json.loads(response.text) == {
             'message': 'oid `missing` does not exists in store'}
 
     def test_app_verify_api_size_mismatch(self, git_lfs_app):
         oid = 'existing'
         oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
         if not os.path.isdir(os.path.dirname(oid_path)):
             os.makedirs(os.path.dirname(oid_path))
         with open(oid_path, 'wb') as f:
             f.write(safe_bytes('OID_CONTENT'))
 
         params = {'oid': oid, 'size': '1024'}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/verify', params=params, status=422)
 
         assert json.loads(response.text) == {
             'message': 'requested file size mismatch '
                         'store size:11 requested:1024'}
 
     def test_app_verify_api(self, git_lfs_app):
         oid = 'existing'
         oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
         if not os.path.isdir(os.path.dirname(oid_path)):
             os.makedirs(os.path.dirname(oid_path))
         with open(oid_path, 'wb') as f:
             f.write(safe_bytes('OID_CONTENT'))
 
         params = {'oid': oid, 'size': 11}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/verify', params=params)
 
         assert json.loads(response.text) == {
             'message': {'size': 'ok', 'in_store': 'ok'}}
 
     def test_app_download_api_oid_not_existing(self, git_lfs_app):
         oid = 'missing'
 
         response = git_lfs_app.get(
             '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404)
 
         assert json.loads(response.text) == {
             'message': 'requested file with oid `missing` not found in store'}
 
     def test_app_download_api(self, git_lfs_app):
         oid = 'existing'
         oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
         if not os.path.isdir(os.path.dirname(oid_path)):
             os.makedirs(os.path.dirname(oid_path))
         with open(oid_path, 'wb') as f:
             f.write(safe_bytes('OID_CONTENT'))
 
         response = git_lfs_app.get(
             '/repo/info/lfs/objects/{oid}'.format(oid=oid))
         assert response
 
     def test_app_upload(self, git_lfs_app):
         oid = 'uploaded'
 
         response = git_lfs_app.put(
             '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT')
 
         assert json.loads(response.text) == {'upload': 'ok'}
 
         # verify that we actually wrote that OID
         oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
         assert os.path.isfile(oid_path)
         assert 'CONTENT' == open(oid_path).read()

vcsserver/hooks.py

@@ -1,832 +1,832 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import io
 import os
 import sys
 import logging
 import collections
 import base64
 import msgpack
 import dataclasses
 import pygit2
 
 import http.client
 from celery import Celery
 
 import mercurial.scmutil
 import mercurial.node
 
-from vcsserver.lib.rc_json import json
+from vcsserver.lib.ext_json import json
 from vcsserver import exceptions, subprocessio, settings
 from vcsserver.str_utils import ascii_str, safe_str
 from vcsserver.remote.git_remote import Repository
 
 celery_app = Celery('__vcsserver__')
 log = logging.getLogger(__name__)
 
 
 class HooksHttpClient:
     proto = 'msgpack.v1'
     connection = None
 
     def __init__(self, hooks_uri):
         self.hooks_uri = hooks_uri
 
     def __repr__(self):
         return f'{self.__class__}(hook_uri={self.hooks_uri}, proto={self.proto})'
 
     def __call__(self, method, extras):
         connection = http.client.HTTPConnection(self.hooks_uri)
         # binary msgpack body
         headers, body = self._serialize(method, extras)
         log.debug('Doing a new hooks call using HTTPConnection to %s', self.hooks_uri)
 
         try:
             try:
                 connection.request('POST', '/', body, headers)
             except Exception as error:
                 log.error('Hooks calling Connection failed on %s, org error: %s', connection.__dict__, error)
                 raise
 
             response = connection.getresponse()
             try:
                 return msgpack.load(response)
             except Exception:
                 response_data = response.read()
                 log.exception('Failed to decode hook response json data. '
                               'response_code:%s, raw_data:%s',
                               response.status, response_data)
                 raise
         finally:
             connection.close()
 
     @classmethod
     def _serialize(cls, hook_name, extras):
         data = {
             'method': hook_name,
             'extras': extras
         }
         headers = {
             "rc-hooks-protocol": cls.proto,
             "Connection": "keep-alive"
         }
         return headers, msgpack.packb(data)
 
 
 class HooksCeleryClient:
     TASK_TIMEOUT = 60  # time in seconds
 
     def __init__(self, queue, backend):
         celery_app.config_from_object({
             'broker_url': queue, 'result_backend': backend,
             'broker_connection_retry_on_startup': True,
             'task_serializer': 'json',
             'accept_content': ['json', 'msgpack'],
             'result_serializer': 'json',
             'result_accept_content': ['json', 'msgpack']
         })
         self.celery_app = celery_app
 
     def __call__(self, method, extras):
         inquired_task = self.celery_app.signature(
             f'rhodecode.lib.celerylib.tasks.{method}'
         )
         return inquired_task.delay(extras).get(timeout=self.TASK_TIMEOUT)
 
 
 class HooksShadowRepoClient:
 
     def __call__(self, hook_name, extras):
         return {'output': '', 'status': 0}
 
 
 class RemoteMessageWriter:
     """Writer base class."""
     def write(self, message):
         raise NotImplementedError()
 
 
 class HgMessageWriter(RemoteMessageWriter):
     """Writer that knows how to send messages to mercurial clients."""
 
     def __init__(self, ui):
         self.ui = ui
 
     def write(self, message: str):
         # TODO: Check why the quiet flag is set by default.
         old = self.ui.quiet
         self.ui.quiet = False
         self.ui.status(message.encode('utf-8'))
         self.ui.quiet = old
 
 
 class GitMessageWriter(RemoteMessageWriter):
     """Writer that knows how to send messages to git clients."""
 
     def __init__(self, stdout=None):
         self.stdout = stdout or sys.stdout
 
     def write(self, message: str):
         self.stdout.write(message)
 
 
 class SvnMessageWriter(RemoteMessageWriter):
     """Writer that knows how to send messages to svn clients."""
 
     def __init__(self, stderr=None):
         # SVN needs data sent to stderr for back-to-client messaging
         self.stderr = stderr or sys.stderr
 
     def write(self, message):
         self.stderr.write(message)
 
 
 def _handle_exception(result):
     exception_class = result.get('exception')
     exception_traceback = result.get('exception_traceback')
     log.debug('Handling hook-call exception: %s', exception_class)
 
     if exception_traceback:
         log.error('Got traceback from remote call:%s', exception_traceback)
 
     if exception_class == 'HTTPLockedRC':
         raise exceptions.RepositoryLockedException()(*result['exception_args'])
     elif exception_class == 'HTTPBranchProtected':
         raise exceptions.RepositoryBranchProtectedException()(*result['exception_args'])
     elif exception_class == 'RepositoryError':
         raise exceptions.VcsException()(*result['exception_args'])
     elif exception_class:
         raise Exception(
             f"""Got remote exception "{exception_class}" with args "{result['exception_args']}" """
         )
 
 
 def _get_hooks_client(extras):
     hooks_uri = extras.get('hooks_uri')
     task_queue = extras.get('task_queue')
     task_backend = extras.get('task_backend')
     is_shadow_repo = extras.get('is_shadow_repo')
 
     if hooks_uri:
         return HooksHttpClient(hooks_uri)
     elif task_queue and task_backend:
         return HooksCeleryClient(task_queue, task_backend)
     elif is_shadow_repo:
         return HooksShadowRepoClient()
     else:
         raise Exception("Hooks client not found!")
 
 
 def _call_hook(hook_name, extras, writer):
     hooks_client = _get_hooks_client(extras)
     log.debug('Hooks, using client:%s', hooks_client)
     result = hooks_client(hook_name, extras)
     log.debug('Hooks got result: %s', result)
     _handle_exception(result)
     writer.write(result['output'])
 
     return result['status']
 
 
 def _extras_from_ui(ui):
     hook_data = ui.config(b'rhodecode', b'RC_SCM_DATA')
     if not hook_data:
         # maybe it's inside environ ?
         env_hook_data = os.environ.get('RC_SCM_DATA')
         if env_hook_data:
             hook_data = env_hook_data
 
     extras = {}
     if hook_data:
         extras = json.loads(hook_data)
     return extras
 
 
 def _rev_range_hash(repo, node, check_heads=False):
     from vcsserver.hgcompat import get_ctx
 
     commits = []
     revs = []
     start = get_ctx(repo, node).rev()
     end = len(repo)
     for rev in range(start, end):
         revs.append(rev)
         ctx = get_ctx(repo, rev)
         commit_id = ascii_str(mercurial.node.hex(ctx.node()))
         branch = safe_str(ctx.branch())
         commits.append((commit_id, branch))
 
     parent_heads = []
     if check_heads:
         parent_heads = _check_heads(repo, start, end, revs)
     return commits, parent_heads
 
 
 def _check_heads(repo, start, end, commits):
     from vcsserver.hgcompat import get_ctx
     changelog = repo.changelog
     parents = set()
 
     for new_rev in commits:
         for p in changelog.parentrevs(new_rev):
             if p == mercurial.node.nullrev:
                 continue
             if p < start:
                 parents.add(p)
 
     for p in parents:
         branch = get_ctx(repo, p).branch()
         # The heads descending from that parent, on the same branch
         parent_heads = {p}
         reachable = {p}
         for x in range(p + 1, end):
             if get_ctx(repo, x).branch() != branch:
                 continue
             for pp in changelog.parentrevs(x):
                 if pp in reachable:
                     reachable.add(x)
                     parent_heads.discard(pp)
                     parent_heads.add(x)
         # More than one head? Suggest merging
         if len(parent_heads) > 1:
             return list(parent_heads)
 
     return []
 
 
 def _get_git_env():
     env = {}
     for k, v in os.environ.items():
         if k.startswith('GIT'):
             env[k] = v
 
     # serialized version
     return [(k, v) for k, v in env.items()]
 
 
 def _get_hg_env(old_rev, new_rev, txnid, repo_path):
     env = {}
     for k, v in os.environ.items():
         if k.startswith('HG'):
             env[k] = v
 
     env['HG_NODE'] = old_rev
     env['HG_NODE_LAST'] = new_rev
     env['HG_TXNID'] = txnid
     env['HG_PENDING'] = repo_path
 
     return [(k, v) for k, v in env.items()]
 
 
 def _fix_hooks_executables(ini_path=''):
     """
     This is a trick to set proper settings.EXECUTABLE paths for certain execution patterns
     especially for subversion where hooks strip entire env, and calling just 'svn' command will most likely fail
     because svn is not on PATH
     """
     from vcsserver.http_main import sanitize_settings_and_apply_defaults
     from vcsserver.lib.config_utils import get_app_config_lightweight
 
     core_binary_dir = settings.BINARY_DIR or '/usr/local/bin/rhodecode_bin/vcs_bin'
     if ini_path:
 
         ini_settings = get_app_config_lightweight(ini_path)
         ini_settings = sanitize_settings_and_apply_defaults({'__file__': ini_path}, ini_settings)
         core_binary_dir = ini_settings['core.binary_dir']
 
     settings.BINARY_DIR = core_binary_dir
 
 
 def repo_size(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     return _call_hook('repo_size', extras, HgMessageWriter(ui))
 
 
 def pre_pull(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     return _call_hook('pre_pull', extras, HgMessageWriter(ui))
 
 
 def pre_pull_ssh(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     if extras and extras.get('SSH'):
         return pre_pull(ui, repo, **kwargs)
     return 0
 
 
 def post_pull(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     return _call_hook('post_pull', extras, HgMessageWriter(ui))
 
 
 def post_pull_ssh(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     if extras and extras.get('SSH'):
         return post_pull(ui, repo, **kwargs)
     return 0
 
 
 def pre_push(ui, repo, node=None, **kwargs):
     """
     Mercurial pre_push hook
     """
     extras = _extras_from_ui(ui)
     detect_force_push = extras.get('detect_force_push')
 
     rev_data = []
     hook_type: str = safe_str(kwargs.get('hooktype'))
 
     if node and hook_type == 'pretxnchangegroup':
         branches = collections.defaultdict(list)
         commits, _heads = _rev_range_hash(repo, node, check_heads=detect_force_push)
         for commit_id, branch in commits:
             branches[branch].append(commit_id)
 
         for branch, commits in branches.items():
             old_rev = ascii_str(kwargs.get('node_last')) or commits[0]
             rev_data.append({
                 'total_commits': len(commits),
                 'old_rev': old_rev,
                 'new_rev': commits[-1],
                 'ref': '',
                 'type': 'branch',
                 'name': branch,
             })
 
         for push_ref in rev_data:
             push_ref['multiple_heads'] = _heads
 
             repo_path = os.path.join(
                 extras.get('repo_store', ''), extras.get('repository', ''))
             push_ref['hg_env'] = _get_hg_env(
                 old_rev=push_ref['old_rev'],
                 new_rev=push_ref['new_rev'], txnid=ascii_str(kwargs.get('txnid')),
                 repo_path=repo_path)
 
     extras['hook_type'] = hook_type or 'pre_push'
     extras['commit_ids'] = rev_data
 
     return _call_hook('pre_push', extras, HgMessageWriter(ui))
 
 
 def pre_push_ssh(ui, repo, node=None, **kwargs):
     extras = _extras_from_ui(ui)
     if extras.get('SSH'):
         return pre_push(ui, repo, node, **kwargs)
 
     return 0
 
 
 def pre_push_ssh_auth(ui, repo, node=None, **kwargs):
     """
     Mercurial pre_push hook for SSH
     """
     extras = _extras_from_ui(ui)
     if extras.get('SSH'):
         permission = extras['SSH_PERMISSIONS']
 
         if 'repository.write' == permission or 'repository.admin' == permission:
             return 0
 
         # non-zero ret code
         return 1
 
     return 0
 
 
 def post_push(ui, repo, node, **kwargs):
     """
     Mercurial post_push hook
     """
     extras = _extras_from_ui(ui)
 
     commit_ids = []
     branches = []
     bookmarks = []
     tags = []
     hook_type: str = safe_str(kwargs.get('hooktype'))
 
     commits, _heads = _rev_range_hash(repo, node)
     for commit_id, branch in commits:
         commit_ids.append(commit_id)
         if branch not in branches:
             branches.append(branch)
 
     if hasattr(ui, '_rc_pushkey_bookmarks'):
         bookmarks = ui._rc_pushkey_bookmarks
 
     extras['hook_type'] = hook_type or 'post_push'
     extras['commit_ids'] = commit_ids
 
     extras['new_refs'] = {
         'branches': branches,
         'bookmarks': bookmarks,
         'tags': tags
     }
 
     return _call_hook('post_push', extras, HgMessageWriter(ui))
 
 
 def post_push_ssh(ui, repo, node, **kwargs):
     """
     Mercurial post_push hook for SSH
     """
     if _extras_from_ui(ui).get('SSH'):
         return post_push(ui, repo, node, **kwargs)
     return 0
 
 
 def key_push(ui, repo, **kwargs):
     from vcsserver.hgcompat import get_ctx
 
     if kwargs['new'] != b'0' and kwargs['namespace'] == b'bookmarks':
         # store new bookmarks in our UI object propagated later to post_push
         ui._rc_pushkey_bookmarks = get_ctx(repo, kwargs['key']).bookmarks()
     return
 
 
 # backward compat
 log_pull_action = post_pull
 
 # backward compat
 log_push_action = post_push
 
 
 def handle_git_pre_receive(unused_repo_path, unused_revs, unused_env):
     """
     Old hook name: keep here for backward compatibility.
 
     This is only required when the installed git hooks are not upgraded.
     """
     pass
 
 
 def handle_git_post_receive(unused_repo_path, unused_revs, unused_env):
     """
     Old hook name: keep here for backward compatibility.
 
     This is only required when the installed git hooks are not upgraded.
     """
     pass
 
 
 @dataclasses.dataclass
 class HookResponse:
     status: int
     output: str
 
 
 def git_pre_pull(extras) -> HookResponse:
     """
     Pre pull hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
 
     if 'pull' not in extras['hooks']:
         return HookResponse(0, '')
 
     stdout = io.StringIO()
     try:
         status_code = _call_hook('pre_pull', extras, GitMessageWriter(stdout))
 
     except Exception as error:
         log.exception('Failed to call pre_pull hook')
         status_code = 128
         stdout.write(f'ERROR: {error}\n')
 
     return HookResponse(status_code, stdout.getvalue())
 
 
 def git_post_pull(extras) -> HookResponse:
     """
     Post pull hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     if 'pull' not in extras['hooks']:
         return HookResponse(0, '')
 
     stdout = io.StringIO()
     try:
         status = _call_hook('post_pull', extras, GitMessageWriter(stdout))
     except Exception as error:
         status = 128
         stdout.write(f'ERROR: {error}\n')
 
     return HookResponse(status, stdout.getvalue())
 
 
 def _parse_git_ref_lines(revision_lines):
     rev_data = []
     for revision_line in revision_lines or []:
         old_rev, new_rev, ref = revision_line.strip().split(' ')
         ref_data = ref.split('/', 2)
         if ref_data[1] in ('tags', 'heads'):
             rev_data.append({
                 # NOTE(marcink):
                 # we're unable to tell total_commits for git at this point
                 # but we set the variable for consistency with GIT
                 'total_commits': -1,
                 'old_rev': old_rev,
                 'new_rev': new_rev,
                 'ref': ref,
                 'type': ref_data[1],
                 'name': ref_data[2],
             })
     return rev_data
 
 
 def git_pre_receive(unused_repo_path, revision_lines, env) -> int:
     """
     Pre push hook.
 
     :return: status code of the hook. 0 for success.
     """
     extras = json.loads(env['RC_SCM_DATA'])
     rev_data = _parse_git_ref_lines(revision_lines)
     if 'push' not in extras['hooks']:
         return 0
     _fix_hooks_executables()
 
     empty_commit_id = '0' * 40
 
     detect_force_push = extras.get('detect_force_push')
 
     for push_ref in rev_data:
         # store our git-env which holds the temp store
         push_ref['git_env'] = _get_git_env()
         push_ref['pruned_sha'] = ''
         if not detect_force_push:
             # don't check for forced-push when we don't need to
             continue
 
         type_ = push_ref['type']
         new_branch = push_ref['old_rev'] == empty_commit_id
         delete_branch = push_ref['new_rev'] == empty_commit_id
         if type_ == 'heads' and not (new_branch or delete_branch):
             old_rev = push_ref['old_rev']
             new_rev = push_ref['new_rev']
             cmd = [settings.GIT_EXECUTABLE(), 'rev-list', old_rev, f'^{new_rev}']
             stdout, stderr = subprocessio.run_command(
                 cmd, env=os.environ.copy())
             # means we're having some non-reachable objects, this forced push was used
             if stdout:
                 push_ref['pruned_sha'] = stdout.splitlines()
 
     extras['hook_type'] = 'pre_receive'
     extras['commit_ids'] = rev_data
 
     stdout = sys.stdout
     status_code = _call_hook('pre_push', extras, GitMessageWriter(stdout))
 
     return status_code
 
 
 def git_post_receive(unused_repo_path, revision_lines, env) -> int:
     """
     Post push hook.
 
     :return: status code of the hook. 0 for success.
     """
     extras = json.loads(env['RC_SCM_DATA'])
     if 'push' not in extras['hooks']:
         return 0
 
     _fix_hooks_executables()
 
     rev_data = _parse_git_ref_lines(revision_lines)
 
     git_revs = []
 
     # N.B.(skreft): it is ok to just call git, as git before calling a
     # subcommand sets the PATH environment variable so that it point to the
     # correct version of the git executable.
     empty_commit_id = '0' * 40
     branches = []
     tags = []
     for push_ref in rev_data:
         type_ = push_ref['type']
 
         if type_ == 'heads':
             # starting new branch case
             if push_ref['old_rev'] == empty_commit_id:
                 push_ref_name = push_ref['name']
 
                 if push_ref_name not in branches:
                     branches.append(push_ref_name)
 
                 need_head_set = ''
                 with Repository(os.getcwd()) as repo:
                     try:
                         repo.head
                     except pygit2.GitError:
                         need_head_set = f'refs/heads/{push_ref_name}'
 
                     if need_head_set:
                         repo.set_head(need_head_set)
                         print(f"Setting default branch to {push_ref_name}")
 
                 cmd = [settings.GIT_EXECUTABLE(), 'for-each-ref', '--format=%(refname)', 'refs/heads/*']
                 stdout, stderr = subprocessio.run_command(
                     cmd, env=os.environ.copy())
                 heads = safe_str(stdout)
                 heads = heads.replace(push_ref['ref'], '')
                 heads = ' '.join(head for head
                                  in heads.splitlines() if head) or '.'
                 cmd = [settings.GIT_EXECUTABLE(), 'log', '--reverse',
                        '--pretty=format:%H', '--', push_ref['new_rev'],
                        '--not', heads]
                 stdout, stderr = subprocessio.run_command(
                     cmd, env=os.environ.copy())
                 git_revs.extend(list(map(ascii_str, stdout.splitlines())))
 
             # delete branch case
             elif push_ref['new_rev'] == empty_commit_id:
                 git_revs.append(f'delete_branch=>{push_ref["name"]}')
             else:
                 if push_ref['name'] not in branches:
                     branches.append(push_ref['name'])
 
                 cmd = [settings.GIT_EXECUTABLE(), 'log',
                        f'{push_ref["old_rev"]}..{push_ref["new_rev"]}',
                        '--reverse', '--pretty=format:%H']
                 stdout, stderr = subprocessio.run_command(
                     cmd, env=os.environ.copy())
                 # we get bytes from stdout, we need str to be consistent
                 log_revs = list(map(ascii_str, stdout.splitlines()))
                 git_revs.extend(log_revs)
 
                 # Pure pygit2 impl. but still 2-3x slower :/
                 # results = []
                 #
                 # with Repository(os.getcwd()) as repo:
                 #     repo_new_rev = repo[push_ref['new_rev']]
                 #     repo_old_rev = repo[push_ref['old_rev']]
                 #     walker = repo.walk(repo_new_rev.id, pygit2.GIT_SORT_TOPOLOGICAL)
                 #
                 #     for commit in walker:
                 #         if commit.id == repo_old_rev.id:
                 #             break
                 #         results.append(commit.id.hex)
                 #     # reverse the order, can't use GIT_SORT_REVERSE
                 #     log_revs = results[::-1]
 
         elif type_ == 'tags':
             if push_ref['name'] not in tags:
                 tags.append(push_ref['name'])
             git_revs.append(f'tag=>{push_ref["name"]}')
 
     extras['hook_type'] = 'post_receive'
     extras['commit_ids'] = git_revs
     extras['new_refs'] = {
         'branches': branches,
         'bookmarks': [],
         'tags': tags,
     }
 
     stdout = sys.stdout
 
     if 'repo_size' in extras['hooks']:
         try:
             _call_hook('repo_size', extras, GitMessageWriter(stdout))
         except Exception:
             pass
 
     status_code = _call_hook('post_push', extras, GitMessageWriter(stdout))
     return status_code
 
 
 def _get_extras_from_txn_id(path, txn_id):
     _fix_hooks_executables()
 
     extras = {}
     try:
         cmd = [settings.SVNLOOK_EXECUTABLE(), 'pget',
                '-t', txn_id,
                '--revprop', path, 'rc-scm-extras']
         stdout, stderr = subprocessio.run_command(
             cmd, env=os.environ.copy())
         extras = json.loads(base64.urlsafe_b64decode(stdout))
     except Exception:
         log.exception('Failed to extract extras info from txn_id')
 
     return extras
 
 
 def _get_extras_from_commit_id(commit_id, path):
     _fix_hooks_executables()
 
     extras = {}
     try:
         cmd = [settings.SVNLOOK_EXECUTABLE(), 'pget',
                '-r', commit_id,
                '--revprop', path, 'rc-scm-extras']
         stdout, stderr = subprocessio.run_command(
             cmd, env=os.environ.copy())
         extras = json.loads(base64.urlsafe_b64decode(stdout))
     except Exception:
         log.exception('Failed to extract extras info from commit_id')
 
     return extras
 
 
 def svn_pre_commit(repo_path, commit_data, env):
 
     path, txn_id = commit_data
     branches = []
     tags = []
 
     if env.get('RC_SCM_DATA'):
         extras = json.loads(env['RC_SCM_DATA'])
     else:
         # fallback method to read from TXN-ID stored data
         extras = _get_extras_from_txn_id(path, txn_id)
 
     if not extras:
         #TODO: temporary fix until svn txn-id changes are merged
         return 0
         raise ValueError('Failed to extract context data called extras for hook execution')
 
     extras['hook_type'] = 'pre_commit'
     extras['commit_ids'] = [txn_id]
     extras['txn_id'] = txn_id
     extras['new_refs'] = {
         'total_commits': 1,
         'branches': branches,
         'bookmarks': [],
         'tags': tags,
     }
 
     return _call_hook('pre_push', extras, SvnMessageWriter())
 
 
 def svn_post_commit(repo_path, commit_data, env):
     """
     commit_data is path, rev, txn_id
     """
 
     if len(commit_data) == 3:
         path, commit_id, txn_id = commit_data
     elif len(commit_data) == 2:
         log.error('Failed to extract txn_id from commit_data using legacy method. '
                   'Some functionality might be limited')
         path, commit_id = commit_data
         txn_id = None
     else:
         return 0
 
     branches = []
     tags = []
 
     if env.get('RC_SCM_DATA'):
         extras = json.loads(env['RC_SCM_DATA'])
     else:
         # fallback method to read from TXN-ID stored data
         extras = _get_extras_from_commit_id(commit_id, path)
 
     if not extras:
         #TODO: temporary fix until svn txn-id changes are merged
         return 0
         raise ValueError('Failed to extract context data called extras for hook execution')
 
     extras['hook_type'] = 'post_commit'
     extras['commit_ids'] = [commit_id]
     extras['txn_id'] = txn_id
     extras['new_refs'] = {
         'branches': branches,
         'bookmarks': [],
         'tags': tags,
         'total_commits': 1,
     }
 
     if 'repo_size' in extras['hooks']:
         try:
             _call_hook('repo_size', extras, SvnMessageWriter())
         except Exception:
             pass
 
     return _call_hook('post_push', extras, SvnMessageWriter())

vcsserver/http_main.py

@@ -1,774 +1,774 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import io
 import os
 import platform
 import sys
 import locale
 import logging
 import uuid
 import time
 import wsgiref.util
 import tempfile
 import psutil
 
 from itertools import chain
 
 import msgpack
 import configparser
 
 from pyramid.config import Configurator
 from pyramid.wsgi import wsgiapp
 from pyramid.response import Response
 
 from vcsserver.base import BytesEnvelope, BinaryEnvelope
-from vcsserver.lib.rc_json import json
+from vcsserver.lib.ext_json import json
 from vcsserver.config.settings_maker import SettingsMaker
 from vcsserver.str_utils import safe_int
 from vcsserver.lib.statsd_client import StatsdClient
 from vcsserver.tweens.request_wrapper import get_headers_call_context
 
 import vcsserver
 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
 from vcsserver.echo_stub.echo_app import EchoApp
 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
 from vcsserver.lib.exc_tracking import store_exception, format_exc
 from vcsserver.server import VcsServer
 
 strict_vcs = True
 
 git_import_err = None
 try:
     from vcsserver.remote.git_remote import GitFactory, GitRemote
 except ImportError as e:
     GitFactory = None
     GitRemote = None
     git_import_err = e
     if strict_vcs:
         raise
 
 
 hg_import_err = None
 try:
     from vcsserver.remote.hg_remote import MercurialFactory, HgRemote
 except ImportError as e:
     MercurialFactory = None
     HgRemote = None
     hg_import_err = e
     if strict_vcs:
         raise
 
 
 svn_import_err = None
 try:
     from vcsserver.remote.svn_remote import SubversionFactory, SvnRemote
 except ImportError as e:
     SubversionFactory = None
     SvnRemote = None
     svn_import_err = e
     if strict_vcs:
         raise
 
 log = logging.getLogger(__name__)
 
 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
 
 try:
     locale.setlocale(locale.LC_ALL, '')
 except locale.Error as e:
     log.error(
         'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
     os.environ['LC_ALL'] = 'C'
 
 
 def _is_request_chunked(environ):
     stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
     return stream
 
 
 def log_max_fd():
     try:
         maxfd = psutil.Process().rlimit(psutil.RLIMIT_NOFILE)[1]
         log.info('Max file descriptors value: %s', maxfd)
     except Exception:
         pass
 
 
 class VCS:
     def __init__(self, locale_conf=None, cache_config=None):
         self.locale = locale_conf
         self.cache_config = cache_config
         self._configure_locale()
 
         log_max_fd()
 
         if GitFactory and GitRemote:
             git_factory = GitFactory()
             self._git_remote = GitRemote(git_factory)
         else:
             log.error("Git client import failed: %s", git_import_err)
 
         if MercurialFactory and HgRemote:
             hg_factory = MercurialFactory()
             self._hg_remote = HgRemote(hg_factory)
         else:
             log.error("Mercurial client import failed: %s", hg_import_err)
 
         if SubversionFactory and SvnRemote:
             svn_factory = SubversionFactory()
 
             # hg factory is used for svn url validation
             hg_factory = MercurialFactory()
             self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
         else:
             log.error("Subversion client import failed: %s", svn_import_err)
 
         self._vcsserver = VcsServer()
 
     def _configure_locale(self):
         if self.locale:
             log.info('Settings locale: `LC_ALL` to %s', self.locale)
         else:
             log.info('Configuring locale subsystem based on environment variables')
         try:
             # If self.locale is the empty string, then the locale
             # module will use the environment variables. See the
             # documentation of the package `locale`.
             locale.setlocale(locale.LC_ALL, self.locale)
 
             language_code, encoding = locale.getlocale()
             log.info(
                 'Locale set to language code "%s" with encoding "%s".',
                 language_code, encoding)
         except locale.Error:
             log.exception('Cannot set locale, not configuring the locale system')
 
 
 class WsgiProxy:
     def __init__(self, wsgi):
         self.wsgi = wsgi
 
     def __call__(self, environ, start_response):
         input_data = environ['wsgi.input'].read()
         input_data = msgpack.unpackb(input_data)
 
         error = None
         try:
             data, status, headers = self.wsgi.handle(
                 input_data['environment'], input_data['input_data'],
                 *input_data['args'], **input_data['kwargs'])
         except Exception as e:
             data, status, headers = [], None, None
             error = {
                 'message': str(e),
                 '_vcs_kind': getattr(e, '_vcs_kind', None)
             }
 
         start_response(200, {})
         return self._iterator(error, status, headers, data)
 
     def _iterator(self, error, status, headers, data):
         initial_data = [
             error,
             status,
             headers,
         ]
 
         for d in chain(initial_data, data):
             yield msgpack.packb(d)
 
 
 def not_found(request):
     return {'status': '404 NOT FOUND'}
 
 
 class VCSViewPredicate:
     def __init__(self, val, config):
         self.remotes = val
 
     def text(self):
         return f'vcs view method = {list(self.remotes.keys())}'
 
     phash = text
 
     def __call__(self, context, request):
         """
         View predicate that returns true if given backend is supported by
         defined remotes.
         """
         backend = request.matchdict.get('backend')
         return backend in self.remotes
 
 
 class HTTPApplication:
     ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
 
     remote_wsgi = remote_wsgi
     _use_echo_app = False
 
     def __init__(self, settings=None, global_config=None):
 
         self.config = Configurator(settings=settings)
         # Init our statsd at very start
         self.config.registry.statsd = StatsdClient.statsd
         self.config.registry.vcs_call_context = {}
 
         self.global_config = global_config
         self.config.include('vcsserver.lib.rc_cache')
         self.config.include('vcsserver.lib.rc_cache.archive_cache')
 
         settings_locale = settings.get('locale', '') or 'en_US.UTF-8'
         vcs = VCS(locale_conf=settings_locale, cache_config=settings)
         self._remotes = {
             'hg': vcs._hg_remote,
             'git': vcs._git_remote,
             'svn': vcs._svn_remote,
             'server': vcs._vcsserver,
         }
         if settings.get('dev.use_echo_app', 'false').lower() == 'true':
             self._use_echo_app = True
             log.warning("Using EchoApp for VCS operations.")
             self.remote_wsgi = remote_wsgi_stub
 
         self._configure_settings(global_config, settings)
 
         self._configure()
 
     def _configure_settings(self, global_config, app_settings):
         """
         Configure the settings module.
         """
         settings_merged = global_config.copy()
         settings_merged.update(app_settings)
 
         binary_dir = app_settings['core.binary_dir']
 
         settings.BINARY_DIR = binary_dir
 
         # Store the settings to make them available to other modules.
         vcsserver.PYRAMID_SETTINGS = settings_merged
         vcsserver.CONFIG = settings_merged
 
     def _configure(self):
         self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
 
         self.config.add_route('service', '/_service')
         self.config.add_route('status', '/status')
         self.config.add_route('hg_proxy', '/proxy/hg')
         self.config.add_route('git_proxy', '/proxy/git')
 
         # rpc methods
         self.config.add_route('vcs', '/{backend}')
 
         # streaming rpc remote methods
         self.config.add_route('vcs_stream', '/{backend}/stream')
 
         # vcs operations clone/push as streaming
         self.config.add_route('stream_git', '/stream/git/*repo_name')
         self.config.add_route('stream_hg', '/stream/hg/*repo_name')
 
         self.config.add_view(self.status_view, route_name='status', renderer='json')
         self.config.add_view(self.service_view, route_name='service', renderer='msgpack')
 
         self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
         self.config.add_view(self.git_proxy(), route_name='git_proxy')
         self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack',
                              vcs_view=self._remotes)
         self.config.add_view(self.vcs_stream_view, route_name='vcs_stream',
                              vcs_view=self._remotes)
 
         self.config.add_view(self.hg_stream(), route_name='stream_hg')
         self.config.add_view(self.git_stream(), route_name='stream_git')
 
         self.config.add_view_predicate('vcs_view', VCSViewPredicate)
 
         self.config.add_notfound_view(not_found, renderer='json')
 
         self.config.add_view(self.handle_vcs_exception, context=Exception)
 
         self.config.add_tween(
             'vcsserver.tweens.request_wrapper.RequestWrapperTween',
         )
         self.config.add_request_method(
             'vcsserver.lib.request_counter.get_request_counter',
             'request_count')
 
     def wsgi_app(self):
         return self.config.make_wsgi_app()
 
     def _vcs_view_params(self, request):
         remote = self._remotes[request.matchdict['backend']]
         payload = msgpack.unpackb(request.body, use_list=True)
 
         method = payload.get('method')
         params = payload['params']
         wire = params.get('wire')
         args = params.get('args')
         kwargs = params.get('kwargs')
         context_uid = None
 
         request.registry.vcs_call_context = {
             'method': method,
             'repo_name': payload.get('_repo_name'),
         }
 
         if wire:
             try:
                 wire['context'] = context_uid = uuid.UUID(wire['context'])
             except KeyError:
                 pass
             args.insert(0, wire)
         repo_state_uid = wire.get('repo_state_uid') if wire else None
 
         # NOTE(marcink): trading complexity for slight performance
         if log.isEnabledFor(logging.DEBUG):
             # also we SKIP printing out any of those methods args since they maybe excessive
             just_args_methods = {
                 'commitctx': ('content', 'removed', 'updated'),
                 'commit': ('content', 'removed', 'updated')
             }
             if method in just_args_methods:
                 skip_args = just_args_methods[method]
                 call_args = ''
                 call_kwargs = {}
                 for k in kwargs:
                     if k in skip_args:
                         # replace our skip key with dummy
                         call_kwargs[k] = f'RemovedParam({k})'
                     else:
                         call_kwargs[k] = kwargs[k]
             else:
                 call_args = args[1:]
                 call_kwargs = kwargs
 
             log.debug('Method requested:`%s` with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
                       method, call_args, call_kwargs, context_uid, repo_state_uid)
 
         statsd = request.registry.statsd
         if statsd:
             statsd.incr(
                 'vcsserver_method_total', tags=[
                     f"method:{method}",
                 ])
         return payload, remote, method, args, kwargs
 
     def vcs_view(self, request):
 
         payload, remote, method, args, kwargs = self._vcs_view_params(request)
         payload_id = payload.get('id')
 
         try:
             resp = getattr(remote, method)(*args, **kwargs)
         except Exception as e:
             exc_info = list(sys.exc_info())
             exc_type, exc_value, exc_traceback = exc_info
 
             org_exc = getattr(e, '_org_exc', None)
             org_exc_name = None
             org_exc_tb = ''
             if org_exc:
                 org_exc_name = org_exc.__class__.__name__
                 org_exc_tb = getattr(e, '_org_exc_tb', '')
                 # replace our "faked" exception with our org
                 exc_info[0] = org_exc.__class__
                 exc_info[1] = org_exc
 
             should_store_exc = True
             if org_exc:
                 def get_exc_fqn(_exc_obj):
                     module_name = getattr(org_exc.__class__, '__module__', 'UNKNOWN')
                     return module_name + '.' + org_exc_name
 
                 exc_fqn = get_exc_fqn(org_exc)
 
                 if exc_fqn in ['mercurial.error.RepoLookupError',
                                'vcsserver.exceptions.RefNotFoundException']:
                     should_store_exc = False
 
             if should_store_exc:
                 store_exception(id(exc_info), exc_info, request_path=request.path)
 
             tb_info = format_exc(exc_info)
 
             type_ = e.__class__.__name__
             if type_ not in self.ALLOWED_EXCEPTIONS:
                 type_ = None
 
             resp = {
                 'id': payload_id,
                 'error': {
                     'message': str(e),
                     'traceback': tb_info,
                     'org_exc': org_exc_name,
                     'org_exc_tb': org_exc_tb,
                     'type': type_
                 }
             }
 
             try:
                 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
             except AttributeError:
                 pass
         else:
             resp = {
                 'id': payload_id,
                 'result': resp
             }
         log.debug('Serving data for method %s', method)
         return resp
 
     def vcs_stream_view(self, request):
         payload, remote, method, args, kwargs = self._vcs_view_params(request)
         # this method has a stream: marker we remove it here
         method = method.split('stream:')[-1]
         chunk_size = safe_int(payload.get('chunk_size')) or 4096
 
         resp = getattr(remote, method)(*args, **kwargs)
 
         def get_chunked_data(method_resp):
             stream = io.BytesIO(method_resp)
             while 1:
                 chunk = stream.read(chunk_size)
                 if not chunk:
                     break
                 yield chunk
 
         response = Response(app_iter=get_chunked_data(resp))
         response.content_type = 'application/octet-stream'
 
         return response
 
     def status_view(self, request):
         import vcsserver
         _platform_id = platform.uname()[1] or 'instance'
 
         return {
             "status": "OK",
             "vcsserver_version": vcsserver.get_version(),
             "platform": _platform_id,
             "pid": os.getpid(),
         }
 
     def service_view(self, request):
         import vcsserver
 
         payload = msgpack.unpackb(request.body, use_list=True)
         server_config, app_config = {}, {}
 
         try:
             path = self.global_config['__file__']
             config = configparser.RawConfigParser()
 
             config.read(path)
 
             if config.has_section('server:main'):
                 server_config = dict(config.items('server:main'))
             if config.has_section('app:main'):
                 app_config = dict(config.items('app:main'))
 
         except Exception:
             log.exception('Failed to read .ini file for display')
 
         environ = list(os.environ.items())
 
         resp = {
             'id': payload.get('id'),
             'result': dict(
                 version=vcsserver.get_version(),
                 config=server_config,
                 app_config=app_config,
                 environ=environ,
                 payload=payload,
             )
         }
         return resp
 
     def _msgpack_renderer_factory(self, info):
 
         def _render(value, system):
             bin_type = False
             res = value.get('result')
             if isinstance(res, BytesEnvelope):
                 log.debug('Result is wrapped in BytesEnvelope type')
                 bin_type = True
             elif isinstance(res, BinaryEnvelope):
                 log.debug('Result is wrapped in BinaryEnvelope type')
                 value['result'] = res.val
                 bin_type = True
 
             request = system.get('request')
             if request is not None:
                 response = request.response
                 ct = response.content_type
                 if ct == response.default_content_type:
                     response.content_type = 'application/x-msgpack'
                     if bin_type:
                         response.content_type = 'application/x-msgpack-bin'
 
             return msgpack.packb(value, use_bin_type=bin_type)
         return _render
 
     def set_env_from_config(self, environ, config):
         dict_conf = {}
         try:
             for elem in config:
                 if elem[0] == 'rhodecode':
                     dict_conf = json.loads(elem[2])
                     break
         except Exception:
             log.exception('Failed to fetch SCM CONFIG')
             return
 
         username = dict_conf.get('username')
         if username:
             environ['REMOTE_USER'] = username
             # mercurial specific, some extension api rely on this
             environ['HGUSER'] = username
 
         ip = dict_conf.get('ip')
         if ip:
             environ['REMOTE_HOST'] = ip
 
         if _is_request_chunked(environ):
             # set the compatibility flag for webob
             environ['wsgi.input_terminated'] = True
 
     def hg_proxy(self):
         @wsgiapp
         def _hg_proxy(environ, start_response):
             app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
             return app(environ, start_response)
         return _hg_proxy
 
     def git_proxy(self):
         @wsgiapp
         def _git_proxy(environ, start_response):
             app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
             return app(environ, start_response)
         return _git_proxy
 
     def hg_stream(self):
         if self._use_echo_app:
             @wsgiapp
             def _hg_stream(environ, start_response):
                 app = EchoApp('fake_path', 'fake_name', None)
                 return app(environ, start_response)
             return _hg_stream
         else:
             @wsgiapp
             def _hg_stream(environ, start_response):
                 log.debug('http-app: handling hg stream')
                 call_context = get_headers_call_context(environ)
 
                 repo_path = call_context['repo_path']
                 repo_name = call_context['repo_name']
                 config = call_context['repo_config']
 
                 app = scm_app.create_hg_wsgi_app(
                     repo_path, repo_name, config)
 
                 # Consistent path information for hgweb
                 environ['PATH_INFO'] = call_context['path_info']
                 environ['REPO_NAME'] = repo_name
                 self.set_env_from_config(environ, config)
 
                 log.debug('http-app: starting app handler '
                           'with %s and process request', app)
                 return app(environ, ResponseFilter(start_response))
             return _hg_stream
 
     def git_stream(self):
         if self._use_echo_app:
             @wsgiapp
             def _git_stream(environ, start_response):
                 app = EchoApp('fake_path', 'fake_name', None)
                 return app(environ, start_response)
             return _git_stream
         else:
             @wsgiapp
             def _git_stream(environ, start_response):
                 log.debug('http-app: handling git stream')
 
                 call_context = get_headers_call_context(environ)
 
                 repo_path = call_context['repo_path']
                 repo_name = call_context['repo_name']
                 config = call_context['repo_config']
 
                 environ['PATH_INFO'] = call_context['path_info']
                 self.set_env_from_config(environ, config)
 
                 content_type = environ.get('CONTENT_TYPE', '')
 
                 path = environ['PATH_INFO']
                 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
                 log.debug(
                     'LFS: Detecting if request `%s` is LFS server path based '
                     'on content type:`%s`, is_lfs:%s',
                     path, content_type, is_lfs_request)
 
                 if not is_lfs_request:
                     # fallback detection by path
                     if GIT_LFS_PROTO_PAT.match(path):
                         is_lfs_request = True
                     log.debug(
                         'LFS: fallback detection by path of: `%s`, is_lfs:%s',
                         path, is_lfs_request)
 
                 if is_lfs_request:
                     app = scm_app.create_git_lfs_wsgi_app(
                         repo_path, repo_name, config)
                 else:
                     app = scm_app.create_git_wsgi_app(
                         repo_path, repo_name, config)
 
                 log.debug('http-app: starting app handler '
                           'with %s and process request', app)
 
                 return app(environ, start_response)
 
             return _git_stream
 
     def handle_vcs_exception(self, exception, request):
         _vcs_kind = getattr(exception, '_vcs_kind', '')
 
         if _vcs_kind == 'repo_locked':
             headers_call_context = get_headers_call_context(request.environ)
             status_code = safe_int(headers_call_context['locked_status_code'])
 
             return HTTPRepoLocked(
                 title=str(exception), status_code=status_code, headers=[('X-Rc-Locked', '1')])
 
         elif _vcs_kind == 'repo_branch_protected':
             # Get custom repo-branch-protected status code if present.
             return HTTPRepoBranchProtected(
                 title=str(exception), headers=[('X-Rc-Branch-Protection', '1')])
 
         exc_info = request.exc_info
         store_exception(id(exc_info), exc_info)
 
         traceback_info = 'unavailable'
         if request.exc_info:
             traceback_info = format_exc(request.exc_info)
 
         log.error(
             'error occurred handling this request for path: %s, \n%s',
             request.path, traceback_info)
 
         statsd = request.registry.statsd
         if statsd:
             exc_type = f"{exception.__class__.__module__}.{exception.__class__.__name__}"
             statsd.incr('vcsserver_exception_total',
                         tags=[f"type:{exc_type}"])
         raise exception
 
 
 class ResponseFilter:
 
     def __init__(self, start_response):
         self._start_response = start_response
 
     def __call__(self, status, response_headers, exc_info=None):
         headers = tuple(
             (h, v) for h, v in response_headers
             if not wsgiref.util.is_hop_by_hop(h))
         return self._start_response(status, headers, exc_info)
 
 
 def sanitize_settings_and_apply_defaults(global_config, settings):
     _global_settings_maker = SettingsMaker(global_config)
     settings_maker = SettingsMaker(settings)
 
     settings_maker.make_setting('logging.autoconfigure', False, parser='bool')
 
     logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini')
     settings_maker.enable_logging(logging_conf)
 
     # Default includes, possible to change as a user
     pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
     log.debug("Using the following pyramid.includes: %s", pyramid_includes)
 
     settings_maker.make_setting('__file__', global_config.get('__file__'))
 
     settings_maker.make_setting('pyramid.default_locale_name', 'en')
     settings_maker.make_setting('locale', 'en_US.UTF-8')
 
     settings_maker.make_setting(
         'core.binary_dir', '/usr/local/bin/rhodecode_bin/vcs_bin',
         default_when_empty=True, parser='string:noquote')
 
     temp_store = tempfile.gettempdir()
     default_cache_dir = os.path.join(temp_store, 'rc_cache')
     # save default, cache dir, and use it for all backends later.
     default_cache_dir = settings_maker.make_setting(
         'cache_dir',
         default=default_cache_dir, default_when_empty=True,
         parser='dir:ensured')
 
     # exception store cache
     settings_maker.make_setting(
         'exception_tracker.store_path',
         default=os.path.join(default_cache_dir, 'exc_store'), default_when_empty=True,
         parser='dir:ensured'
     )
 
     # repo_object cache defaults
     settings_maker.make_setting(
         'rc_cache.repo_object.backend',
         default='dogpile.cache.rc.file_namespace',
         parser='string')
     settings_maker.make_setting(
         'rc_cache.repo_object.expiration_time',
         default=30 * 24 * 60 * 60,  # 30days
         parser='int')
     settings_maker.make_setting(
         'rc_cache.repo_object.arguments.filename',
         default=os.path.join(default_cache_dir, 'vcsserver_cache_repo_object.db'),
         parser='string')
 
     # statsd
     settings_maker.make_setting('statsd.enabled', False, parser='bool')
     settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
     settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
     settings_maker.make_setting('statsd.statsd_prefix', '')
     settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
 
     settings_maker.env_expand()
 
 
 def main(global_config, **settings):
     start_time = time.time()
     log.info('Pyramid app config starting')
 
     if MercurialFactory:
         hgpatches.patch_largefiles_capabilities()
         hgpatches.patch_subrepo_type_mapping()
 
     # Fill in and sanitize the defaults & do ENV expansion
     sanitize_settings_and_apply_defaults(global_config, settings)
 
     # init and bootstrap StatsdClient
     StatsdClient.setup(settings)
 
     pyramid_app = HTTPApplication(settings=settings, global_config=global_config).wsgi_app()
     total_time = time.time() - start_time
     log.info('Pyramid app created and configured in %.2fs', total_time)
     return pyramid_app

vcsserver/pygrack.py

@@ -1,417 +1,417 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 """Handles the Git smart protocol."""
 
 import os
 import socket
 import logging
 
 import dulwich.protocol
 from dulwich.protocol import CAPABILITY_SIDE_BAND, CAPABILITY_SIDE_BAND_64K
 from webob import Request, Response, exc
 
-from vcsserver.lib.rc_json import json
+from vcsserver.lib.ext_json import json
 from vcsserver import hooks, subprocessio
 from vcsserver.str_utils import ascii_bytes
 
 
 log = logging.getLogger(__name__)
 
 
 class FileWrapper:
     """File wrapper that ensures how much data is read from it."""
 
     def __init__(self, fd, content_length):
         self.fd = fd
         self.content_length = content_length
         self.remain = content_length
 
     def read(self, size):
         if size <= self.remain:
             try:
                 data = self.fd.read(size)
             except socket.error:
                 raise IOError(self)
             self.remain -= size
         elif self.remain:
             data = self.fd.read(self.remain)
             self.remain = 0
         else:
             data = None
         return data
 
     def __repr__(self):
         return '<FileWrapper {} len: {}, read: {}>'.format(
             self.fd, self.content_length, self.content_length - self.remain
         )
 
 
 class GitRepository:
     """WSGI app for handling Git smart protocol endpoints."""
 
     git_folder_signature = frozenset(('config', 'head', 'info', 'objects', 'refs'))
     commands = frozenset(('git-upload-pack', 'git-receive-pack'))
     valid_accepts = frozenset(f'application/x-{c}-result' for c in commands)
 
     # The last bytes are the SHA1 of the first 12 bytes.
     EMPTY_PACK = (
         b'PACK\x00\x00\x00\x02\x00\x00\x00\x00\x02\x9d\x08' +
         b'\x82;\xd8\xa8\xea\xb5\x10\xadj\xc7\\\x82<\xfd>\xd3\x1e'
     )
     FLUSH_PACKET = b"0000"
 
     SIDE_BAND_CAPS = frozenset((CAPABILITY_SIDE_BAND, CAPABILITY_SIDE_BAND_64K))
 
     def __init__(self, repo_name, content_path, git_path, update_server_info, extras):
         files = frozenset(f.lower() for f in os.listdir(content_path))
         valid_dir_signature = self.git_folder_signature.issubset(files)
 
         if not valid_dir_signature:
             raise OSError(f'{content_path} missing git signature')
 
         self.content_path = content_path
         self.repo_name = repo_name
         self.extras = extras
         self.git_path = git_path
         self.update_server_info = update_server_info
 
     def _get_fixedpath(self, path):
         """
         Small fix for repo_path
 
         :param path:
         """
         path = path.split(self.repo_name, 1)[-1]
         if path.startswith('.git'):
             # for bare repos we still get the .git prefix inside, we skip it
             # here, and remove from the service command
             path = path[4:]
 
         return path.strip('/')
 
     def inforefs(self, request, unused_environ):
         """
         WSGI Response producer for HTTP GET Git Smart
         HTTP /info/refs request.
         """
 
         git_command = request.GET.get('service')
         if git_command not in self.commands:
             log.debug('command %s not allowed', git_command)
             return exc.HTTPForbidden()
 
         # please, resist the urge to add '\n' to git capture and increment
         # line count by 1.
         # by git docs: Documentation/technical/http-protocol.txt#L214 \n is
         # a part of protocol.
         # The code in Git client not only does NOT need '\n', but actually
         # blows up if you sprinkle "flush" (0000) as "0001\n".
         # It reads binary, per number of bytes specified.
         # if you do add '\n' as part of data, count it.
         server_advert = f'# service={git_command}\n'
         packet_len = hex(len(server_advert) + 4)[2:].rjust(4, '0').lower()
         try:
             gitenv = dict(os.environ)
             # forget all configs
             gitenv['RC_SCM_DATA'] = json.dumps(self.extras)
             command = [self.git_path, git_command[4:], '--stateless-rpc',
                        '--advertise-refs', self.content_path]
             out = subprocessio.SubprocessIOChunker(
                 command,
                 env=gitenv,
                 starting_values=[ascii_bytes(packet_len + server_advert) + self.FLUSH_PACKET],
                 shell=False
             )
         except OSError:
             log.exception('Error processing command')
             raise exc.HTTPExpectationFailed()
 
         resp = Response()
         resp.content_type = f'application/x-{git_command}-advertisement'
         resp.charset = None
         resp.app_iter = out
 
         return resp
 
     def _get_want_capabilities(self, request):
         """Read the capabilities found in the first want line of the request."""
         pos = request.body_file_seekable.tell()
         first_line = request.body_file_seekable.readline()
         request.body_file_seekable.seek(pos)
 
         return frozenset(
             dulwich.protocol.extract_want_line_capabilities(first_line)[1])
 
     def _build_failed_pre_pull_response(self, capabilities, pre_pull_messages):
         """
         Construct a response with an empty PACK file.
 
         We use an empty PACK file, as that would trigger the failure of the pull
         or clone command.
 
         We also print in the error output a message explaining why the command
         was aborted.
 
         If additionally, the user is accepting messages we send them the output
         of the pre-pull hook.
 
         Note that for clients not supporting side-band we just send them the
         emtpy PACK file.
         """
 
         if self.SIDE_BAND_CAPS.intersection(capabilities):
             response = []
             proto = dulwich.protocol.Protocol(None, response.append)
             proto.write_pkt_line(dulwich.protocol.NAK_LINE)
 
             self._write_sideband_to_proto(proto, ascii_bytes(pre_pull_messages, allow_bytes=True), capabilities)
             # N.B.(skreft): Do not change the sideband channel to 3, as that
             # produces a fatal error in the client:
             #   fatal: error in sideband demultiplexer
             proto.write_sideband(
                 dulwich.protocol.SIDE_BAND_CHANNEL_PROGRESS,
                 ascii_bytes('Pre pull hook failed: aborting\n', allow_bytes=True))
             proto.write_sideband(
                 dulwich.protocol.SIDE_BAND_CHANNEL_DATA,
                 ascii_bytes(self.EMPTY_PACK, allow_bytes=True))
 
             # writes b"0000" as default
             proto.write_pkt_line(None)
 
             return response
         else:
             return [ascii_bytes(self.EMPTY_PACK, allow_bytes=True)]
 
     def _build_post_pull_response(self, response, capabilities, start_message, end_message):
         """
         Given a list response we inject the post-pull messages.
 
         We only inject the messages if the client supports sideband, and the
         response has the format:
             0008NAK\n...0000
 
         Note that we do not check the no-progress capability as by default, git
         sends it, which effectively would block all messages.
         """
 
         if not self.SIDE_BAND_CAPS.intersection(capabilities):
             return response
 
         if not start_message and not end_message:
             return response
 
         try:
             iter(response)
             # iterator probably will work, we continue
         except TypeError:
             raise TypeError(f'response must be an iterator: got {type(response)}')
         if isinstance(response, (list, tuple)):
             raise TypeError(f'response must be an iterator: got {type(response)}')
 
         def injected_response():
 
             do_loop = 1
             header_injected = 0
             next_item = None
             has_item = False
             item = b''
 
             while do_loop:
 
                 try:
                     next_item = next(response)
                 except StopIteration:
                     do_loop = 0
 
                 if has_item:
                     # last item ! alter it now
                     if do_loop == 0 and item.endswith(self.FLUSH_PACKET):
                         new_response = [item[:-4]]
                         new_response.extend(self._get_messages(end_message, capabilities))
                         new_response.append(self.FLUSH_PACKET)
                         item = b''.join(new_response)
 
                     yield item
 
                 has_item = True
                 item = next_item
 
                 # alter item if it's the initial chunk
                 if not header_injected and item.startswith(b'0008NAK\n'):
                     new_response = [b'0008NAK\n']
                     new_response.extend(self._get_messages(start_message, capabilities))
                     new_response.append(item[8:])
                     item = b''.join(new_response)
                     header_injected = 1
 
         return injected_response()
 
     def _write_sideband_to_proto(self, proto, data, capabilities):
         """
         Write the data to the proto's sideband number 2 == SIDE_BAND_CHANNEL_PROGRESS
 
         We do not use dulwich's write_sideband directly as it only supports
         side-band-64k.
         """
         if not data:
             return
 
         # N.B.(skreft): The values below are explained in the pack protocol
         # documentation, section Packfile Data.
         # https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt
         if CAPABILITY_SIDE_BAND_64K in capabilities:
             chunk_size = 65515
         elif CAPABILITY_SIDE_BAND in capabilities:
             chunk_size = 995
         else:
             return
 
         chunker = (data[i:i + chunk_size] for i in range(0, len(data), chunk_size))
 
         for chunk in chunker:
             proto.write_sideband(dulwich.protocol.SIDE_BAND_CHANNEL_PROGRESS, ascii_bytes(chunk, allow_bytes=True))
 
     def _get_messages(self, data, capabilities):
         """Return a list with packets for sending data in sideband number 2."""
         response = []
         proto = dulwich.protocol.Protocol(None, response.append)
 
         self._write_sideband_to_proto(proto, data, capabilities)
 
         return response
 
     def backend(self, request, environ):
         """
         WSGI Response producer for HTTP POST Git Smart HTTP requests.
         Reads commands and data from HTTP POST's body.
         returns an iterator obj with contents of git command's
         response to stdout
         """
         # TODO(skreft): think how we could detect an HTTPLockedException, as
         # we probably want to have the same mechanism used by mercurial and
         # simplevcs.
         # For that we would need to parse the output of the command looking for
         # some signs of the HTTPLockedError, parse the data and reraise it in
         # pygrack. However, that would interfere with the streaming.
         #
         # Now the output of a blocked push is:
         # Pushing to http://test_regular:test12@127.0.0.1:5001/vcs_test_git
         # POST git-receive-pack (1047 bytes)
         # remote: ERROR: Repository `vcs_test_git` locked by user `test_admin`. Reason:`lock_auto`
         # To http://test_regular:test12@127.0.0.1:5001/vcs_test_git
         # ! [remote rejected] master -> master (pre-receive hook declined)
         # error: failed to push some refs to 'http://test_regular:test12@127.0.0.1:5001/vcs_test_git'
 
         git_command = self._get_fixedpath(request.path_info)
         if git_command not in self.commands:
             log.debug('command %s not allowed', git_command)
             return exc.HTTPForbidden()
 
         capabilities = None
         if git_command == 'git-upload-pack':
             capabilities = self._get_want_capabilities(request)
 
         if 'CONTENT_LENGTH' in environ:
             inputstream = FileWrapper(request.body_file_seekable,
                                       request.content_length)
         else:
             inputstream = request.body_file_seekable
 
         resp = Response()
         resp.content_type = f'application/x-{git_command}-result'
         resp.charset = None
 
         pre_pull_messages = ''
         # Upload-pack == clone
         if git_command == 'git-upload-pack':
             hook_response = hooks.git_pre_pull(self.extras)
             if hook_response.status != 0:
                 pre_pull_messages = hook_response.output
                 resp.app_iter = self._build_failed_pre_pull_response(
                     capabilities, pre_pull_messages)
                 return resp
 
         gitenv = dict(os.environ)
         # forget all configs
         gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
         gitenv['RC_SCM_DATA'] = json.dumps(self.extras)
         cmd = [self.git_path, git_command[4:], '--stateless-rpc',
                self.content_path]
         log.debug('handling cmd %s', cmd)
 
         out = subprocessio.SubprocessIOChunker(
             cmd,
             input_stream=inputstream,
             env=gitenv,
             cwd=self.content_path,
             shell=False,
             fail_on_stderr=False,
             fail_on_return_code=False
         )
 
         if self.update_server_info and git_command == 'git-receive-pack':
             # We need to fully consume the iterator here, as the
             # update-server-info command needs to be run after the push.
             out = list(out)
 
             # Updating refs manually after each push.
             # This is required as some clients are exposing Git repos internally
             # with the dumb protocol.
             cmd = [self.git_path, 'update-server-info']
             log.debug('handling cmd %s', cmd)
             output = subprocessio.SubprocessIOChunker(
                 cmd,
                 input_stream=inputstream,
                 env=gitenv,
                 cwd=self.content_path,
                 shell=False,
                 fail_on_stderr=False,
                 fail_on_return_code=False
             )
             # Consume all the output so the subprocess finishes
             for _ in output:
                 pass
 
         # Upload-pack == clone
         if git_command == 'git-upload-pack':
             hook_response = hooks.git_post_pull(self.extras)
             post_pull_messages = hook_response.output
             resp.app_iter = self._build_post_pull_response(out, capabilities, pre_pull_messages, post_pull_messages)
         else:
             resp.app_iter = out
 
         return resp
 
     def __call__(self, environ, start_response):
         request = Request(environ)
         _path = self._get_fixedpath(request.path_info)
         if _path.startswith('info/refs'):
             app = self.inforefs
         else:
             app = self.backend
 
         try:
             resp = app(request, environ)
         except exc.HTTPException as error:
             log.exception('HTTP Error')
             resp = error
         except Exception:
             log.exception('Unknown error')
             resp = exc.HTTPInternalServerError()
 
         return resp(environ, start_response)

vcsserver/tests/test_hooks.py

@@ -1,257 +1,257 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import threading
 import msgpack
 
 from http.server import BaseHTTPRequestHandler
 from socketserver import TCPServer
 
 import mercurial.ui
 import mock
 import pytest
 
 from vcsserver.hooks import HooksHttpClient
-from vcsserver.lib.rc_json import json
+from vcsserver.lib.ext_json import json
 from vcsserver import hooks
 
 
 def get_hg_ui(extras=None):
     """Create a Config object with a valid RC_SCM_DATA entry."""
     extras = extras or {}
     required_extras = {
         'username': '',
         'repository': '',
         'locked_by': '',
         'scm': '',
         'make_lock': '',
         'action': '',
         'ip': '',
         'hooks_uri': 'fake_hooks_uri',
     }
     required_extras.update(extras)
     hg_ui = mercurial.ui.ui()
     hg_ui.setconfig(b'rhodecode', b'RC_SCM_DATA', json.dumps(required_extras))
 
     return hg_ui
 
 
 def test_git_pre_receive_is_disabled():
     extras = {'hooks': ['pull']}
     response = hooks.git_pre_receive(None, None,
                                      {'RC_SCM_DATA': json.dumps(extras)})
 
     assert response == 0
 
 
 def test_git_post_receive_is_disabled():
     extras = {'hooks': ['pull']}
     response = hooks.git_post_receive(None, '',
                                       {'RC_SCM_DATA': json.dumps(extras)})
 
     assert response == 0
 
 
 def test_git_post_receive_calls_repo_size():
     extras = {'hooks': ['push', 'repo_size']}
 
     with mock.patch.object(hooks, '_call_hook') as call_hook_mock:
         hooks.git_post_receive(
             None, '', {'RC_SCM_DATA': json.dumps(extras)})
     extras.update({'commit_ids': [], 'hook_type': 'post_receive',
                    'new_refs': {'bookmarks': [], 'branches': [], 'tags': []}})
     expected_calls = [
         mock.call('repo_size', extras, mock.ANY),
         mock.call('post_push', extras, mock.ANY),
     ]
     assert call_hook_mock.call_args_list == expected_calls
 
 
 def test_git_post_receive_does_not_call_disabled_repo_size():
     extras = {'hooks': ['push']}
 
     with mock.patch.object(hooks, '_call_hook') as call_hook_mock:
         hooks.git_post_receive(
             None, '', {'RC_SCM_DATA': json.dumps(extras)})
     extras.update({'commit_ids': [], 'hook_type': 'post_receive',
                    'new_refs': {'bookmarks': [], 'branches': [], 'tags': []}})
     expected_calls = [
         mock.call('post_push', extras, mock.ANY)
     ]
     assert call_hook_mock.call_args_list == expected_calls
 
 
 def test_repo_size_exception_does_not_affect_git_post_receive():
     extras = {'hooks': ['push', 'repo_size']}
     status = 0
 
     def side_effect(name, *args, **kwargs):
         if name == 'repo_size':
             raise Exception('Fake exception')
         else:
             return status
 
     with mock.patch.object(hooks, '_call_hook') as call_hook_mock:
         call_hook_mock.side_effect = side_effect
         result = hooks.git_post_receive(
             None, '', {'RC_SCM_DATA': json.dumps(extras)})
     assert result == status
 
 
 def test_git_pre_pull_is_disabled():
     assert hooks.git_pre_pull({'hooks': ['push']}) == hooks.HookResponse(0, '')
 
 
 def test_git_post_pull_is_disabled():
     assert (
         hooks.git_post_pull({'hooks': ['push']}) == hooks.HookResponse(0, ''))
 
 
 class TestGetHooksClient:
 
     def test_returns_http_client_when_protocol_matches(self):
         hooks_uri = 'localhost:8000'
         result = hooks._get_hooks_client({
             'hooks_uri': hooks_uri,
             'hooks_protocol': 'http'
         })
         assert isinstance(result, hooks.HooksHttpClient)
         assert result.hooks_uri == hooks_uri
 
     def test_return_celery_client_when_queue_and_backend_provided(self):
         task_queue = 'redis://task_queue:0'
         task_backend = task_queue
         result = hooks._get_hooks_client({
             'task_queue': task_queue,
             'task_backend': task_backend
         })
         assert isinstance(result, hooks.HooksCeleryClient)
 
 
 class TestHooksHttpClient:
     def test_init_sets_hooks_uri(self):
         uri = 'localhost:3000'
         client = hooks.HooksHttpClient(uri)
         assert client.hooks_uri == uri
 
     def test_serialize_returns_serialized_string(self):
         client = hooks.HooksHttpClient('localhost:3000')
         hook_name = 'test'
         extras = {
             'first': 1,
             'second': 'two'
         }
         hooks_proto, result = client._serialize(hook_name, extras)
         expected_result = msgpack.packb({
             'method': hook_name,
             'extras': extras,
         })
         assert hooks_proto == {'rc-hooks-protocol': 'msgpack.v1', 'Connection': 'keep-alive'}
         assert result == expected_result
 
     def test_call_queries_http_server(self, http_mirror):
         client = hooks.HooksHttpClient(http_mirror.uri)
         hook_name = 'test'
         extras = {
             'first': 1,
             'second': 'two'
         }
         result = client(hook_name, extras)
         expected_result = msgpack.unpackb(msgpack.packb({
             'method': hook_name,
             'extras': extras
         }), raw=False)
         assert result == expected_result
 
 
 @pytest.fixture
 def http_mirror(request):
     server = MirrorHttpServer()
     request.addfinalizer(server.stop)
     return server
 
 
 class MirrorHttpHandler(BaseHTTPRequestHandler):
 
     def do_POST(self):
         length = int(self.headers['Content-Length'])
         body = self.rfile.read(length)
         self.send_response(200)
         self.end_headers()
         self.wfile.write(body)
 
 
 class MirrorHttpServer:
     ip_address = '127.0.0.1'
     port = 0
 
     def __init__(self):
         self._daemon = TCPServer((self.ip_address, 0), MirrorHttpHandler)
         _, self.port = self._daemon.server_address
         self._thread = threading.Thread(target=self._daemon.serve_forever)
         self._thread.daemon = True
         self._thread.start()
 
     def stop(self):
         self._daemon.shutdown()
         self._thread.join()
         self._daemon = None
         self._thread = None
 
     @property
     def uri(self):
         return '{}:{}'.format(self.ip_address, self.port)
 
 
 def test_hooks_http_client_init():
     hooks_uri = 'http://localhost:8000'
     client = HooksHttpClient(hooks_uri)
     assert client.hooks_uri == hooks_uri
 
 
 def test_hooks_http_client_call():
     hooks_uri = 'http://localhost:8000'
 
     method = 'test_method'
     extras = {'key': 'value'}
 
     with \
         mock.patch('http.client.HTTPConnection') as mock_connection,\
         mock.patch('msgpack.load') as mock_load:
 
         client = HooksHttpClient(hooks_uri)
 
         mock_load.return_value = {'result': 'success'}
         response = mock.MagicMock()
         response.status = 200
         mock_connection.request.side_effect = None
         mock_connection.getresponse.return_value = response
 
         result = client(method, extras)
 
         mock_connection.assert_called_with(hooks_uri)
         mock_connection.return_value.request.assert_called_once()
         assert result == {'result': 'success'}
 
 
 def test_hooks_http_client_serialize():
     method = 'test_method'
     extras = {'key': 'value'}
     headers, body = HooksHttpClient._serialize(method, extras)
 
     assert headers == {'rc-hooks-protocol': HooksHttpClient.proto, 'Connection': 'keep-alive'}
     assert msgpack.unpackb(body) == {'method': method, 'extras': extras}