##// END OF EJS Templates
chore(refactor): renamed rc_json to ext_json for ce compat
super-admin -
r1243:d32b737d default
parent child Browse files
Show More
@@ -1,296 +1,296 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import re
19 19 import logging
20 20
21 21 from pyramid.config import Configurator
22 22 from pyramid.response import Response, FileIter
23 23 from pyramid.httpexceptions import (
24 24 HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden,
25 25 HTTPUnprocessableEntity)
26 26
27 from vcsserver.lib.rc_json import json
27 from vcsserver.lib.ext_json import json
28 28 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
29 29 from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator
30 30 from vcsserver.str_utils import safe_int
31 31
32 32 log = logging.getLogger(__name__)
33 33
34 34
35 35 GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs' # +json ?
36 36 GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))')
37 37
38 38
39 39 def write_response_error(http_exception, text=None):
40 40 content_type = GIT_LFS_CONTENT_TYPE + '+json'
41 41 _exception = http_exception(content_type=content_type)
42 42 _exception.content_type = content_type
43 43 if text:
44 44 _exception.body = json.dumps({'message': text})
45 45 log.debug('LFS: writing response of type %s to client with text:%s',
46 46 http_exception, text)
47 47 return _exception
48 48
49 49
50 50 class AuthHeaderRequired:
51 51 """
52 52 Decorator to check if request has proper auth-header
53 53 """
54 54
55 55 def __call__(self, func):
56 56 return get_cython_compat_decorator(self.__wrapper, func)
57 57
58 58 def __wrapper(self, func, *fargs, **fkwargs):
59 59 request = fargs[1]
60 60 auth = request.authorization
61 61 if not auth:
62 62 return write_response_error(HTTPForbidden)
63 63 return func(*fargs[1:], **fkwargs)
64 64
65 65
66 66 # views
67 67
68 68 def lfs_objects(request):
69 69 # indicate not supported, V1 API
70 70 log.warning('LFS: v1 api not supported, reporting it back to client')
71 71 return write_response_error(HTTPNotImplemented, 'LFS: v1 api not supported')
72 72
73 73
74 74 @AuthHeaderRequired()
75 75 def lfs_objects_batch(request):
76 76 """
77 77 The client sends the following information to the Batch endpoint to transfer some objects:
78 78
79 79 operation - Should be download or upload.
80 80 transfers - An optional Array of String identifiers for transfer
81 81 adapters that the client has configured. If omitted, the basic
82 82 transfer adapter MUST be assumed by the server.
83 83 objects - An Array of objects to download.
84 84 oid - String OID of the LFS object.
85 85 size - Integer byte size of the LFS object. Must be at least zero.
86 86 """
87 87 request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
88 88 auth = request.authorization
89 89 repo = request.matchdict.get('repo')
90 90 data = request.json
91 91 operation = data.get('operation')
92 92 http_scheme = request.registry.git_lfs_http_scheme
93 93
94 94 if operation not in ('download', 'upload'):
95 95 log.debug('LFS: unsupported operation:%s', operation)
96 96 return write_response_error(
97 97 HTTPBadRequest, f'unsupported operation mode: `{operation}`')
98 98
99 99 if 'objects' not in data:
100 100 log.debug('LFS: missing objects data')
101 101 return write_response_error(
102 102 HTTPBadRequest, 'missing objects data')
103 103
104 104 log.debug('LFS: handling operation of type: %s', operation)
105 105
106 106 objects = []
107 107 for o in data['objects']:
108 108 try:
109 109 oid = o['oid']
110 110 obj_size = o['size']
111 111 except KeyError:
112 112 log.exception('LFS, failed to extract data')
113 113 return write_response_error(
114 114 HTTPBadRequest, 'unsupported data in objects')
115 115
116 116 obj_data = {'oid': oid}
117 117 if http_scheme == 'http':
118 118 # Note(marcink): when using http, we might have a custom port
119 119 # so we skip setting it to http, url dispatch then wont generate a port in URL
120 120 # for development we need this
121 121 http_scheme = None
122 122
123 123 obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid,
124 124 _scheme=http_scheme)
125 125 obj_verify_href = request.route_url('lfs_objects_verify', repo=repo,
126 126 _scheme=http_scheme)
127 127 store = LFSOidStore(
128 128 oid, repo, store_location=request.registry.git_lfs_store_path)
129 129 handler = OidHandler(
130 130 store, repo, auth, oid, obj_size, obj_data,
131 131 obj_href, obj_verify_href)
132 132
133 133 # this verifies also OIDs
134 134 actions, errors = handler.exec_operation(operation)
135 135 if errors:
136 136 log.warning('LFS: got following errors: %s', errors)
137 137 obj_data['errors'] = errors
138 138
139 139 if actions:
140 140 obj_data['actions'] = actions
141 141
142 142 obj_data['size'] = obj_size
143 143 obj_data['authenticated'] = True
144 144 objects.append(obj_data)
145 145
146 146 result = {'objects': objects, 'transfer': 'basic'}
147 147 log.debug('LFS Response %s', safe_result(result))
148 148
149 149 return result
150 150
151 151
152 152 def lfs_objects_oid_upload(request):
153 153 request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
154 154 repo = request.matchdict.get('repo')
155 155 oid = request.matchdict.get('oid')
156 156 store = LFSOidStore(
157 157 oid, repo, store_location=request.registry.git_lfs_store_path)
158 158 engine = store.get_engine(mode='wb')
159 159 log.debug('LFS: starting chunked write of LFS oid: %s to storage', oid)
160 160
161 161 body = request.environ['wsgi.input']
162 162
163 163 with engine as f:
164 164 blksize = 64 * 1024 # 64kb
165 165 while True:
166 166 # read in chunks as stream comes in from Gunicorn
167 167 # this is a specific Gunicorn support function.
168 168 # might work differently on waitress
169 169 chunk = body.read(blksize)
170 170 if not chunk:
171 171 break
172 172 f.write(chunk)
173 173
174 174 return {'upload': 'ok'}
175 175
176 176
177 177 def lfs_objects_oid_download(request):
178 178 repo = request.matchdict.get('repo')
179 179 oid = request.matchdict.get('oid')
180 180
181 181 store = LFSOidStore(
182 182 oid, repo, store_location=request.registry.git_lfs_store_path)
183 183 if not store.has_oid():
184 184 log.debug('LFS: oid %s does not exists in store', oid)
185 185 return write_response_error(
186 186 HTTPNotFound, f'requested file with oid `{oid}` not found in store')
187 187
188 188 # TODO(marcink): support range header ?
189 189 # Range: bytes=0-, `bytes=(\d+)\-.*`
190 190
191 191 f = open(store.oid_path, 'rb')
192 192 response = Response(
193 193 content_type='application/octet-stream', app_iter=FileIter(f))
194 194 response.headers.add('X-RC-LFS-Response-Oid', str(oid))
195 195 return response
196 196
197 197
198 198 def lfs_objects_verify(request):
199 199 request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
200 200 repo = request.matchdict.get('repo')
201 201
202 202 data = request.json
203 203 oid = data.get('oid')
204 204 size = safe_int(data.get('size'))
205 205
206 206 if not (oid and size):
207 207 return write_response_error(
208 208 HTTPBadRequest, 'missing oid and size in request data')
209 209
210 210 store = LFSOidStore(
211 211 oid, repo, store_location=request.registry.git_lfs_store_path)
212 212 if not store.has_oid():
213 213 log.debug('LFS: oid %s does not exists in store', oid)
214 214 return write_response_error(
215 215 HTTPNotFound, f'oid `{oid}` does not exists in store')
216 216
217 217 store_size = store.size_oid()
218 218 if store_size != size:
219 219 msg = 'requested file size mismatch store size:{} requested:{}'.format(
220 220 store_size, size)
221 221 return write_response_error(
222 222 HTTPUnprocessableEntity, msg)
223 223
224 224 return {'message': {'size': 'ok', 'in_store': 'ok'}}
225 225
226 226
227 227 def lfs_objects_lock(request):
228 228 return write_response_error(
229 229 HTTPNotImplemented, 'GIT LFS locking api not supported')
230 230
231 231
232 232 def not_found(request):
233 233 return write_response_error(
234 234 HTTPNotFound, 'request path not found')
235 235
236 236
237 237 def lfs_disabled(request):
238 238 return write_response_error(
239 239 HTTPNotImplemented, 'GIT LFS disabled for this repo')
240 240
241 241
242 242 def git_lfs_app(config):
243 243
244 244 # v1 API deprecation endpoint
245 245 config.add_route('lfs_objects',
246 246 '/{repo:.*?[^/]}/info/lfs/objects')
247 247 config.add_view(lfs_objects, route_name='lfs_objects',
248 248 request_method='POST', renderer='json')
249 249
250 250 # locking API
251 251 config.add_route('lfs_objects_lock',
252 252 '/{repo:.*?[^/]}/info/lfs/locks')
253 253 config.add_view(lfs_objects_lock, route_name='lfs_objects_lock',
254 254 request_method=('POST', 'GET'), renderer='json')
255 255
256 256 config.add_route('lfs_objects_lock_verify',
257 257 '/{repo:.*?[^/]}/info/lfs/locks/verify')
258 258 config.add_view(lfs_objects_lock, route_name='lfs_objects_lock_verify',
259 259 request_method=('POST', 'GET'), renderer='json')
260 260
261 261 # batch API
262 262 config.add_route('lfs_objects_batch',
263 263 '/{repo:.*?[^/]}/info/lfs/objects/batch')
264 264 config.add_view(lfs_objects_batch, route_name='lfs_objects_batch',
265 265 request_method='POST', renderer='json')
266 266
267 267 # oid upload/download API
268 268 config.add_route('lfs_objects_oid',
269 269 '/{repo:.*?[^/]}/info/lfs/objects/{oid}')
270 270 config.add_view(lfs_objects_oid_upload, route_name='lfs_objects_oid',
271 271 request_method='PUT', renderer='json')
272 272 config.add_view(lfs_objects_oid_download, route_name='lfs_objects_oid',
273 273 request_method='GET', renderer='json')
274 274
275 275 # verification API
276 276 config.add_route('lfs_objects_verify',
277 277 '/{repo:.*?[^/]}/info/lfs/verify')
278 278 config.add_view(lfs_objects_verify, route_name='lfs_objects_verify',
279 279 request_method='POST', renderer='json')
280 280
281 281 # not found handler for API
282 282 config.add_notfound_view(not_found, renderer='json')
283 283
284 284
285 285 def create_app(git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme):
286 286 config = Configurator()
287 287 if git_lfs_enabled:
288 288 config.include(git_lfs_app)
289 289 config.registry.git_lfs_store_path = git_lfs_store_path
290 290 config.registry.git_lfs_http_scheme = git_lfs_http_scheme
291 291 else:
292 292 # not found handler for API, reporting disabled LFS support
293 293 config.add_notfound_view(lfs_disabled, renderer='json')
294 294
295 295 app = config.make_wsgi_app()
296 296 return app
@@ -1,274 +1,274 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import os
19 19 import pytest
20 20 from webtest.app import TestApp as WebObTestApp
21 21
22 from vcsserver.lib.rc_json import json
22 from vcsserver.lib.ext_json import json
23 23 from vcsserver.str_utils import safe_bytes
24 24 from vcsserver.git_lfs.app import create_app
25 25 from vcsserver.git_lfs.lib import LFSOidStore
26 26
27 27
28 28 @pytest.fixture(scope='function')
29 29 def git_lfs_app(tmpdir):
30 30 custom_app = WebObTestApp(create_app(
31 31 git_lfs_enabled=True, git_lfs_store_path=str(tmpdir),
32 32 git_lfs_http_scheme='http'))
33 33 custom_app._store = str(tmpdir)
34 34 return custom_app
35 35
36 36
37 37 @pytest.fixture(scope='function')
38 38 def git_lfs_https_app(tmpdir):
39 39 custom_app = WebObTestApp(create_app(
40 40 git_lfs_enabled=True, git_lfs_store_path=str(tmpdir),
41 41 git_lfs_http_scheme='https'))
42 42 custom_app._store = str(tmpdir)
43 43 return custom_app
44 44
45 45
46 46 @pytest.fixture()
47 47 def http_auth():
48 48 return {'HTTP_AUTHORIZATION': "Basic XXXXX"}
49 49
50 50
51 51 class TestLFSApplication:
52 52
53 53 def test_app_wrong_path(self, git_lfs_app):
54 54 git_lfs_app.get('/repo/info/lfs/xxx', status=404)
55 55
56 56 def test_app_deprecated_endpoint(self, git_lfs_app):
57 57 response = git_lfs_app.post('/repo/info/lfs/objects', status=501)
58 58 assert response.status_code == 501
59 59 assert json.loads(response.text) == {'message': 'LFS: v1 api not supported'}
60 60
61 61 def test_app_lock_verify_api_not_available(self, git_lfs_app):
62 62 response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501)
63 63 assert response.status_code == 501
64 64 assert json.loads(response.text) == {
65 65 'message': 'GIT LFS locking api not supported'}
66 66
67 67 def test_app_lock_api_not_available(self, git_lfs_app):
68 68 response = git_lfs_app.post('/repo/info/lfs/locks', status=501)
69 69 assert response.status_code == 501
70 70 assert json.loads(response.text) == {
71 71 'message': 'GIT LFS locking api not supported'}
72 72
73 73 def test_app_batch_api_missing_auth(self, git_lfs_app):
74 74 git_lfs_app.post_json(
75 75 '/repo/info/lfs/objects/batch', params={}, status=403)
76 76
77 77 def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth):
78 78 response = git_lfs_app.post_json(
79 79 '/repo/info/lfs/objects/batch', params={}, status=400,
80 80 extra_environ=http_auth)
81 81 assert json.loads(response.text) == {
82 82 'message': 'unsupported operation mode: `None`'}
83 83
84 84 def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth):
85 85 response = git_lfs_app.post_json(
86 86 '/repo/info/lfs/objects/batch', params={'operation': 'download'},
87 87 status=400, extra_environ=http_auth)
88 88 assert json.loads(response.text) == {
89 89 'message': 'missing objects data'}
90 90
91 91 def test_app_batch_api_unsupported_data_in_objects(
92 92 self, git_lfs_app, http_auth):
93 93 params = {'operation': 'download',
94 94 'objects': [{}]}
95 95 response = git_lfs_app.post_json(
96 96 '/repo/info/lfs/objects/batch', params=params, status=400,
97 97 extra_environ=http_auth)
98 98 assert json.loads(response.text) == {
99 99 'message': 'unsupported data in objects'}
100 100
101 101 def test_app_batch_api_download_missing_object(
102 102 self, git_lfs_app, http_auth):
103 103 params = {'operation': 'download',
104 104 'objects': [{'oid': '123', 'size': '1024'}]}
105 105 response = git_lfs_app.post_json(
106 106 '/repo/info/lfs/objects/batch', params=params,
107 107 extra_environ=http_auth)
108 108
109 109 expected_objects = [
110 110 {'authenticated': True,
111 111 'errors': {'error': {
112 112 'code': 404,
113 113 'message': 'object: 123 does not exist in store'}},
114 114 'oid': '123',
115 115 'size': '1024'}
116 116 ]
117 117 assert json.loads(response.text) == {
118 118 'objects': expected_objects, 'transfer': 'basic'}
119 119
120 120 def test_app_batch_api_download(self, git_lfs_app, http_auth):
121 121 oid = '456'
122 122 oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
123 123 if not os.path.isdir(os.path.dirname(oid_path)):
124 124 os.makedirs(os.path.dirname(oid_path))
125 125 with open(oid_path, 'wb') as f:
126 126 f.write(safe_bytes('OID_CONTENT'))
127 127
128 128 params = {'operation': 'download',
129 129 'objects': [{'oid': oid, 'size': '1024'}]}
130 130 response = git_lfs_app.post_json(
131 131 '/repo/info/lfs/objects/batch', params=params,
132 132 extra_environ=http_auth)
133 133
134 134 expected_objects = [
135 135 {'authenticated': True,
136 136 'actions': {
137 137 'download': {
138 138 'header': {'Authorization': 'Basic XXXXX'},
139 139 'href': 'http://localhost/repo/info/lfs/objects/456'},
140 140 },
141 141 'oid': '456',
142 142 'size': '1024'}
143 143 ]
144 144 assert json.loads(response.text) == {
145 145 'objects': expected_objects, 'transfer': 'basic'}
146 146
147 147 def test_app_batch_api_upload(self, git_lfs_app, http_auth):
148 148 params = {'operation': 'upload',
149 149 'objects': [{'oid': '123', 'size': '1024'}]}
150 150 response = git_lfs_app.post_json(
151 151 '/repo/info/lfs/objects/batch', params=params,
152 152 extra_environ=http_auth)
153 153 expected_objects = [
154 154 {'authenticated': True,
155 155 'actions': {
156 156 'upload': {
157 157 'header': {'Authorization': 'Basic XXXXX',
158 158 'Transfer-Encoding': 'chunked'},
159 159 'href': 'http://localhost/repo/info/lfs/objects/123'},
160 160 'verify': {
161 161 'header': {'Authorization': 'Basic XXXXX'},
162 162 'href': 'http://localhost/repo/info/lfs/verify'}
163 163 },
164 164 'oid': '123',
165 165 'size': '1024'}
166 166 ]
167 167 assert json.loads(response.text) == {
168 168 'objects': expected_objects, 'transfer': 'basic'}
169 169
170 170 def test_app_batch_api_upload_for_https(self, git_lfs_https_app, http_auth):
171 171 params = {'operation': 'upload',
172 172 'objects': [{'oid': '123', 'size': '1024'}]}
173 173 response = git_lfs_https_app.post_json(
174 174 '/repo/info/lfs/objects/batch', params=params,
175 175 extra_environ=http_auth)
176 176 expected_objects = [
177 177 {'authenticated': True,
178 178 'actions': {
179 179 'upload': {
180 180 'header': {'Authorization': 'Basic XXXXX',
181 181 'Transfer-Encoding': 'chunked'},
182 182 'href': 'https://localhost/repo/info/lfs/objects/123'},
183 183 'verify': {
184 184 'header': {'Authorization': 'Basic XXXXX'},
185 185 'href': 'https://localhost/repo/info/lfs/verify'}
186 186 },
187 187 'oid': '123',
188 188 'size': '1024'}
189 189 ]
190 190 assert json.loads(response.text) == {
191 191 'objects': expected_objects, 'transfer': 'basic'}
192 192
193 193 def test_app_verify_api_missing_data(self, git_lfs_app):
194 194 params = {'oid': 'missing'}
195 195 response = git_lfs_app.post_json(
196 196 '/repo/info/lfs/verify', params=params,
197 197 status=400)
198 198
199 199 assert json.loads(response.text) == {
200 200 'message': 'missing oid and size in request data'}
201 201
202 202 def test_app_verify_api_missing_obj(self, git_lfs_app):
203 203 params = {'oid': 'missing', 'size': '1024'}
204 204 response = git_lfs_app.post_json(
205 205 '/repo/info/lfs/verify', params=params,
206 206 status=404)
207 207
208 208 assert json.loads(response.text) == {
209 209 'message': 'oid `missing` does not exists in store'}
210 210
211 211 def test_app_verify_api_size_mismatch(self, git_lfs_app):
212 212 oid = 'existing'
213 213 oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
214 214 if not os.path.isdir(os.path.dirname(oid_path)):
215 215 os.makedirs(os.path.dirname(oid_path))
216 216 with open(oid_path, 'wb') as f:
217 217 f.write(safe_bytes('OID_CONTENT'))
218 218
219 219 params = {'oid': oid, 'size': '1024'}
220 220 response = git_lfs_app.post_json(
221 221 '/repo/info/lfs/verify', params=params, status=422)
222 222
223 223 assert json.loads(response.text) == {
224 224 'message': 'requested file size mismatch '
225 225 'store size:11 requested:1024'}
226 226
227 227 def test_app_verify_api(self, git_lfs_app):
228 228 oid = 'existing'
229 229 oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
230 230 if not os.path.isdir(os.path.dirname(oid_path)):
231 231 os.makedirs(os.path.dirname(oid_path))
232 232 with open(oid_path, 'wb') as f:
233 233 f.write(safe_bytes('OID_CONTENT'))
234 234
235 235 params = {'oid': oid, 'size': 11}
236 236 response = git_lfs_app.post_json(
237 237 '/repo/info/lfs/verify', params=params)
238 238
239 239 assert json.loads(response.text) == {
240 240 'message': {'size': 'ok', 'in_store': 'ok'}}
241 241
242 242 def test_app_download_api_oid_not_existing(self, git_lfs_app):
243 243 oid = 'missing'
244 244
245 245 response = git_lfs_app.get(
246 246 '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404)
247 247
248 248 assert json.loads(response.text) == {
249 249 'message': 'requested file with oid `missing` not found in store'}
250 250
251 251 def test_app_download_api(self, git_lfs_app):
252 252 oid = 'existing'
253 253 oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
254 254 if not os.path.isdir(os.path.dirname(oid_path)):
255 255 os.makedirs(os.path.dirname(oid_path))
256 256 with open(oid_path, 'wb') as f:
257 257 f.write(safe_bytes('OID_CONTENT'))
258 258
259 259 response = git_lfs_app.get(
260 260 '/repo/info/lfs/objects/{oid}'.format(oid=oid))
261 261 assert response
262 262
263 263 def test_app_upload(self, git_lfs_app):
264 264 oid = 'uploaded'
265 265
266 266 response = git_lfs_app.put(
267 267 '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT')
268 268
269 269 assert json.loads(response.text) == {'upload': 'ok'}
270 270
271 271 # verify that we actually wrote that OID
272 272 oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
273 273 assert os.path.isfile(oid_path)
274 274 assert 'CONTENT' == open(oid_path).read()
@@ -1,832 +1,832 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import io
19 19 import os
20 20 import sys
21 21 import logging
22 22 import collections
23 23 import base64
24 24 import msgpack
25 25 import dataclasses
26 26 import pygit2
27 27
28 28 import http.client
29 29 from celery import Celery
30 30
31 31 import mercurial.scmutil
32 32 import mercurial.node
33 33
34 from vcsserver.lib.rc_json import json
34 from vcsserver.lib.ext_json import json
35 35 from vcsserver import exceptions, subprocessio, settings
36 36 from vcsserver.str_utils import ascii_str, safe_str
37 37 from vcsserver.remote.git_remote import Repository
38 38
39 39 celery_app = Celery('__vcsserver__')
40 40 log = logging.getLogger(__name__)
41 41
42 42
43 43 class HooksHttpClient:
44 44 proto = 'msgpack.v1'
45 45 connection = None
46 46
47 47 def __init__(self, hooks_uri):
48 48 self.hooks_uri = hooks_uri
49 49
50 50 def __repr__(self):
51 51 return f'{self.__class__}(hook_uri={self.hooks_uri}, proto={self.proto})'
52 52
53 53 def __call__(self, method, extras):
54 54 connection = http.client.HTTPConnection(self.hooks_uri)
55 55 # binary msgpack body
56 56 headers, body = self._serialize(method, extras)
57 57 log.debug('Doing a new hooks call using HTTPConnection to %s', self.hooks_uri)
58 58
59 59 try:
60 60 try:
61 61 connection.request('POST', '/', body, headers)
62 62 except Exception as error:
63 63 log.error('Hooks calling Connection failed on %s, org error: %s', connection.__dict__, error)
64 64 raise
65 65
66 66 response = connection.getresponse()
67 67 try:
68 68 return msgpack.load(response)
69 69 except Exception:
70 70 response_data = response.read()
71 71 log.exception('Failed to decode hook response json data. '
72 72 'response_code:%s, raw_data:%s',
73 73 response.status, response_data)
74 74 raise
75 75 finally:
76 76 connection.close()
77 77
78 78 @classmethod
79 79 def _serialize(cls, hook_name, extras):
80 80 data = {
81 81 'method': hook_name,
82 82 'extras': extras
83 83 }
84 84 headers = {
85 85 "rc-hooks-protocol": cls.proto,
86 86 "Connection": "keep-alive"
87 87 }
88 88 return headers, msgpack.packb(data)
89 89
90 90
91 91 class HooksCeleryClient:
92 92 TASK_TIMEOUT = 60 # time in seconds
93 93
94 94 def __init__(self, queue, backend):
95 95 celery_app.config_from_object({
96 96 'broker_url': queue, 'result_backend': backend,
97 97 'broker_connection_retry_on_startup': True,
98 98 'task_serializer': 'json',
99 99 'accept_content': ['json', 'msgpack'],
100 100 'result_serializer': 'json',
101 101 'result_accept_content': ['json', 'msgpack']
102 102 })
103 103 self.celery_app = celery_app
104 104
105 105 def __call__(self, method, extras):
106 106 inquired_task = self.celery_app.signature(
107 107 f'rhodecode.lib.celerylib.tasks.{method}'
108 108 )
109 109 return inquired_task.delay(extras).get(timeout=self.TASK_TIMEOUT)
110 110
111 111
112 112 class HooksShadowRepoClient:
113 113
114 114 def __call__(self, hook_name, extras):
115 115 return {'output': '', 'status': 0}
116 116
117 117
118 118 class RemoteMessageWriter:
119 119 """Writer base class."""
120 120 def write(self, message):
121 121 raise NotImplementedError()
122 122
123 123
124 124 class HgMessageWriter(RemoteMessageWriter):
125 125 """Writer that knows how to send messages to mercurial clients."""
126 126
127 127 def __init__(self, ui):
128 128 self.ui = ui
129 129
130 130 def write(self, message: str):
131 131 # TODO: Check why the quiet flag is set by default.
132 132 old = self.ui.quiet
133 133 self.ui.quiet = False
134 134 self.ui.status(message.encode('utf-8'))
135 135 self.ui.quiet = old
136 136
137 137
138 138 class GitMessageWriter(RemoteMessageWriter):
139 139 """Writer that knows how to send messages to git clients."""
140 140
141 141 def __init__(self, stdout=None):
142 142 self.stdout = stdout or sys.stdout
143 143
144 144 def write(self, message: str):
145 145 self.stdout.write(message)
146 146
147 147
148 148 class SvnMessageWriter(RemoteMessageWriter):
149 149 """Writer that knows how to send messages to svn clients."""
150 150
151 151 def __init__(self, stderr=None):
152 152 # SVN needs data sent to stderr for back-to-client messaging
153 153 self.stderr = stderr or sys.stderr
154 154
155 155 def write(self, message):
156 156 self.stderr.write(message)
157 157
158 158
159 159 def _handle_exception(result):
160 160 exception_class = result.get('exception')
161 161 exception_traceback = result.get('exception_traceback')
162 162 log.debug('Handling hook-call exception: %s', exception_class)
163 163
164 164 if exception_traceback:
165 165 log.error('Got traceback from remote call:%s', exception_traceback)
166 166
167 167 if exception_class == 'HTTPLockedRC':
168 168 raise exceptions.RepositoryLockedException()(*result['exception_args'])
169 169 elif exception_class == 'HTTPBranchProtected':
170 170 raise exceptions.RepositoryBranchProtectedException()(*result['exception_args'])
171 171 elif exception_class == 'RepositoryError':
172 172 raise exceptions.VcsException()(*result['exception_args'])
173 173 elif exception_class:
174 174 raise Exception(
175 175 f"""Got remote exception "{exception_class}" with args "{result['exception_args']}" """
176 176 )
177 177
178 178
179 179 def _get_hooks_client(extras):
180 180 hooks_uri = extras.get('hooks_uri')
181 181 task_queue = extras.get('task_queue')
182 182 task_backend = extras.get('task_backend')
183 183 is_shadow_repo = extras.get('is_shadow_repo')
184 184
185 185 if hooks_uri:
186 186 return HooksHttpClient(hooks_uri)
187 187 elif task_queue and task_backend:
188 188 return HooksCeleryClient(task_queue, task_backend)
189 189 elif is_shadow_repo:
190 190 return HooksShadowRepoClient()
191 191 else:
192 192 raise Exception("Hooks client not found!")
193 193
194 194
195 195 def _call_hook(hook_name, extras, writer):
196 196 hooks_client = _get_hooks_client(extras)
197 197 log.debug('Hooks, using client:%s', hooks_client)
198 198 result = hooks_client(hook_name, extras)
199 199 log.debug('Hooks got result: %s', result)
200 200 _handle_exception(result)
201 201 writer.write(result['output'])
202 202
203 203 return result['status']
204 204
205 205
206 206 def _extras_from_ui(ui):
207 207 hook_data = ui.config(b'rhodecode', b'RC_SCM_DATA')
208 208 if not hook_data:
209 209 # maybe it's inside environ ?
210 210 env_hook_data = os.environ.get('RC_SCM_DATA')
211 211 if env_hook_data:
212 212 hook_data = env_hook_data
213 213
214 214 extras = {}
215 215 if hook_data:
216 216 extras = json.loads(hook_data)
217 217 return extras
218 218
219 219
220 220 def _rev_range_hash(repo, node, check_heads=False):
221 221 from vcsserver.hgcompat import get_ctx
222 222
223 223 commits = []
224 224 revs = []
225 225 start = get_ctx(repo, node).rev()
226 226 end = len(repo)
227 227 for rev in range(start, end):
228 228 revs.append(rev)
229 229 ctx = get_ctx(repo, rev)
230 230 commit_id = ascii_str(mercurial.node.hex(ctx.node()))
231 231 branch = safe_str(ctx.branch())
232 232 commits.append((commit_id, branch))
233 233
234 234 parent_heads = []
235 235 if check_heads:
236 236 parent_heads = _check_heads(repo, start, end, revs)
237 237 return commits, parent_heads
238 238
239 239
240 240 def _check_heads(repo, start, end, commits):
241 241 from vcsserver.hgcompat import get_ctx
242 242 changelog = repo.changelog
243 243 parents = set()
244 244
245 245 for new_rev in commits:
246 246 for p in changelog.parentrevs(new_rev):
247 247 if p == mercurial.node.nullrev:
248 248 continue
249 249 if p < start:
250 250 parents.add(p)
251 251
252 252 for p in parents:
253 253 branch = get_ctx(repo, p).branch()
254 254 # The heads descending from that parent, on the same branch
255 255 parent_heads = {p}
256 256 reachable = {p}
257 257 for x in range(p + 1, end):
258 258 if get_ctx(repo, x).branch() != branch:
259 259 continue
260 260 for pp in changelog.parentrevs(x):
261 261 if pp in reachable:
262 262 reachable.add(x)
263 263 parent_heads.discard(pp)
264 264 parent_heads.add(x)
265 265 # More than one head? Suggest merging
266 266 if len(parent_heads) > 1:
267 267 return list(parent_heads)
268 268
269 269 return []
270 270
271 271
272 272 def _get_git_env():
273 273 env = {}
274 274 for k, v in os.environ.items():
275 275 if k.startswith('GIT'):
276 276 env[k] = v
277 277
278 278 # serialized version
279 279 return [(k, v) for k, v in env.items()]
280 280
281 281
282 282 def _get_hg_env(old_rev, new_rev, txnid, repo_path):
283 283 env = {}
284 284 for k, v in os.environ.items():
285 285 if k.startswith('HG'):
286 286 env[k] = v
287 287
288 288 env['HG_NODE'] = old_rev
289 289 env['HG_NODE_LAST'] = new_rev
290 290 env['HG_TXNID'] = txnid
291 291 env['HG_PENDING'] = repo_path
292 292
293 293 return [(k, v) for k, v in env.items()]
294 294
295 295
296 296 def _fix_hooks_executables(ini_path=''):
297 297 """
298 298 This is a trick to set proper settings.EXECUTABLE paths for certain execution patterns
299 299 especially for subversion where hooks strip entire env, and calling just 'svn' command will most likely fail
300 300 because svn is not on PATH
301 301 """
302 302 from vcsserver.http_main import sanitize_settings_and_apply_defaults
303 303 from vcsserver.lib.config_utils import get_app_config_lightweight
304 304
305 305 core_binary_dir = settings.BINARY_DIR or '/usr/local/bin/rhodecode_bin/vcs_bin'
306 306 if ini_path:
307 307
308 308 ini_settings = get_app_config_lightweight(ini_path)
309 309 ini_settings = sanitize_settings_and_apply_defaults({'__file__': ini_path}, ini_settings)
310 310 core_binary_dir = ini_settings['core.binary_dir']
311 311
312 312 settings.BINARY_DIR = core_binary_dir
313 313
314 314
315 315 def repo_size(ui, repo, **kwargs):
316 316 extras = _extras_from_ui(ui)
317 317 return _call_hook('repo_size', extras, HgMessageWriter(ui))
318 318
319 319
320 320 def pre_pull(ui, repo, **kwargs):
321 321 extras = _extras_from_ui(ui)
322 322 return _call_hook('pre_pull', extras, HgMessageWriter(ui))
323 323
324 324
325 325 def pre_pull_ssh(ui, repo, **kwargs):
326 326 extras = _extras_from_ui(ui)
327 327 if extras and extras.get('SSH'):
328 328 return pre_pull(ui, repo, **kwargs)
329 329 return 0
330 330
331 331
332 332 def post_pull(ui, repo, **kwargs):
333 333 extras = _extras_from_ui(ui)
334 334 return _call_hook('post_pull', extras, HgMessageWriter(ui))
335 335
336 336
337 337 def post_pull_ssh(ui, repo, **kwargs):
338 338 extras = _extras_from_ui(ui)
339 339 if extras and extras.get('SSH'):
340 340 return post_pull(ui, repo, **kwargs)
341 341 return 0
342 342
343 343
344 344 def pre_push(ui, repo, node=None, **kwargs):
345 345 """
346 346 Mercurial pre_push hook
347 347 """
348 348 extras = _extras_from_ui(ui)
349 349 detect_force_push = extras.get('detect_force_push')
350 350
351 351 rev_data = []
352 352 hook_type: str = safe_str(kwargs.get('hooktype'))
353 353
354 354 if node and hook_type == 'pretxnchangegroup':
355 355 branches = collections.defaultdict(list)
356 356 commits, _heads = _rev_range_hash(repo, node, check_heads=detect_force_push)
357 357 for commit_id, branch in commits:
358 358 branches[branch].append(commit_id)
359 359
360 360 for branch, commits in branches.items():
361 361 old_rev = ascii_str(kwargs.get('node_last')) or commits[0]
362 362 rev_data.append({
363 363 'total_commits': len(commits),
364 364 'old_rev': old_rev,
365 365 'new_rev': commits[-1],
366 366 'ref': '',
367 367 'type': 'branch',
368 368 'name': branch,
369 369 })
370 370
371 371 for push_ref in rev_data:
372 372 push_ref['multiple_heads'] = _heads
373 373
374 374 repo_path = os.path.join(
375 375 extras.get('repo_store', ''), extras.get('repository', ''))
376 376 push_ref['hg_env'] = _get_hg_env(
377 377 old_rev=push_ref['old_rev'],
378 378 new_rev=push_ref['new_rev'], txnid=ascii_str(kwargs.get('txnid')),
379 379 repo_path=repo_path)
380 380
381 381 extras['hook_type'] = hook_type or 'pre_push'
382 382 extras['commit_ids'] = rev_data
383 383
384 384 return _call_hook('pre_push', extras, HgMessageWriter(ui))
385 385
386 386
387 387 def pre_push_ssh(ui, repo, node=None, **kwargs):
388 388 extras = _extras_from_ui(ui)
389 389 if extras.get('SSH'):
390 390 return pre_push(ui, repo, node, **kwargs)
391 391
392 392 return 0
393 393
394 394
395 395 def pre_push_ssh_auth(ui, repo, node=None, **kwargs):
396 396 """
397 397 Mercurial pre_push hook for SSH
398 398 """
399 399 extras = _extras_from_ui(ui)
400 400 if extras.get('SSH'):
401 401 permission = extras['SSH_PERMISSIONS']
402 402
403 403 if 'repository.write' == permission or 'repository.admin' == permission:
404 404 return 0
405 405
406 406 # non-zero ret code
407 407 return 1
408 408
409 409 return 0
410 410
411 411
412 412 def post_push(ui, repo, node, **kwargs):
413 413 """
414 414 Mercurial post_push hook
415 415 """
416 416 extras = _extras_from_ui(ui)
417 417
418 418 commit_ids = []
419 419 branches = []
420 420 bookmarks = []
421 421 tags = []
422 422 hook_type: str = safe_str(kwargs.get('hooktype'))
423 423
424 424 commits, _heads = _rev_range_hash(repo, node)
425 425 for commit_id, branch in commits:
426 426 commit_ids.append(commit_id)
427 427 if branch not in branches:
428 428 branches.append(branch)
429 429
430 430 if hasattr(ui, '_rc_pushkey_bookmarks'):
431 431 bookmarks = ui._rc_pushkey_bookmarks
432 432
433 433 extras['hook_type'] = hook_type or 'post_push'
434 434 extras['commit_ids'] = commit_ids
435 435
436 436 extras['new_refs'] = {
437 437 'branches': branches,
438 438 'bookmarks': bookmarks,
439 439 'tags': tags
440 440 }
441 441
442 442 return _call_hook('post_push', extras, HgMessageWriter(ui))
443 443
444 444
445 445 def post_push_ssh(ui, repo, node, **kwargs):
446 446 """
447 447 Mercurial post_push hook for SSH
448 448 """
449 449 if _extras_from_ui(ui).get('SSH'):
450 450 return post_push(ui, repo, node, **kwargs)
451 451 return 0
452 452
453 453
454 454 def key_push(ui, repo, **kwargs):
455 455 from vcsserver.hgcompat import get_ctx
456 456
457 457 if kwargs['new'] != b'0' and kwargs['namespace'] == b'bookmarks':
458 458 # store new bookmarks in our UI object propagated later to post_push
459 459 ui._rc_pushkey_bookmarks = get_ctx(repo, kwargs['key']).bookmarks()
460 460 return
461 461
462 462
463 463 # backward compat
464 464 log_pull_action = post_pull
465 465
466 466 # backward compat
467 467 log_push_action = post_push
468 468
469 469
470 470 def handle_git_pre_receive(unused_repo_path, unused_revs, unused_env):
471 471 """
472 472 Old hook name: keep here for backward compatibility.
473 473
474 474 This is only required when the installed git hooks are not upgraded.
475 475 """
476 476 pass
477 477
478 478
479 479 def handle_git_post_receive(unused_repo_path, unused_revs, unused_env):
480 480 """
481 481 Old hook name: keep here for backward compatibility.
482 482
483 483 This is only required when the installed git hooks are not upgraded.
484 484 """
485 485 pass
486 486
487 487
488 488 @dataclasses.dataclass
489 489 class HookResponse:
490 490 status: int
491 491 output: str
492 492
493 493
494 494 def git_pre_pull(extras) -> HookResponse:
495 495 """
496 496 Pre pull hook.
497 497
498 498 :param extras: dictionary containing the keys defined in simplevcs
499 499 :type extras: dict
500 500
501 501 :return: status code of the hook. 0 for success.
502 502 :rtype: int
503 503 """
504 504
505 505 if 'pull' not in extras['hooks']:
506 506 return HookResponse(0, '')
507 507
508 508 stdout = io.StringIO()
509 509 try:
510 510 status_code = _call_hook('pre_pull', extras, GitMessageWriter(stdout))
511 511
512 512 except Exception as error:
513 513 log.exception('Failed to call pre_pull hook')
514 514 status_code = 128
515 515 stdout.write(f'ERROR: {error}\n')
516 516
517 517 return HookResponse(status_code, stdout.getvalue())
518 518
519 519
520 520 def git_post_pull(extras) -> HookResponse:
521 521 """
522 522 Post pull hook.
523 523
524 524 :param extras: dictionary containing the keys defined in simplevcs
525 525 :type extras: dict
526 526
527 527 :return: status code of the hook. 0 for success.
528 528 :rtype: int
529 529 """
530 530 if 'pull' not in extras['hooks']:
531 531 return HookResponse(0, '')
532 532
533 533 stdout = io.StringIO()
534 534 try:
535 535 status = _call_hook('post_pull', extras, GitMessageWriter(stdout))
536 536 except Exception as error:
537 537 status = 128
538 538 stdout.write(f'ERROR: {error}\n')
539 539
540 540 return HookResponse(status, stdout.getvalue())
541 541
542 542
543 543 def _parse_git_ref_lines(revision_lines):
544 544 rev_data = []
545 545 for revision_line in revision_lines or []:
546 546 old_rev, new_rev, ref = revision_line.strip().split(' ')
547 547 ref_data = ref.split('/', 2)
548 548 if ref_data[1] in ('tags', 'heads'):
549 549 rev_data.append({
550 550 # NOTE(marcink):
551 551 # we're unable to tell total_commits for git at this point
552 552 # but we set the variable for consistency with GIT
553 553 'total_commits': -1,
554 554 'old_rev': old_rev,
555 555 'new_rev': new_rev,
556 556 'ref': ref,
557 557 'type': ref_data[1],
558 558 'name': ref_data[2],
559 559 })
560 560 return rev_data
561 561
562 562
563 563 def git_pre_receive(unused_repo_path, revision_lines, env) -> int:
564 564 """
565 565 Pre push hook.
566 566
567 567 :return: status code of the hook. 0 for success.
568 568 """
569 569 extras = json.loads(env['RC_SCM_DATA'])
570 570 rev_data = _parse_git_ref_lines(revision_lines)
571 571 if 'push' not in extras['hooks']:
572 572 return 0
573 573 _fix_hooks_executables()
574 574
575 575 empty_commit_id = '0' * 40
576 576
577 577 detect_force_push = extras.get('detect_force_push')
578 578
579 579 for push_ref in rev_data:
580 580 # store our git-env which holds the temp store
581 581 push_ref['git_env'] = _get_git_env()
582 582 push_ref['pruned_sha'] = ''
583 583 if not detect_force_push:
584 584 # don't check for forced-push when we don't need to
585 585 continue
586 586
587 587 type_ = push_ref['type']
588 588 new_branch = push_ref['old_rev'] == empty_commit_id
589 589 delete_branch = push_ref['new_rev'] == empty_commit_id
590 590 if type_ == 'heads' and not (new_branch or delete_branch):
591 591 old_rev = push_ref['old_rev']
592 592 new_rev = push_ref['new_rev']
593 593 cmd = [settings.GIT_EXECUTABLE(), 'rev-list', old_rev, f'^{new_rev}']
594 594 stdout, stderr = subprocessio.run_command(
595 595 cmd, env=os.environ.copy())
596 596 # means we're having some non-reachable objects, this forced push was used
597 597 if stdout:
598 598 push_ref['pruned_sha'] = stdout.splitlines()
599 599
600 600 extras['hook_type'] = 'pre_receive'
601 601 extras['commit_ids'] = rev_data
602 602
603 603 stdout = sys.stdout
604 604 status_code = _call_hook('pre_push', extras, GitMessageWriter(stdout))
605 605
606 606 return status_code
607 607
608 608
609 609 def git_post_receive(unused_repo_path, revision_lines, env) -> int:
610 610 """
611 611 Post push hook.
612 612
613 613 :return: status code of the hook. 0 for success.
614 614 """
615 615 extras = json.loads(env['RC_SCM_DATA'])
616 616 if 'push' not in extras['hooks']:
617 617 return 0
618 618
619 619 _fix_hooks_executables()
620 620
621 621 rev_data = _parse_git_ref_lines(revision_lines)
622 622
623 623 git_revs = []
624 624
625 625 # N.B.(skreft): it is ok to just call git, as git before calling a
626 626 # subcommand sets the PATH environment variable so that it point to the
627 627 # correct version of the git executable.
628 628 empty_commit_id = '0' * 40
629 629 branches = []
630 630 tags = []
631 631 for push_ref in rev_data:
632 632 type_ = push_ref['type']
633 633
634 634 if type_ == 'heads':
635 635 # starting new branch case
636 636 if push_ref['old_rev'] == empty_commit_id:
637 637 push_ref_name = push_ref['name']
638 638
639 639 if push_ref_name not in branches:
640 640 branches.append(push_ref_name)
641 641
642 642 need_head_set = ''
643 643 with Repository(os.getcwd()) as repo:
644 644 try:
645 645 repo.head
646 646 except pygit2.GitError:
647 647 need_head_set = f'refs/heads/{push_ref_name}'
648 648
649 649 if need_head_set:
650 650 repo.set_head(need_head_set)
651 651 print(f"Setting default branch to {push_ref_name}")
652 652
653 653 cmd = [settings.GIT_EXECUTABLE(), 'for-each-ref', '--format=%(refname)', 'refs/heads/*']
654 654 stdout, stderr = subprocessio.run_command(
655 655 cmd, env=os.environ.copy())
656 656 heads = safe_str(stdout)
657 657 heads = heads.replace(push_ref['ref'], '')
658 658 heads = ' '.join(head for head
659 659 in heads.splitlines() if head) or '.'
660 660 cmd = [settings.GIT_EXECUTABLE(), 'log', '--reverse',
661 661 '--pretty=format:%H', '--', push_ref['new_rev'],
662 662 '--not', heads]
663 663 stdout, stderr = subprocessio.run_command(
664 664 cmd, env=os.environ.copy())
665 665 git_revs.extend(list(map(ascii_str, stdout.splitlines())))
666 666
667 667 # delete branch case
668 668 elif push_ref['new_rev'] == empty_commit_id:
669 669 git_revs.append(f'delete_branch=>{push_ref["name"]}')
670 670 else:
671 671 if push_ref['name'] not in branches:
672 672 branches.append(push_ref['name'])
673 673
674 674 cmd = [settings.GIT_EXECUTABLE(), 'log',
675 675 f'{push_ref["old_rev"]}..{push_ref["new_rev"]}',
676 676 '--reverse', '--pretty=format:%H']
677 677 stdout, stderr = subprocessio.run_command(
678 678 cmd, env=os.environ.copy())
679 679 # we get bytes from stdout, we need str to be consistent
680 680 log_revs = list(map(ascii_str, stdout.splitlines()))
681 681 git_revs.extend(log_revs)
682 682
683 683 # Pure pygit2 impl. but still 2-3x slower :/
684 684 # results = []
685 685 #
686 686 # with Repository(os.getcwd()) as repo:
687 687 # repo_new_rev = repo[push_ref['new_rev']]
688 688 # repo_old_rev = repo[push_ref['old_rev']]
689 689 # walker = repo.walk(repo_new_rev.id, pygit2.GIT_SORT_TOPOLOGICAL)
690 690 #
691 691 # for commit in walker:
692 692 # if commit.id == repo_old_rev.id:
693 693 # break
694 694 # results.append(commit.id.hex)
695 695 # # reverse the order, can't use GIT_SORT_REVERSE
696 696 # log_revs = results[::-1]
697 697
698 698 elif type_ == 'tags':
699 699 if push_ref['name'] not in tags:
700 700 tags.append(push_ref['name'])
701 701 git_revs.append(f'tag=>{push_ref["name"]}')
702 702
703 703 extras['hook_type'] = 'post_receive'
704 704 extras['commit_ids'] = git_revs
705 705 extras['new_refs'] = {
706 706 'branches': branches,
707 707 'bookmarks': [],
708 708 'tags': tags,
709 709 }
710 710
711 711 stdout = sys.stdout
712 712
713 713 if 'repo_size' in extras['hooks']:
714 714 try:
715 715 _call_hook('repo_size', extras, GitMessageWriter(stdout))
716 716 except Exception:
717 717 pass
718 718
719 719 status_code = _call_hook('post_push', extras, GitMessageWriter(stdout))
720 720 return status_code
721 721
722 722
723 723 def _get_extras_from_txn_id(path, txn_id):
724 724 _fix_hooks_executables()
725 725
726 726 extras = {}
727 727 try:
728 728 cmd = [settings.SVNLOOK_EXECUTABLE(), 'pget',
729 729 '-t', txn_id,
730 730 '--revprop', path, 'rc-scm-extras']
731 731 stdout, stderr = subprocessio.run_command(
732 732 cmd, env=os.environ.copy())
733 733 extras = json.loads(base64.urlsafe_b64decode(stdout))
734 734 except Exception:
735 735 log.exception('Failed to extract extras info from txn_id')
736 736
737 737 return extras
738 738
739 739
740 740 def _get_extras_from_commit_id(commit_id, path):
741 741 _fix_hooks_executables()
742 742
743 743 extras = {}
744 744 try:
745 745 cmd = [settings.SVNLOOK_EXECUTABLE(), 'pget',
746 746 '-r', commit_id,
747 747 '--revprop', path, 'rc-scm-extras']
748 748 stdout, stderr = subprocessio.run_command(
749 749 cmd, env=os.environ.copy())
750 750 extras = json.loads(base64.urlsafe_b64decode(stdout))
751 751 except Exception:
752 752 log.exception('Failed to extract extras info from commit_id')
753 753
754 754 return extras
755 755
756 756
757 757 def svn_pre_commit(repo_path, commit_data, env):
758 758
759 759 path, txn_id = commit_data
760 760 branches = []
761 761 tags = []
762 762
763 763 if env.get('RC_SCM_DATA'):
764 764 extras = json.loads(env['RC_SCM_DATA'])
765 765 else:
766 766 # fallback method to read from TXN-ID stored data
767 767 extras = _get_extras_from_txn_id(path, txn_id)
768 768
769 769 if not extras:
770 770 #TODO: temporary fix until svn txn-id changes are merged
771 771 return 0
772 772 raise ValueError('Failed to extract context data called extras for hook execution')
773 773
774 774 extras['hook_type'] = 'pre_commit'
775 775 extras['commit_ids'] = [txn_id]
776 776 extras['txn_id'] = txn_id
777 777 extras['new_refs'] = {
778 778 'total_commits': 1,
779 779 'branches': branches,
780 780 'bookmarks': [],
781 781 'tags': tags,
782 782 }
783 783
784 784 return _call_hook('pre_push', extras, SvnMessageWriter())
785 785
786 786
787 787 def svn_post_commit(repo_path, commit_data, env):
788 788 """
789 789 commit_data is path, rev, txn_id
790 790 """
791 791
792 792 if len(commit_data) == 3:
793 793 path, commit_id, txn_id = commit_data
794 794 elif len(commit_data) == 2:
795 795 log.error('Failed to extract txn_id from commit_data using legacy method. '
796 796 'Some functionality might be limited')
797 797 path, commit_id = commit_data
798 798 txn_id = None
799 799 else:
800 800 return 0
801 801
802 802 branches = []
803 803 tags = []
804 804
805 805 if env.get('RC_SCM_DATA'):
806 806 extras = json.loads(env['RC_SCM_DATA'])
807 807 else:
808 808 # fallback method to read from TXN-ID stored data
809 809 extras = _get_extras_from_commit_id(commit_id, path)
810 810
811 811 if not extras:
812 812 #TODO: temporary fix until svn txn-id changes are merged
813 813 return 0
814 814 raise ValueError('Failed to extract context data called extras for hook execution')
815 815
816 816 extras['hook_type'] = 'post_commit'
817 817 extras['commit_ids'] = [commit_id]
818 818 extras['txn_id'] = txn_id
819 819 extras['new_refs'] = {
820 820 'branches': branches,
821 821 'bookmarks': [],
822 822 'tags': tags,
823 823 'total_commits': 1,
824 824 }
825 825
826 826 if 'repo_size' in extras['hooks']:
827 827 try:
828 828 _call_hook('repo_size', extras, SvnMessageWriter())
829 829 except Exception:
830 830 pass
831 831
832 832 return _call_hook('post_push', extras, SvnMessageWriter())
@@ -1,774 +1,774 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import io
19 19 import os
20 20 import platform
21 21 import sys
22 22 import locale
23 23 import logging
24 24 import uuid
25 25 import time
26 26 import wsgiref.util
27 27 import tempfile
28 28 import psutil
29 29
30 30 from itertools import chain
31 31
32 32 import msgpack
33 33 import configparser
34 34
35 35 from pyramid.config import Configurator
36 36 from pyramid.wsgi import wsgiapp
37 37 from pyramid.response import Response
38 38
39 39 from vcsserver.base import BytesEnvelope, BinaryEnvelope
40 from vcsserver.lib.rc_json import json
40 from vcsserver.lib.ext_json import json
41 41 from vcsserver.config.settings_maker import SettingsMaker
42 42 from vcsserver.str_utils import safe_int
43 43 from vcsserver.lib.statsd_client import StatsdClient
44 44 from vcsserver.tweens.request_wrapper import get_headers_call_context
45 45
46 46 import vcsserver
47 47 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
48 48 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
49 49 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
50 50 from vcsserver.echo_stub.echo_app import EchoApp
51 51 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
52 52 from vcsserver.lib.exc_tracking import store_exception, format_exc
53 53 from vcsserver.server import VcsServer
54 54
55 55 strict_vcs = True
56 56
57 57 git_import_err = None
58 58 try:
59 59 from vcsserver.remote.git_remote import GitFactory, GitRemote
60 60 except ImportError as e:
61 61 GitFactory = None
62 62 GitRemote = None
63 63 git_import_err = e
64 64 if strict_vcs:
65 65 raise
66 66
67 67
68 68 hg_import_err = None
69 69 try:
70 70 from vcsserver.remote.hg_remote import MercurialFactory, HgRemote
71 71 except ImportError as e:
72 72 MercurialFactory = None
73 73 HgRemote = None
74 74 hg_import_err = e
75 75 if strict_vcs:
76 76 raise
77 77
78 78
79 79 svn_import_err = None
80 80 try:
81 81 from vcsserver.remote.svn_remote import SubversionFactory, SvnRemote
82 82 except ImportError as e:
83 83 SubversionFactory = None
84 84 SvnRemote = None
85 85 svn_import_err = e
86 86 if strict_vcs:
87 87 raise
88 88
89 89 log = logging.getLogger(__name__)
90 90
91 91 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
92 92 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
93 93
94 94 try:
95 95 locale.setlocale(locale.LC_ALL, '')
96 96 except locale.Error as e:
97 97 log.error(
98 98 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
99 99 os.environ['LC_ALL'] = 'C'
100 100
101 101
102 102 def _is_request_chunked(environ):
103 103 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
104 104 return stream
105 105
106 106
107 107 def log_max_fd():
108 108 try:
109 109 maxfd = psutil.Process().rlimit(psutil.RLIMIT_NOFILE)[1]
110 110 log.info('Max file descriptors value: %s', maxfd)
111 111 except Exception:
112 112 pass
113 113
114 114
115 115 class VCS:
116 116 def __init__(self, locale_conf=None, cache_config=None):
117 117 self.locale = locale_conf
118 118 self.cache_config = cache_config
119 119 self._configure_locale()
120 120
121 121 log_max_fd()
122 122
123 123 if GitFactory and GitRemote:
124 124 git_factory = GitFactory()
125 125 self._git_remote = GitRemote(git_factory)
126 126 else:
127 127 log.error("Git client import failed: %s", git_import_err)
128 128
129 129 if MercurialFactory and HgRemote:
130 130 hg_factory = MercurialFactory()
131 131 self._hg_remote = HgRemote(hg_factory)
132 132 else:
133 133 log.error("Mercurial client import failed: %s", hg_import_err)
134 134
135 135 if SubversionFactory and SvnRemote:
136 136 svn_factory = SubversionFactory()
137 137
138 138 # hg factory is used for svn url validation
139 139 hg_factory = MercurialFactory()
140 140 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
141 141 else:
142 142 log.error("Subversion client import failed: %s", svn_import_err)
143 143
144 144 self._vcsserver = VcsServer()
145 145
146 146 def _configure_locale(self):
147 147 if self.locale:
148 148 log.info('Settings locale: `LC_ALL` to %s', self.locale)
149 149 else:
150 150 log.info('Configuring locale subsystem based on environment variables')
151 151 try:
152 152 # If self.locale is the empty string, then the locale
153 153 # module will use the environment variables. See the
154 154 # documentation of the package `locale`.
155 155 locale.setlocale(locale.LC_ALL, self.locale)
156 156
157 157 language_code, encoding = locale.getlocale()
158 158 log.info(
159 159 'Locale set to language code "%s" with encoding "%s".',
160 160 language_code, encoding)
161 161 except locale.Error:
162 162 log.exception('Cannot set locale, not configuring the locale system')
163 163
164 164
165 165 class WsgiProxy:
166 166 def __init__(self, wsgi):
167 167 self.wsgi = wsgi
168 168
169 169 def __call__(self, environ, start_response):
170 170 input_data = environ['wsgi.input'].read()
171 171 input_data = msgpack.unpackb(input_data)
172 172
173 173 error = None
174 174 try:
175 175 data, status, headers = self.wsgi.handle(
176 176 input_data['environment'], input_data['input_data'],
177 177 *input_data['args'], **input_data['kwargs'])
178 178 except Exception as e:
179 179 data, status, headers = [], None, None
180 180 error = {
181 181 'message': str(e),
182 182 '_vcs_kind': getattr(e, '_vcs_kind', None)
183 183 }
184 184
185 185 start_response(200, {})
186 186 return self._iterator(error, status, headers, data)
187 187
188 188 def _iterator(self, error, status, headers, data):
189 189 initial_data = [
190 190 error,
191 191 status,
192 192 headers,
193 193 ]
194 194
195 195 for d in chain(initial_data, data):
196 196 yield msgpack.packb(d)
197 197
198 198
199 199 def not_found(request):
200 200 return {'status': '404 NOT FOUND'}
201 201
202 202
203 203 class VCSViewPredicate:
204 204 def __init__(self, val, config):
205 205 self.remotes = val
206 206
207 207 def text(self):
208 208 return f'vcs view method = {list(self.remotes.keys())}'
209 209
210 210 phash = text
211 211
212 212 def __call__(self, context, request):
213 213 """
214 214 View predicate that returns true if given backend is supported by
215 215 defined remotes.
216 216 """
217 217 backend = request.matchdict.get('backend')
218 218 return backend in self.remotes
219 219
220 220
221 221 class HTTPApplication:
222 222 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
223 223
224 224 remote_wsgi = remote_wsgi
225 225 _use_echo_app = False
226 226
227 227 def __init__(self, settings=None, global_config=None):
228 228
229 229 self.config = Configurator(settings=settings)
230 230 # Init our statsd at very start
231 231 self.config.registry.statsd = StatsdClient.statsd
232 232 self.config.registry.vcs_call_context = {}
233 233
234 234 self.global_config = global_config
235 235 self.config.include('vcsserver.lib.rc_cache')
236 236 self.config.include('vcsserver.lib.rc_cache.archive_cache')
237 237
238 238 settings_locale = settings.get('locale', '') or 'en_US.UTF-8'
239 239 vcs = VCS(locale_conf=settings_locale, cache_config=settings)
240 240 self._remotes = {
241 241 'hg': vcs._hg_remote,
242 242 'git': vcs._git_remote,
243 243 'svn': vcs._svn_remote,
244 244 'server': vcs._vcsserver,
245 245 }
246 246 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
247 247 self._use_echo_app = True
248 248 log.warning("Using EchoApp for VCS operations.")
249 249 self.remote_wsgi = remote_wsgi_stub
250 250
251 251 self._configure_settings(global_config, settings)
252 252
253 253 self._configure()
254 254
255 255 def _configure_settings(self, global_config, app_settings):
256 256 """
257 257 Configure the settings module.
258 258 """
259 259 settings_merged = global_config.copy()
260 260 settings_merged.update(app_settings)
261 261
262 262 binary_dir = app_settings['core.binary_dir']
263 263
264 264 settings.BINARY_DIR = binary_dir
265 265
266 266 # Store the settings to make them available to other modules.
267 267 vcsserver.PYRAMID_SETTINGS = settings_merged
268 268 vcsserver.CONFIG = settings_merged
269 269
270 270 def _configure(self):
271 271 self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
272 272
273 273 self.config.add_route('service', '/_service')
274 274 self.config.add_route('status', '/status')
275 275 self.config.add_route('hg_proxy', '/proxy/hg')
276 276 self.config.add_route('git_proxy', '/proxy/git')
277 277
278 278 # rpc methods
279 279 self.config.add_route('vcs', '/{backend}')
280 280
281 281 # streaming rpc remote methods
282 282 self.config.add_route('vcs_stream', '/{backend}/stream')
283 283
284 284 # vcs operations clone/push as streaming
285 285 self.config.add_route('stream_git', '/stream/git/*repo_name')
286 286 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
287 287
288 288 self.config.add_view(self.status_view, route_name='status', renderer='json')
289 289 self.config.add_view(self.service_view, route_name='service', renderer='msgpack')
290 290
291 291 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
292 292 self.config.add_view(self.git_proxy(), route_name='git_proxy')
293 293 self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack',
294 294 vcs_view=self._remotes)
295 295 self.config.add_view(self.vcs_stream_view, route_name='vcs_stream',
296 296 vcs_view=self._remotes)
297 297
298 298 self.config.add_view(self.hg_stream(), route_name='stream_hg')
299 299 self.config.add_view(self.git_stream(), route_name='stream_git')
300 300
301 301 self.config.add_view_predicate('vcs_view', VCSViewPredicate)
302 302
303 303 self.config.add_notfound_view(not_found, renderer='json')
304 304
305 305 self.config.add_view(self.handle_vcs_exception, context=Exception)
306 306
307 307 self.config.add_tween(
308 308 'vcsserver.tweens.request_wrapper.RequestWrapperTween',
309 309 )
310 310 self.config.add_request_method(
311 311 'vcsserver.lib.request_counter.get_request_counter',
312 312 'request_count')
313 313
314 314 def wsgi_app(self):
315 315 return self.config.make_wsgi_app()
316 316
317 317 def _vcs_view_params(self, request):
318 318 remote = self._remotes[request.matchdict['backend']]
319 319 payload = msgpack.unpackb(request.body, use_list=True)
320 320
321 321 method = payload.get('method')
322 322 params = payload['params']
323 323 wire = params.get('wire')
324 324 args = params.get('args')
325 325 kwargs = params.get('kwargs')
326 326 context_uid = None
327 327
328 328 request.registry.vcs_call_context = {
329 329 'method': method,
330 330 'repo_name': payload.get('_repo_name'),
331 331 }
332 332
333 333 if wire:
334 334 try:
335 335 wire['context'] = context_uid = uuid.UUID(wire['context'])
336 336 except KeyError:
337 337 pass
338 338 args.insert(0, wire)
339 339 repo_state_uid = wire.get('repo_state_uid') if wire else None
340 340
341 341 # NOTE(marcink): trading complexity for slight performance
342 342 if log.isEnabledFor(logging.DEBUG):
343 343 # also we SKIP printing out any of those methods args since they maybe excessive
344 344 just_args_methods = {
345 345 'commitctx': ('content', 'removed', 'updated'),
346 346 'commit': ('content', 'removed', 'updated')
347 347 }
348 348 if method in just_args_methods:
349 349 skip_args = just_args_methods[method]
350 350 call_args = ''
351 351 call_kwargs = {}
352 352 for k in kwargs:
353 353 if k in skip_args:
354 354 # replace our skip key with dummy
355 355 call_kwargs[k] = f'RemovedParam({k})'
356 356 else:
357 357 call_kwargs[k] = kwargs[k]
358 358 else:
359 359 call_args = args[1:]
360 360 call_kwargs = kwargs
361 361
362 362 log.debug('Method requested:`%s` with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
363 363 method, call_args, call_kwargs, context_uid, repo_state_uid)
364 364
365 365 statsd = request.registry.statsd
366 366 if statsd:
367 367 statsd.incr(
368 368 'vcsserver_method_total', tags=[
369 369 f"method:{method}",
370 370 ])
371 371 return payload, remote, method, args, kwargs
372 372
373 373 def vcs_view(self, request):
374 374
375 375 payload, remote, method, args, kwargs = self._vcs_view_params(request)
376 376 payload_id = payload.get('id')
377 377
378 378 try:
379 379 resp = getattr(remote, method)(*args, **kwargs)
380 380 except Exception as e:
381 381 exc_info = list(sys.exc_info())
382 382 exc_type, exc_value, exc_traceback = exc_info
383 383
384 384 org_exc = getattr(e, '_org_exc', None)
385 385 org_exc_name = None
386 386 org_exc_tb = ''
387 387 if org_exc:
388 388 org_exc_name = org_exc.__class__.__name__
389 389 org_exc_tb = getattr(e, '_org_exc_tb', '')
390 390 # replace our "faked" exception with our org
391 391 exc_info[0] = org_exc.__class__
392 392 exc_info[1] = org_exc
393 393
394 394 should_store_exc = True
395 395 if org_exc:
396 396 def get_exc_fqn(_exc_obj):
397 397 module_name = getattr(org_exc.__class__, '__module__', 'UNKNOWN')
398 398 return module_name + '.' + org_exc_name
399 399
400 400 exc_fqn = get_exc_fqn(org_exc)
401 401
402 402 if exc_fqn in ['mercurial.error.RepoLookupError',
403 403 'vcsserver.exceptions.RefNotFoundException']:
404 404 should_store_exc = False
405 405
406 406 if should_store_exc:
407 407 store_exception(id(exc_info), exc_info, request_path=request.path)
408 408
409 409 tb_info = format_exc(exc_info)
410 410
411 411 type_ = e.__class__.__name__
412 412 if type_ not in self.ALLOWED_EXCEPTIONS:
413 413 type_ = None
414 414
415 415 resp = {
416 416 'id': payload_id,
417 417 'error': {
418 418 'message': str(e),
419 419 'traceback': tb_info,
420 420 'org_exc': org_exc_name,
421 421 'org_exc_tb': org_exc_tb,
422 422 'type': type_
423 423 }
424 424 }
425 425
426 426 try:
427 427 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
428 428 except AttributeError:
429 429 pass
430 430 else:
431 431 resp = {
432 432 'id': payload_id,
433 433 'result': resp
434 434 }
435 435 log.debug('Serving data for method %s', method)
436 436 return resp
437 437
438 438 def vcs_stream_view(self, request):
439 439 payload, remote, method, args, kwargs = self._vcs_view_params(request)
440 440 # this method has a stream: marker we remove it here
441 441 method = method.split('stream:')[-1]
442 442 chunk_size = safe_int(payload.get('chunk_size')) or 4096
443 443
444 444 resp = getattr(remote, method)(*args, **kwargs)
445 445
446 446 def get_chunked_data(method_resp):
447 447 stream = io.BytesIO(method_resp)
448 448 while 1:
449 449 chunk = stream.read(chunk_size)
450 450 if not chunk:
451 451 break
452 452 yield chunk
453 453
454 454 response = Response(app_iter=get_chunked_data(resp))
455 455 response.content_type = 'application/octet-stream'
456 456
457 457 return response
458 458
459 459 def status_view(self, request):
460 460 import vcsserver
461 461 _platform_id = platform.uname()[1] or 'instance'
462 462
463 463 return {
464 464 "status": "OK",
465 465 "vcsserver_version": vcsserver.get_version(),
466 466 "platform": _platform_id,
467 467 "pid": os.getpid(),
468 468 }
469 469
470 470 def service_view(self, request):
471 471 import vcsserver
472 472
473 473 payload = msgpack.unpackb(request.body, use_list=True)
474 474 server_config, app_config = {}, {}
475 475
476 476 try:
477 477 path = self.global_config['__file__']
478 478 config = configparser.RawConfigParser()
479 479
480 480 config.read(path)
481 481
482 482 if config.has_section('server:main'):
483 483 server_config = dict(config.items('server:main'))
484 484 if config.has_section('app:main'):
485 485 app_config = dict(config.items('app:main'))
486 486
487 487 except Exception:
488 488 log.exception('Failed to read .ini file for display')
489 489
490 490 environ = list(os.environ.items())
491 491
492 492 resp = {
493 493 'id': payload.get('id'),
494 494 'result': dict(
495 495 version=vcsserver.get_version(),
496 496 config=server_config,
497 497 app_config=app_config,
498 498 environ=environ,
499 499 payload=payload,
500 500 )
501 501 }
502 502 return resp
503 503
504 504 def _msgpack_renderer_factory(self, info):
505 505
506 506 def _render(value, system):
507 507 bin_type = False
508 508 res = value.get('result')
509 509 if isinstance(res, BytesEnvelope):
510 510 log.debug('Result is wrapped in BytesEnvelope type')
511 511 bin_type = True
512 512 elif isinstance(res, BinaryEnvelope):
513 513 log.debug('Result is wrapped in BinaryEnvelope type')
514 514 value['result'] = res.val
515 515 bin_type = True
516 516
517 517 request = system.get('request')
518 518 if request is not None:
519 519 response = request.response
520 520 ct = response.content_type
521 521 if ct == response.default_content_type:
522 522 response.content_type = 'application/x-msgpack'
523 523 if bin_type:
524 524 response.content_type = 'application/x-msgpack-bin'
525 525
526 526 return msgpack.packb(value, use_bin_type=bin_type)
527 527 return _render
528 528
529 529 def set_env_from_config(self, environ, config):
530 530 dict_conf = {}
531 531 try:
532 532 for elem in config:
533 533 if elem[0] == 'rhodecode':
534 534 dict_conf = json.loads(elem[2])
535 535 break
536 536 except Exception:
537 537 log.exception('Failed to fetch SCM CONFIG')
538 538 return
539 539
540 540 username = dict_conf.get('username')
541 541 if username:
542 542 environ['REMOTE_USER'] = username
543 543 # mercurial specific, some extension api rely on this
544 544 environ['HGUSER'] = username
545 545
546 546 ip = dict_conf.get('ip')
547 547 if ip:
548 548 environ['REMOTE_HOST'] = ip
549 549
550 550 if _is_request_chunked(environ):
551 551 # set the compatibility flag for webob
552 552 environ['wsgi.input_terminated'] = True
553 553
554 554 def hg_proxy(self):
555 555 @wsgiapp
556 556 def _hg_proxy(environ, start_response):
557 557 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
558 558 return app(environ, start_response)
559 559 return _hg_proxy
560 560
561 561 def git_proxy(self):
562 562 @wsgiapp
563 563 def _git_proxy(environ, start_response):
564 564 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
565 565 return app(environ, start_response)
566 566 return _git_proxy
567 567
568 568 def hg_stream(self):
569 569 if self._use_echo_app:
570 570 @wsgiapp
571 571 def _hg_stream(environ, start_response):
572 572 app = EchoApp('fake_path', 'fake_name', None)
573 573 return app(environ, start_response)
574 574 return _hg_stream
575 575 else:
576 576 @wsgiapp
577 577 def _hg_stream(environ, start_response):
578 578 log.debug('http-app: handling hg stream')
579 579 call_context = get_headers_call_context(environ)
580 580
581 581 repo_path = call_context['repo_path']
582 582 repo_name = call_context['repo_name']
583 583 config = call_context['repo_config']
584 584
585 585 app = scm_app.create_hg_wsgi_app(
586 586 repo_path, repo_name, config)
587 587
588 588 # Consistent path information for hgweb
589 589 environ['PATH_INFO'] = call_context['path_info']
590 590 environ['REPO_NAME'] = repo_name
591 591 self.set_env_from_config(environ, config)
592 592
593 593 log.debug('http-app: starting app handler '
594 594 'with %s and process request', app)
595 595 return app(environ, ResponseFilter(start_response))
596 596 return _hg_stream
597 597
598 598 def git_stream(self):
599 599 if self._use_echo_app:
600 600 @wsgiapp
601 601 def _git_stream(environ, start_response):
602 602 app = EchoApp('fake_path', 'fake_name', None)
603 603 return app(environ, start_response)
604 604 return _git_stream
605 605 else:
606 606 @wsgiapp
607 607 def _git_stream(environ, start_response):
608 608 log.debug('http-app: handling git stream')
609 609
610 610 call_context = get_headers_call_context(environ)
611 611
612 612 repo_path = call_context['repo_path']
613 613 repo_name = call_context['repo_name']
614 614 config = call_context['repo_config']
615 615
616 616 environ['PATH_INFO'] = call_context['path_info']
617 617 self.set_env_from_config(environ, config)
618 618
619 619 content_type = environ.get('CONTENT_TYPE', '')
620 620
621 621 path = environ['PATH_INFO']
622 622 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
623 623 log.debug(
624 624 'LFS: Detecting if request `%s` is LFS server path based '
625 625 'on content type:`%s`, is_lfs:%s',
626 626 path, content_type, is_lfs_request)
627 627
628 628 if not is_lfs_request:
629 629 # fallback detection by path
630 630 if GIT_LFS_PROTO_PAT.match(path):
631 631 is_lfs_request = True
632 632 log.debug(
633 633 'LFS: fallback detection by path of: `%s`, is_lfs:%s',
634 634 path, is_lfs_request)
635 635
636 636 if is_lfs_request:
637 637 app = scm_app.create_git_lfs_wsgi_app(
638 638 repo_path, repo_name, config)
639 639 else:
640 640 app = scm_app.create_git_wsgi_app(
641 641 repo_path, repo_name, config)
642 642
643 643 log.debug('http-app: starting app handler '
644 644 'with %s and process request', app)
645 645
646 646 return app(environ, start_response)
647 647
648 648 return _git_stream
649 649
650 650 def handle_vcs_exception(self, exception, request):
651 651 _vcs_kind = getattr(exception, '_vcs_kind', '')
652 652
653 653 if _vcs_kind == 'repo_locked':
654 654 headers_call_context = get_headers_call_context(request.environ)
655 655 status_code = safe_int(headers_call_context['locked_status_code'])
656 656
657 657 return HTTPRepoLocked(
658 658 title=str(exception), status_code=status_code, headers=[('X-Rc-Locked', '1')])
659 659
660 660 elif _vcs_kind == 'repo_branch_protected':
661 661 # Get custom repo-branch-protected status code if present.
662 662 return HTTPRepoBranchProtected(
663 663 title=str(exception), headers=[('X-Rc-Branch-Protection', '1')])
664 664
665 665 exc_info = request.exc_info
666 666 store_exception(id(exc_info), exc_info)
667 667
668 668 traceback_info = 'unavailable'
669 669 if request.exc_info:
670 670 traceback_info = format_exc(request.exc_info)
671 671
672 672 log.error(
673 673 'error occurred handling this request for path: %s, \n%s',
674 674 request.path, traceback_info)
675 675
676 676 statsd = request.registry.statsd
677 677 if statsd:
678 678 exc_type = f"{exception.__class__.__module__}.{exception.__class__.__name__}"
679 679 statsd.incr('vcsserver_exception_total',
680 680 tags=[f"type:{exc_type}"])
681 681 raise exception
682 682
683 683
684 684 class ResponseFilter:
685 685
686 686 def __init__(self, start_response):
687 687 self._start_response = start_response
688 688
689 689 def __call__(self, status, response_headers, exc_info=None):
690 690 headers = tuple(
691 691 (h, v) for h, v in response_headers
692 692 if not wsgiref.util.is_hop_by_hop(h))
693 693 return self._start_response(status, headers, exc_info)
694 694
695 695
696 696 def sanitize_settings_and_apply_defaults(global_config, settings):
697 697 _global_settings_maker = SettingsMaker(global_config)
698 698 settings_maker = SettingsMaker(settings)
699 699
700 700 settings_maker.make_setting('logging.autoconfigure', False, parser='bool')
701 701
702 702 logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini')
703 703 settings_maker.enable_logging(logging_conf)
704 704
705 705 # Default includes, possible to change as a user
706 706 pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
707 707 log.debug("Using the following pyramid.includes: %s", pyramid_includes)
708 708
709 709 settings_maker.make_setting('__file__', global_config.get('__file__'))
710 710
711 711 settings_maker.make_setting('pyramid.default_locale_name', 'en')
712 712 settings_maker.make_setting('locale', 'en_US.UTF-8')
713 713
714 714 settings_maker.make_setting(
715 715 'core.binary_dir', '/usr/local/bin/rhodecode_bin/vcs_bin',
716 716 default_when_empty=True, parser='string:noquote')
717 717
718 718 temp_store = tempfile.gettempdir()
719 719 default_cache_dir = os.path.join(temp_store, 'rc_cache')
720 720 # save default, cache dir, and use it for all backends later.
721 721 default_cache_dir = settings_maker.make_setting(
722 722 'cache_dir',
723 723 default=default_cache_dir, default_when_empty=True,
724 724 parser='dir:ensured')
725 725
726 726 # exception store cache
727 727 settings_maker.make_setting(
728 728 'exception_tracker.store_path',
729 729 default=os.path.join(default_cache_dir, 'exc_store'), default_when_empty=True,
730 730 parser='dir:ensured'
731 731 )
732 732
733 733 # repo_object cache defaults
734 734 settings_maker.make_setting(
735 735 'rc_cache.repo_object.backend',
736 736 default='dogpile.cache.rc.file_namespace',
737 737 parser='string')
738 738 settings_maker.make_setting(
739 739 'rc_cache.repo_object.expiration_time',
740 740 default=30 * 24 * 60 * 60, # 30days
741 741 parser='int')
742 742 settings_maker.make_setting(
743 743 'rc_cache.repo_object.arguments.filename',
744 744 default=os.path.join(default_cache_dir, 'vcsserver_cache_repo_object.db'),
745 745 parser='string')
746 746
747 747 # statsd
748 748 settings_maker.make_setting('statsd.enabled', False, parser='bool')
749 749 settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
750 750 settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
751 751 settings_maker.make_setting('statsd.statsd_prefix', '')
752 752 settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
753 753
754 754 settings_maker.env_expand()
755 755
756 756
757 757 def main(global_config, **settings):
758 758 start_time = time.time()
759 759 log.info('Pyramid app config starting')
760 760
761 761 if MercurialFactory:
762 762 hgpatches.patch_largefiles_capabilities()
763 763 hgpatches.patch_subrepo_type_mapping()
764 764
765 765 # Fill in and sanitize the defaults & do ENV expansion
766 766 sanitize_settings_and_apply_defaults(global_config, settings)
767 767
768 768 # init and bootstrap StatsdClient
769 769 StatsdClient.setup(settings)
770 770
771 771 pyramid_app = HTTPApplication(settings=settings, global_config=global_config).wsgi_app()
772 772 total_time = time.time() - start_time
773 773 log.info('Pyramid app created and configured in %.2fs', total_time)
774 774 return pyramid_app
1 NO CONTENT: file renamed from vcsserver/lib/rc_json.py to vcsserver/lib/ext_json.py
@@ -1,417 +1,417 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 """Handles the Git smart protocol."""
19 19
20 20 import os
21 21 import socket
22 22 import logging
23 23
24 24 import dulwich.protocol
25 25 from dulwich.protocol import CAPABILITY_SIDE_BAND, CAPABILITY_SIDE_BAND_64K
26 26 from webob import Request, Response, exc
27 27
28 from vcsserver.lib.rc_json import json
28 from vcsserver.lib.ext_json import json
29 29 from vcsserver import hooks, subprocessio
30 30 from vcsserver.str_utils import ascii_bytes
31 31
32 32
33 33 log = logging.getLogger(__name__)
34 34
35 35
36 36 class FileWrapper:
37 37 """File wrapper that ensures how much data is read from it."""
38 38
39 39 def __init__(self, fd, content_length):
40 40 self.fd = fd
41 41 self.content_length = content_length
42 42 self.remain = content_length
43 43
44 44 def read(self, size):
45 45 if size <= self.remain:
46 46 try:
47 47 data = self.fd.read(size)
48 48 except socket.error:
49 49 raise IOError(self)
50 50 self.remain -= size
51 51 elif self.remain:
52 52 data = self.fd.read(self.remain)
53 53 self.remain = 0
54 54 else:
55 55 data = None
56 56 return data
57 57
58 58 def __repr__(self):
59 59 return '<FileWrapper {} len: {}, read: {}>'.format(
60 60 self.fd, self.content_length, self.content_length - self.remain
61 61 )
62 62
63 63
64 64 class GitRepository:
65 65 """WSGI app for handling Git smart protocol endpoints."""
66 66
67 67 git_folder_signature = frozenset(('config', 'head', 'info', 'objects', 'refs'))
68 68 commands = frozenset(('git-upload-pack', 'git-receive-pack'))
69 69 valid_accepts = frozenset(f'application/x-{c}-result' for c in commands)
70 70
71 71 # The last bytes are the SHA1 of the first 12 bytes.
72 72 EMPTY_PACK = (
73 73 b'PACK\x00\x00\x00\x02\x00\x00\x00\x00\x02\x9d\x08' +
74 74 b'\x82;\xd8\xa8\xea\xb5\x10\xadj\xc7\\\x82<\xfd>\xd3\x1e'
75 75 )
76 76 FLUSH_PACKET = b"0000"
77 77
78 78 SIDE_BAND_CAPS = frozenset((CAPABILITY_SIDE_BAND, CAPABILITY_SIDE_BAND_64K))
79 79
80 80 def __init__(self, repo_name, content_path, git_path, update_server_info, extras):
81 81 files = frozenset(f.lower() for f in os.listdir(content_path))
82 82 valid_dir_signature = self.git_folder_signature.issubset(files)
83 83
84 84 if not valid_dir_signature:
85 85 raise OSError(f'{content_path} missing git signature')
86 86
87 87 self.content_path = content_path
88 88 self.repo_name = repo_name
89 89 self.extras = extras
90 90 self.git_path = git_path
91 91 self.update_server_info = update_server_info
92 92
93 93 def _get_fixedpath(self, path):
94 94 """
95 95 Small fix for repo_path
96 96
97 97 :param path:
98 98 """
99 99 path = path.split(self.repo_name, 1)[-1]
100 100 if path.startswith('.git'):
101 101 # for bare repos we still get the .git prefix inside, we skip it
102 102 # here, and remove from the service command
103 103 path = path[4:]
104 104
105 105 return path.strip('/')
106 106
107 107 def inforefs(self, request, unused_environ):
108 108 """
109 109 WSGI Response producer for HTTP GET Git Smart
110 110 HTTP /info/refs request.
111 111 """
112 112
113 113 git_command = request.GET.get('service')
114 114 if git_command not in self.commands:
115 115 log.debug('command %s not allowed', git_command)
116 116 return exc.HTTPForbidden()
117 117
118 118 # please, resist the urge to add '\n' to git capture and increment
119 119 # line count by 1.
120 120 # by git docs: Documentation/technical/http-protocol.txt#L214 \n is
121 121 # a part of protocol.
122 122 # The code in Git client not only does NOT need '\n', but actually
123 123 # blows up if you sprinkle "flush" (0000) as "0001\n".
124 124 # It reads binary, per number of bytes specified.
125 125 # if you do add '\n' as part of data, count it.
126 126 server_advert = f'# service={git_command}\n'
127 127 packet_len = hex(len(server_advert) + 4)[2:].rjust(4, '0').lower()
128 128 try:
129 129 gitenv = dict(os.environ)
130 130 # forget all configs
131 131 gitenv['RC_SCM_DATA'] = json.dumps(self.extras)
132 132 command = [self.git_path, git_command[4:], '--stateless-rpc',
133 133 '--advertise-refs', self.content_path]
134 134 out = subprocessio.SubprocessIOChunker(
135 135 command,
136 136 env=gitenv,
137 137 starting_values=[ascii_bytes(packet_len + server_advert) + self.FLUSH_PACKET],
138 138 shell=False
139 139 )
140 140 except OSError:
141 141 log.exception('Error processing command')
142 142 raise exc.HTTPExpectationFailed()
143 143
144 144 resp = Response()
145 145 resp.content_type = f'application/x-{git_command}-advertisement'
146 146 resp.charset = None
147 147 resp.app_iter = out
148 148
149 149 return resp
150 150
151 151 def _get_want_capabilities(self, request):
152 152 """Read the capabilities found in the first want line of the request."""
153 153 pos = request.body_file_seekable.tell()
154 154 first_line = request.body_file_seekable.readline()
155 155 request.body_file_seekable.seek(pos)
156 156
157 157 return frozenset(
158 158 dulwich.protocol.extract_want_line_capabilities(first_line)[1])
159 159
160 160 def _build_failed_pre_pull_response(self, capabilities, pre_pull_messages):
161 161 """
162 162 Construct a response with an empty PACK file.
163 163
164 164 We use an empty PACK file, as that would trigger the failure of the pull
165 165 or clone command.
166 166
167 167 We also print in the error output a message explaining why the command
168 168 was aborted.
169 169
170 170 If additionally, the user is accepting messages we send them the output
171 171 of the pre-pull hook.
172 172
173 173 Note that for clients not supporting side-band we just send them the
174 174 emtpy PACK file.
175 175 """
176 176
177 177 if self.SIDE_BAND_CAPS.intersection(capabilities):
178 178 response = []
179 179 proto = dulwich.protocol.Protocol(None, response.append)
180 180 proto.write_pkt_line(dulwich.protocol.NAK_LINE)
181 181
182 182 self._write_sideband_to_proto(proto, ascii_bytes(pre_pull_messages, allow_bytes=True), capabilities)
183 183 # N.B.(skreft): Do not change the sideband channel to 3, as that
184 184 # produces a fatal error in the client:
185 185 # fatal: error in sideband demultiplexer
186 186 proto.write_sideband(
187 187 dulwich.protocol.SIDE_BAND_CHANNEL_PROGRESS,
188 188 ascii_bytes('Pre pull hook failed: aborting\n', allow_bytes=True))
189 189 proto.write_sideband(
190 190 dulwich.protocol.SIDE_BAND_CHANNEL_DATA,
191 191 ascii_bytes(self.EMPTY_PACK, allow_bytes=True))
192 192
193 193 # writes b"0000" as default
194 194 proto.write_pkt_line(None)
195 195
196 196 return response
197 197 else:
198 198 return [ascii_bytes(self.EMPTY_PACK, allow_bytes=True)]
199 199
200 200 def _build_post_pull_response(self, response, capabilities, start_message, end_message):
201 201 """
202 202 Given a list response we inject the post-pull messages.
203 203
204 204 We only inject the messages if the client supports sideband, and the
205 205 response has the format:
206 206 0008NAK\n...0000
207 207
208 208 Note that we do not check the no-progress capability as by default, git
209 209 sends it, which effectively would block all messages.
210 210 """
211 211
212 212 if not self.SIDE_BAND_CAPS.intersection(capabilities):
213 213 return response
214 214
215 215 if not start_message and not end_message:
216 216 return response
217 217
218 218 try:
219 219 iter(response)
220 220 # iterator probably will work, we continue
221 221 except TypeError:
222 222 raise TypeError(f'response must be an iterator: got {type(response)}')
223 223 if isinstance(response, (list, tuple)):
224 224 raise TypeError(f'response must be an iterator: got {type(response)}')
225 225
226 226 def injected_response():
227 227
228 228 do_loop = 1
229 229 header_injected = 0
230 230 next_item = None
231 231 has_item = False
232 232 item = b''
233 233
234 234 while do_loop:
235 235
236 236 try:
237 237 next_item = next(response)
238 238 except StopIteration:
239 239 do_loop = 0
240 240
241 241 if has_item:
242 242 # last item ! alter it now
243 243 if do_loop == 0 and item.endswith(self.FLUSH_PACKET):
244 244 new_response = [item[:-4]]
245 245 new_response.extend(self._get_messages(end_message, capabilities))
246 246 new_response.append(self.FLUSH_PACKET)
247 247 item = b''.join(new_response)
248 248
249 249 yield item
250 250
251 251 has_item = True
252 252 item = next_item
253 253
254 254 # alter item if it's the initial chunk
255 255 if not header_injected and item.startswith(b'0008NAK\n'):
256 256 new_response = [b'0008NAK\n']
257 257 new_response.extend(self._get_messages(start_message, capabilities))
258 258 new_response.append(item[8:])
259 259 item = b''.join(new_response)
260 260 header_injected = 1
261 261
262 262 return injected_response()
263 263
264 264 def _write_sideband_to_proto(self, proto, data, capabilities):
265 265 """
266 266 Write the data to the proto's sideband number 2 == SIDE_BAND_CHANNEL_PROGRESS
267 267
268 268 We do not use dulwich's write_sideband directly as it only supports
269 269 side-band-64k.
270 270 """
271 271 if not data:
272 272 return
273 273
274 274 # N.B.(skreft): The values below are explained in the pack protocol
275 275 # documentation, section Packfile Data.
276 276 # https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt
277 277 if CAPABILITY_SIDE_BAND_64K in capabilities:
278 278 chunk_size = 65515
279 279 elif CAPABILITY_SIDE_BAND in capabilities:
280 280 chunk_size = 995
281 281 else:
282 282 return
283 283
284 284 chunker = (data[i:i + chunk_size] for i in range(0, len(data), chunk_size))
285 285
286 286 for chunk in chunker:
287 287 proto.write_sideband(dulwich.protocol.SIDE_BAND_CHANNEL_PROGRESS, ascii_bytes(chunk, allow_bytes=True))
288 288
289 289 def _get_messages(self, data, capabilities):
290 290 """Return a list with packets for sending data in sideband number 2."""
291 291 response = []
292 292 proto = dulwich.protocol.Protocol(None, response.append)
293 293
294 294 self._write_sideband_to_proto(proto, data, capabilities)
295 295
296 296 return response
297 297
298 298 def backend(self, request, environ):
299 299 """
300 300 WSGI Response producer for HTTP POST Git Smart HTTP requests.
301 301 Reads commands and data from HTTP POST's body.
302 302 returns an iterator obj with contents of git command's
303 303 response to stdout
304 304 """
305 305 # TODO(skreft): think how we could detect an HTTPLockedException, as
306 306 # we probably want to have the same mechanism used by mercurial and
307 307 # simplevcs.
308 308 # For that we would need to parse the output of the command looking for
309 309 # some signs of the HTTPLockedError, parse the data and reraise it in
310 310 # pygrack. However, that would interfere with the streaming.
311 311 #
312 312 # Now the output of a blocked push is:
313 313 # Pushing to http://test_regular:test12@127.0.0.1:5001/vcs_test_git
314 314 # POST git-receive-pack (1047 bytes)
315 315 # remote: ERROR: Repository `vcs_test_git` locked by user `test_admin`. Reason:`lock_auto`
316 316 # To http://test_regular:test12@127.0.0.1:5001/vcs_test_git
317 317 # ! [remote rejected] master -> master (pre-receive hook declined)
318 318 # error: failed to push some refs to 'http://test_regular:test12@127.0.0.1:5001/vcs_test_git'
319 319
320 320 git_command = self._get_fixedpath(request.path_info)
321 321 if git_command not in self.commands:
322 322 log.debug('command %s not allowed', git_command)
323 323 return exc.HTTPForbidden()
324 324
325 325 capabilities = None
326 326 if git_command == 'git-upload-pack':
327 327 capabilities = self._get_want_capabilities(request)
328 328
329 329 if 'CONTENT_LENGTH' in environ:
330 330 inputstream = FileWrapper(request.body_file_seekable,
331 331 request.content_length)
332 332 else:
333 333 inputstream = request.body_file_seekable
334 334
335 335 resp = Response()
336 336 resp.content_type = f'application/x-{git_command}-result'
337 337 resp.charset = None
338 338
339 339 pre_pull_messages = ''
340 340 # Upload-pack == clone
341 341 if git_command == 'git-upload-pack':
342 342 hook_response = hooks.git_pre_pull(self.extras)
343 343 if hook_response.status != 0:
344 344 pre_pull_messages = hook_response.output
345 345 resp.app_iter = self._build_failed_pre_pull_response(
346 346 capabilities, pre_pull_messages)
347 347 return resp
348 348
349 349 gitenv = dict(os.environ)
350 350 # forget all configs
351 351 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
352 352 gitenv['RC_SCM_DATA'] = json.dumps(self.extras)
353 353 cmd = [self.git_path, git_command[4:], '--stateless-rpc',
354 354 self.content_path]
355 355 log.debug('handling cmd %s', cmd)
356 356
357 357 out = subprocessio.SubprocessIOChunker(
358 358 cmd,
359 359 input_stream=inputstream,
360 360 env=gitenv,
361 361 cwd=self.content_path,
362 362 shell=False,
363 363 fail_on_stderr=False,
364 364 fail_on_return_code=False
365 365 )
366 366
367 367 if self.update_server_info and git_command == 'git-receive-pack':
368 368 # We need to fully consume the iterator here, as the
369 369 # update-server-info command needs to be run after the push.
370 370 out = list(out)
371 371
372 372 # Updating refs manually after each push.
373 373 # This is required as some clients are exposing Git repos internally
374 374 # with the dumb protocol.
375 375 cmd = [self.git_path, 'update-server-info']
376 376 log.debug('handling cmd %s', cmd)
377 377 output = subprocessio.SubprocessIOChunker(
378 378 cmd,
379 379 input_stream=inputstream,
380 380 env=gitenv,
381 381 cwd=self.content_path,
382 382 shell=False,
383 383 fail_on_stderr=False,
384 384 fail_on_return_code=False
385 385 )
386 386 # Consume all the output so the subprocess finishes
387 387 for _ in output:
388 388 pass
389 389
390 390 # Upload-pack == clone
391 391 if git_command == 'git-upload-pack':
392 392 hook_response = hooks.git_post_pull(self.extras)
393 393 post_pull_messages = hook_response.output
394 394 resp.app_iter = self._build_post_pull_response(out, capabilities, pre_pull_messages, post_pull_messages)
395 395 else:
396 396 resp.app_iter = out
397 397
398 398 return resp
399 399
400 400 def __call__(self, environ, start_response):
401 401 request = Request(environ)
402 402 _path = self._get_fixedpath(request.path_info)
403 403 if _path.startswith('info/refs'):
404 404 app = self.inforefs
405 405 else:
406 406 app = self.backend
407 407
408 408 try:
409 409 resp = app(request, environ)
410 410 except exc.HTTPException as error:
411 411 log.exception('HTTP Error')
412 412 resp = error
413 413 except Exception:
414 414 log.exception('Unknown error')
415 415 resp = exc.HTTPInternalServerError()
416 416
417 417 return resp(environ, start_response)
@@ -1,257 +1,257 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import threading
19 19 import msgpack
20 20
21 21 from http.server import BaseHTTPRequestHandler
22 22 from socketserver import TCPServer
23 23
24 24 import mercurial.ui
25 25 import mock
26 26 import pytest
27 27
28 28 from vcsserver.hooks import HooksHttpClient
29 from vcsserver.lib.rc_json import json
29 from vcsserver.lib.ext_json import json
30 30 from vcsserver import hooks
31 31
32 32
33 33 def get_hg_ui(extras=None):
34 34 """Create a Config object with a valid RC_SCM_DATA entry."""
35 35 extras = extras or {}
36 36 required_extras = {
37 37 'username': '',
38 38 'repository': '',
39 39 'locked_by': '',
40 40 'scm': '',
41 41 'make_lock': '',
42 42 'action': '',
43 43 'ip': '',
44 44 'hooks_uri': 'fake_hooks_uri',
45 45 }
46 46 required_extras.update(extras)
47 47 hg_ui = mercurial.ui.ui()
48 48 hg_ui.setconfig(b'rhodecode', b'RC_SCM_DATA', json.dumps(required_extras))
49 49
50 50 return hg_ui
51 51
52 52
53 53 def test_git_pre_receive_is_disabled():
54 54 extras = {'hooks': ['pull']}
55 55 response = hooks.git_pre_receive(None, None,
56 56 {'RC_SCM_DATA': json.dumps(extras)})
57 57
58 58 assert response == 0
59 59
60 60
61 61 def test_git_post_receive_is_disabled():
62 62 extras = {'hooks': ['pull']}
63 63 response = hooks.git_post_receive(None, '',
64 64 {'RC_SCM_DATA': json.dumps(extras)})
65 65
66 66 assert response == 0
67 67
68 68
69 69 def test_git_post_receive_calls_repo_size():
70 70 extras = {'hooks': ['push', 'repo_size']}
71 71
72 72 with mock.patch.object(hooks, '_call_hook') as call_hook_mock:
73 73 hooks.git_post_receive(
74 74 None, '', {'RC_SCM_DATA': json.dumps(extras)})
75 75 extras.update({'commit_ids': [], 'hook_type': 'post_receive',
76 76 'new_refs': {'bookmarks': [], 'branches': [], 'tags': []}})
77 77 expected_calls = [
78 78 mock.call('repo_size', extras, mock.ANY),
79 79 mock.call('post_push', extras, mock.ANY),
80 80 ]
81 81 assert call_hook_mock.call_args_list == expected_calls
82 82
83 83
84 84 def test_git_post_receive_does_not_call_disabled_repo_size():
85 85 extras = {'hooks': ['push']}
86 86
87 87 with mock.patch.object(hooks, '_call_hook') as call_hook_mock:
88 88 hooks.git_post_receive(
89 89 None, '', {'RC_SCM_DATA': json.dumps(extras)})
90 90 extras.update({'commit_ids': [], 'hook_type': 'post_receive',
91 91 'new_refs': {'bookmarks': [], 'branches': [], 'tags': []}})
92 92 expected_calls = [
93 93 mock.call('post_push', extras, mock.ANY)
94 94 ]
95 95 assert call_hook_mock.call_args_list == expected_calls
96 96
97 97
98 98 def test_repo_size_exception_does_not_affect_git_post_receive():
99 99 extras = {'hooks': ['push', 'repo_size']}
100 100 status = 0
101 101
102 102 def side_effect(name, *args, **kwargs):
103 103 if name == 'repo_size':
104 104 raise Exception('Fake exception')
105 105 else:
106 106 return status
107 107
108 108 with mock.patch.object(hooks, '_call_hook') as call_hook_mock:
109 109 call_hook_mock.side_effect = side_effect
110 110 result = hooks.git_post_receive(
111 111 None, '', {'RC_SCM_DATA': json.dumps(extras)})
112 112 assert result == status
113 113
114 114
115 115 def test_git_pre_pull_is_disabled():
116 116 assert hooks.git_pre_pull({'hooks': ['push']}) == hooks.HookResponse(0, '')
117 117
118 118
119 119 def test_git_post_pull_is_disabled():
120 120 assert (
121 121 hooks.git_post_pull({'hooks': ['push']}) == hooks.HookResponse(0, ''))
122 122
123 123
124 124 class TestGetHooksClient:
125 125
126 126 def test_returns_http_client_when_protocol_matches(self):
127 127 hooks_uri = 'localhost:8000'
128 128 result = hooks._get_hooks_client({
129 129 'hooks_uri': hooks_uri,
130 130 'hooks_protocol': 'http'
131 131 })
132 132 assert isinstance(result, hooks.HooksHttpClient)
133 133 assert result.hooks_uri == hooks_uri
134 134
135 135 def test_return_celery_client_when_queue_and_backend_provided(self):
136 136 task_queue = 'redis://task_queue:0'
137 137 task_backend = task_queue
138 138 result = hooks._get_hooks_client({
139 139 'task_queue': task_queue,
140 140 'task_backend': task_backend
141 141 })
142 142 assert isinstance(result, hooks.HooksCeleryClient)
143 143
144 144
145 145 class TestHooksHttpClient:
146 146 def test_init_sets_hooks_uri(self):
147 147 uri = 'localhost:3000'
148 148 client = hooks.HooksHttpClient(uri)
149 149 assert client.hooks_uri == uri
150 150
151 151 def test_serialize_returns_serialized_string(self):
152 152 client = hooks.HooksHttpClient('localhost:3000')
153 153 hook_name = 'test'
154 154 extras = {
155 155 'first': 1,
156 156 'second': 'two'
157 157 }
158 158 hooks_proto, result = client._serialize(hook_name, extras)
159 159 expected_result = msgpack.packb({
160 160 'method': hook_name,
161 161 'extras': extras,
162 162 })
163 163 assert hooks_proto == {'rc-hooks-protocol': 'msgpack.v1', 'Connection': 'keep-alive'}
164 164 assert result == expected_result
165 165
166 166 def test_call_queries_http_server(self, http_mirror):
167 167 client = hooks.HooksHttpClient(http_mirror.uri)
168 168 hook_name = 'test'
169 169 extras = {
170 170 'first': 1,
171 171 'second': 'two'
172 172 }
173 173 result = client(hook_name, extras)
174 174 expected_result = msgpack.unpackb(msgpack.packb({
175 175 'method': hook_name,
176 176 'extras': extras
177 177 }), raw=False)
178 178 assert result == expected_result
179 179
180 180
181 181 @pytest.fixture
182 182 def http_mirror(request):
183 183 server = MirrorHttpServer()
184 184 request.addfinalizer(server.stop)
185 185 return server
186 186
187 187
188 188 class MirrorHttpHandler(BaseHTTPRequestHandler):
189 189
190 190 def do_POST(self):
191 191 length = int(self.headers['Content-Length'])
192 192 body = self.rfile.read(length)
193 193 self.send_response(200)
194 194 self.end_headers()
195 195 self.wfile.write(body)
196 196
197 197
198 198 class MirrorHttpServer:
199 199 ip_address = '127.0.0.1'
200 200 port = 0
201 201
202 202 def __init__(self):
203 203 self._daemon = TCPServer((self.ip_address, 0), MirrorHttpHandler)
204 204 _, self.port = self._daemon.server_address
205 205 self._thread = threading.Thread(target=self._daemon.serve_forever)
206 206 self._thread.daemon = True
207 207 self._thread.start()
208 208
209 209 def stop(self):
210 210 self._daemon.shutdown()
211 211 self._thread.join()
212 212 self._daemon = None
213 213 self._thread = None
214 214
215 215 @property
216 216 def uri(self):
217 217 return '{}:{}'.format(self.ip_address, self.port)
218 218
219 219
220 220 def test_hooks_http_client_init():
221 221 hooks_uri = 'http://localhost:8000'
222 222 client = HooksHttpClient(hooks_uri)
223 223 assert client.hooks_uri == hooks_uri
224 224
225 225
226 226 def test_hooks_http_client_call():
227 227 hooks_uri = 'http://localhost:8000'
228 228
229 229 method = 'test_method'
230 230 extras = {'key': 'value'}
231 231
232 232 with \
233 233 mock.patch('http.client.HTTPConnection') as mock_connection,\
234 234 mock.patch('msgpack.load') as mock_load:
235 235
236 236 client = HooksHttpClient(hooks_uri)
237 237
238 238 mock_load.return_value = {'result': 'success'}
239 239 response = mock.MagicMock()
240 240 response.status = 200
241 241 mock_connection.request.side_effect = None
242 242 mock_connection.getresponse.return_value = response
243 243
244 244 result = client(method, extras)
245 245
246 246 mock_connection.assert_called_with(hooks_uri)
247 247 mock_connection.return_value.request.assert_called_once()
248 248 assert result == {'result': 'success'}
249 249
250 250
251 251 def test_hooks_http_client_serialize():
252 252 method = 'test_method'
253 253 extras = {'key': 'value'}
254 254 headers, body = HooksHttpClient._serialize(method, extras)
255 255
256 256 assert headers == {'rc-hooks-protocol': HooksHttpClient.proto, 'Connection': 'keep-alive'}
257 257 assert msgpack.unpackb(body) == {'method': method, 'extras': extras}
General Comments 0
You need to be logged in to leave comments. Login now