Show More
@@ -1,296 +1,296 b'' | |||
|
1 | 1 | # RhodeCode VCSServer provides access to different vcs backends via network. |
|
2 | 2 | # Copyright (C) 2014-2023 RhodeCode GmbH |
|
3 | 3 | # |
|
4 | 4 | # This program is free software; you can redistribute it and/or modify |
|
5 | 5 | # it under the terms of the GNU General Public License as published by |
|
6 | 6 | # the Free Software Foundation; either version 3 of the License, or |
|
7 | 7 | # (at your option) any later version. |
|
8 | 8 | # |
|
9 | 9 | # This program is distributed in the hope that it will be useful, |
|
10 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 | 12 | # GNU General Public License for more details. |
|
13 | 13 | # |
|
14 | 14 | # You should have received a copy of the GNU General Public License |
|
15 | 15 | # along with this program; if not, write to the Free Software Foundation, |
|
16 | 16 | # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
|
17 | 17 | |
|
18 | 18 | import re |
|
19 | 19 | import logging |
|
20 | 20 | |
|
21 | 21 | from pyramid.config import Configurator |
|
22 | 22 | from pyramid.response import Response, FileIter |
|
23 | 23 | from pyramid.httpexceptions import ( |
|
24 | 24 | HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden, |
|
25 | 25 | HTTPUnprocessableEntity) |
|
26 | 26 | |
|
27 |
from vcsserver.lib. |
|
|
27 | from vcsserver.lib.ext_json import json | |
|
28 | 28 | from vcsserver.git_lfs.lib import OidHandler, LFSOidStore |
|
29 | 29 | from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator |
|
30 | 30 | from vcsserver.str_utils import safe_int |
|
31 | 31 | |
|
32 | 32 | log = logging.getLogger(__name__) |
|
33 | 33 | |
|
34 | 34 | |
|
35 | 35 | GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs' # +json ? |
|
36 | 36 | GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))') |
|
37 | 37 | |
|
38 | 38 | |
|
39 | 39 | def write_response_error(http_exception, text=None): |
|
40 | 40 | content_type = GIT_LFS_CONTENT_TYPE + '+json' |
|
41 | 41 | _exception = http_exception(content_type=content_type) |
|
42 | 42 | _exception.content_type = content_type |
|
43 | 43 | if text: |
|
44 | 44 | _exception.body = json.dumps({'message': text}) |
|
45 | 45 | log.debug('LFS: writing response of type %s to client with text:%s', |
|
46 | 46 | http_exception, text) |
|
47 | 47 | return _exception |
|
48 | 48 | |
|
49 | 49 | |
|
50 | 50 | class AuthHeaderRequired: |
|
51 | 51 | """ |
|
52 | 52 | Decorator to check if request has proper auth-header |
|
53 | 53 | """ |
|
54 | 54 | |
|
55 | 55 | def __call__(self, func): |
|
56 | 56 | return get_cython_compat_decorator(self.__wrapper, func) |
|
57 | 57 | |
|
58 | 58 | def __wrapper(self, func, *fargs, **fkwargs): |
|
59 | 59 | request = fargs[1] |
|
60 | 60 | auth = request.authorization |
|
61 | 61 | if not auth: |
|
62 | 62 | return write_response_error(HTTPForbidden) |
|
63 | 63 | return func(*fargs[1:], **fkwargs) |
|
64 | 64 | |
|
65 | 65 | |
|
66 | 66 | # views |
|
67 | 67 | |
|
68 | 68 | def lfs_objects(request): |
|
69 | 69 | # indicate not supported, V1 API |
|
70 | 70 | log.warning('LFS: v1 api not supported, reporting it back to client') |
|
71 | 71 | return write_response_error(HTTPNotImplemented, 'LFS: v1 api not supported') |
|
72 | 72 | |
|
73 | 73 | |
|
74 | 74 | @AuthHeaderRequired() |
|
75 | 75 | def lfs_objects_batch(request): |
|
76 | 76 | """ |
|
77 | 77 | The client sends the following information to the Batch endpoint to transfer some objects: |
|
78 | 78 | |
|
79 | 79 | operation - Should be download or upload. |
|
80 | 80 | transfers - An optional Array of String identifiers for transfer |
|
81 | 81 | adapters that the client has configured. If omitted, the basic |
|
82 | 82 | transfer adapter MUST be assumed by the server. |
|
83 | 83 | objects - An Array of objects to download. |
|
84 | 84 | oid - String OID of the LFS object. |
|
85 | 85 | size - Integer byte size of the LFS object. Must be at least zero. |
|
86 | 86 | """ |
|
87 | 87 | request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json' |
|
88 | 88 | auth = request.authorization |
|
89 | 89 | repo = request.matchdict.get('repo') |
|
90 | 90 | data = request.json |
|
91 | 91 | operation = data.get('operation') |
|
92 | 92 | http_scheme = request.registry.git_lfs_http_scheme |
|
93 | 93 | |
|
94 | 94 | if operation not in ('download', 'upload'): |
|
95 | 95 | log.debug('LFS: unsupported operation:%s', operation) |
|
96 | 96 | return write_response_error( |
|
97 | 97 | HTTPBadRequest, f'unsupported operation mode: `{operation}`') |
|
98 | 98 | |
|
99 | 99 | if 'objects' not in data: |
|
100 | 100 | log.debug('LFS: missing objects data') |
|
101 | 101 | return write_response_error( |
|
102 | 102 | HTTPBadRequest, 'missing objects data') |
|
103 | 103 | |
|
104 | 104 | log.debug('LFS: handling operation of type: %s', operation) |
|
105 | 105 | |
|
106 | 106 | objects = [] |
|
107 | 107 | for o in data['objects']: |
|
108 | 108 | try: |
|
109 | 109 | oid = o['oid'] |
|
110 | 110 | obj_size = o['size'] |
|
111 | 111 | except KeyError: |
|
112 | 112 | log.exception('LFS, failed to extract data') |
|
113 | 113 | return write_response_error( |
|
114 | 114 | HTTPBadRequest, 'unsupported data in objects') |
|
115 | 115 | |
|
116 | 116 | obj_data = {'oid': oid} |
|
117 | 117 | if http_scheme == 'http': |
|
118 | 118 | # Note(marcink): when using http, we might have a custom port |
|
119 | 119 | # so we skip setting it to http, url dispatch then wont generate a port in URL |
|
120 | 120 | # for development we need this |
|
121 | 121 | http_scheme = None |
|
122 | 122 | |
|
123 | 123 | obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid, |
|
124 | 124 | _scheme=http_scheme) |
|
125 | 125 | obj_verify_href = request.route_url('lfs_objects_verify', repo=repo, |
|
126 | 126 | _scheme=http_scheme) |
|
127 | 127 | store = LFSOidStore( |
|
128 | 128 | oid, repo, store_location=request.registry.git_lfs_store_path) |
|
129 | 129 | handler = OidHandler( |
|
130 | 130 | store, repo, auth, oid, obj_size, obj_data, |
|
131 | 131 | obj_href, obj_verify_href) |
|
132 | 132 | |
|
133 | 133 | # this verifies also OIDs |
|
134 | 134 | actions, errors = handler.exec_operation(operation) |
|
135 | 135 | if errors: |
|
136 | 136 | log.warning('LFS: got following errors: %s', errors) |
|
137 | 137 | obj_data['errors'] = errors |
|
138 | 138 | |
|
139 | 139 | if actions: |
|
140 | 140 | obj_data['actions'] = actions |
|
141 | 141 | |
|
142 | 142 | obj_data['size'] = obj_size |
|
143 | 143 | obj_data['authenticated'] = True |
|
144 | 144 | objects.append(obj_data) |
|
145 | 145 | |
|
146 | 146 | result = {'objects': objects, 'transfer': 'basic'} |
|
147 | 147 | log.debug('LFS Response %s', safe_result(result)) |
|
148 | 148 | |
|
149 | 149 | return result |
|
150 | 150 | |
|
151 | 151 | |
|
152 | 152 | def lfs_objects_oid_upload(request): |
|
153 | 153 | request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json' |
|
154 | 154 | repo = request.matchdict.get('repo') |
|
155 | 155 | oid = request.matchdict.get('oid') |
|
156 | 156 | store = LFSOidStore( |
|
157 | 157 | oid, repo, store_location=request.registry.git_lfs_store_path) |
|
158 | 158 | engine = store.get_engine(mode='wb') |
|
159 | 159 | log.debug('LFS: starting chunked write of LFS oid: %s to storage', oid) |
|
160 | 160 | |
|
161 | 161 | body = request.environ['wsgi.input'] |
|
162 | 162 | |
|
163 | 163 | with engine as f: |
|
164 | 164 | blksize = 64 * 1024 # 64kb |
|
165 | 165 | while True: |
|
166 | 166 | # read in chunks as stream comes in from Gunicorn |
|
167 | 167 | # this is a specific Gunicorn support function. |
|
168 | 168 | # might work differently on waitress |
|
169 | 169 | chunk = body.read(blksize) |
|
170 | 170 | if not chunk: |
|
171 | 171 | break |
|
172 | 172 | f.write(chunk) |
|
173 | 173 | |
|
174 | 174 | return {'upload': 'ok'} |
|
175 | 175 | |
|
176 | 176 | |
|
177 | 177 | def lfs_objects_oid_download(request): |
|
178 | 178 | repo = request.matchdict.get('repo') |
|
179 | 179 | oid = request.matchdict.get('oid') |
|
180 | 180 | |
|
181 | 181 | store = LFSOidStore( |
|
182 | 182 | oid, repo, store_location=request.registry.git_lfs_store_path) |
|
183 | 183 | if not store.has_oid(): |
|
184 | 184 | log.debug('LFS: oid %s does not exists in store', oid) |
|
185 | 185 | return write_response_error( |
|
186 | 186 | HTTPNotFound, f'requested file with oid `{oid}` not found in store') |
|
187 | 187 | |
|
188 | 188 | # TODO(marcink): support range header ? |
|
189 | 189 | # Range: bytes=0-, `bytes=(\d+)\-.*` |
|
190 | 190 | |
|
191 | 191 | f = open(store.oid_path, 'rb') |
|
192 | 192 | response = Response( |
|
193 | 193 | content_type='application/octet-stream', app_iter=FileIter(f)) |
|
194 | 194 | response.headers.add('X-RC-LFS-Response-Oid', str(oid)) |
|
195 | 195 | return response |
|
196 | 196 | |
|
197 | 197 | |
|
198 | 198 | def lfs_objects_verify(request): |
|
199 | 199 | request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json' |
|
200 | 200 | repo = request.matchdict.get('repo') |
|
201 | 201 | |
|
202 | 202 | data = request.json |
|
203 | 203 | oid = data.get('oid') |
|
204 | 204 | size = safe_int(data.get('size')) |
|
205 | 205 | |
|
206 | 206 | if not (oid and size): |
|
207 | 207 | return write_response_error( |
|
208 | 208 | HTTPBadRequest, 'missing oid and size in request data') |
|
209 | 209 | |
|
210 | 210 | store = LFSOidStore( |
|
211 | 211 | oid, repo, store_location=request.registry.git_lfs_store_path) |
|
212 | 212 | if not store.has_oid(): |
|
213 | 213 | log.debug('LFS: oid %s does not exists in store', oid) |
|
214 | 214 | return write_response_error( |
|
215 | 215 | HTTPNotFound, f'oid `{oid}` does not exists in store') |
|
216 | 216 | |
|
217 | 217 | store_size = store.size_oid() |
|
218 | 218 | if store_size != size: |
|
219 | 219 | msg = 'requested file size mismatch store size:{} requested:{}'.format( |
|
220 | 220 | store_size, size) |
|
221 | 221 | return write_response_error( |
|
222 | 222 | HTTPUnprocessableEntity, msg) |
|
223 | 223 | |
|
224 | 224 | return {'message': {'size': 'ok', 'in_store': 'ok'}} |
|
225 | 225 | |
|
226 | 226 | |
|
227 | 227 | def lfs_objects_lock(request): |
|
228 | 228 | return write_response_error( |
|
229 | 229 | HTTPNotImplemented, 'GIT LFS locking api not supported') |
|
230 | 230 | |
|
231 | 231 | |
|
232 | 232 | def not_found(request): |
|
233 | 233 | return write_response_error( |
|
234 | 234 | HTTPNotFound, 'request path not found') |
|
235 | 235 | |
|
236 | 236 | |
|
237 | 237 | def lfs_disabled(request): |
|
238 | 238 | return write_response_error( |
|
239 | 239 | HTTPNotImplemented, 'GIT LFS disabled for this repo') |
|
240 | 240 | |
|
241 | 241 | |
|
242 | 242 | def git_lfs_app(config): |
|
243 | 243 | |
|
244 | 244 | # v1 API deprecation endpoint |
|
245 | 245 | config.add_route('lfs_objects', |
|
246 | 246 | '/{repo:.*?[^/]}/info/lfs/objects') |
|
247 | 247 | config.add_view(lfs_objects, route_name='lfs_objects', |
|
248 | 248 | request_method='POST', renderer='json') |
|
249 | 249 | |
|
250 | 250 | # locking API |
|
251 | 251 | config.add_route('lfs_objects_lock', |
|
252 | 252 | '/{repo:.*?[^/]}/info/lfs/locks') |
|
253 | 253 | config.add_view(lfs_objects_lock, route_name='lfs_objects_lock', |
|
254 | 254 | request_method=('POST', 'GET'), renderer='json') |
|
255 | 255 | |
|
256 | 256 | config.add_route('lfs_objects_lock_verify', |
|
257 | 257 | '/{repo:.*?[^/]}/info/lfs/locks/verify') |
|
258 | 258 | config.add_view(lfs_objects_lock, route_name='lfs_objects_lock_verify', |
|
259 | 259 | request_method=('POST', 'GET'), renderer='json') |
|
260 | 260 | |
|
261 | 261 | # batch API |
|
262 | 262 | config.add_route('lfs_objects_batch', |
|
263 | 263 | '/{repo:.*?[^/]}/info/lfs/objects/batch') |
|
264 | 264 | config.add_view(lfs_objects_batch, route_name='lfs_objects_batch', |
|
265 | 265 | request_method='POST', renderer='json') |
|
266 | 266 | |
|
267 | 267 | # oid upload/download API |
|
268 | 268 | config.add_route('lfs_objects_oid', |
|
269 | 269 | '/{repo:.*?[^/]}/info/lfs/objects/{oid}') |
|
270 | 270 | config.add_view(lfs_objects_oid_upload, route_name='lfs_objects_oid', |
|
271 | 271 | request_method='PUT', renderer='json') |
|
272 | 272 | config.add_view(lfs_objects_oid_download, route_name='lfs_objects_oid', |
|
273 | 273 | request_method='GET', renderer='json') |
|
274 | 274 | |
|
275 | 275 | # verification API |
|
276 | 276 | config.add_route('lfs_objects_verify', |
|
277 | 277 | '/{repo:.*?[^/]}/info/lfs/verify') |
|
278 | 278 | config.add_view(lfs_objects_verify, route_name='lfs_objects_verify', |
|
279 | 279 | request_method='POST', renderer='json') |
|
280 | 280 | |
|
281 | 281 | # not found handler for API |
|
282 | 282 | config.add_notfound_view(not_found, renderer='json') |
|
283 | 283 | |
|
284 | 284 | |
|
285 | 285 | def create_app(git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme): |
|
286 | 286 | config = Configurator() |
|
287 | 287 | if git_lfs_enabled: |
|
288 | 288 | config.include(git_lfs_app) |
|
289 | 289 | config.registry.git_lfs_store_path = git_lfs_store_path |
|
290 | 290 | config.registry.git_lfs_http_scheme = git_lfs_http_scheme |
|
291 | 291 | else: |
|
292 | 292 | # not found handler for API, reporting disabled LFS support |
|
293 | 293 | config.add_notfound_view(lfs_disabled, renderer='json') |
|
294 | 294 | |
|
295 | 295 | app = config.make_wsgi_app() |
|
296 | 296 | return app |
@@ -1,274 +1,274 b'' | |||
|
1 | 1 | # RhodeCode VCSServer provides access to different vcs backends via network. |
|
2 | 2 | # Copyright (C) 2014-2023 RhodeCode GmbH |
|
3 | 3 | # |
|
4 | 4 | # This program is free software; you can redistribute it and/or modify |
|
5 | 5 | # it under the terms of the GNU General Public License as published by |
|
6 | 6 | # the Free Software Foundation; either version 3 of the License, or |
|
7 | 7 | # (at your option) any later version. |
|
8 | 8 | # |
|
9 | 9 | # This program is distributed in the hope that it will be useful, |
|
10 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 | 12 | # GNU General Public License for more details. |
|
13 | 13 | # |
|
14 | 14 | # You should have received a copy of the GNU General Public License |
|
15 | 15 | # along with this program; if not, write to the Free Software Foundation, |
|
16 | 16 | # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
|
17 | 17 | |
|
18 | 18 | import os |
|
19 | 19 | import pytest |
|
20 | 20 | from webtest.app import TestApp as WebObTestApp |
|
21 | 21 | |
|
22 |
from vcsserver.lib. |
|
|
22 | from vcsserver.lib.ext_json import json | |
|
23 | 23 | from vcsserver.str_utils import safe_bytes |
|
24 | 24 | from vcsserver.git_lfs.app import create_app |
|
25 | 25 | from vcsserver.git_lfs.lib import LFSOidStore |
|
26 | 26 | |
|
27 | 27 | |
|
28 | 28 | @pytest.fixture(scope='function') |
|
29 | 29 | def git_lfs_app(tmpdir): |
|
30 | 30 | custom_app = WebObTestApp(create_app( |
|
31 | 31 | git_lfs_enabled=True, git_lfs_store_path=str(tmpdir), |
|
32 | 32 | git_lfs_http_scheme='http')) |
|
33 | 33 | custom_app._store = str(tmpdir) |
|
34 | 34 | return custom_app |
|
35 | 35 | |
|
36 | 36 | |
|
37 | 37 | @pytest.fixture(scope='function') |
|
38 | 38 | def git_lfs_https_app(tmpdir): |
|
39 | 39 | custom_app = WebObTestApp(create_app( |
|
40 | 40 | git_lfs_enabled=True, git_lfs_store_path=str(tmpdir), |
|
41 | 41 | git_lfs_http_scheme='https')) |
|
42 | 42 | custom_app._store = str(tmpdir) |
|
43 | 43 | return custom_app |
|
44 | 44 | |
|
45 | 45 | |
|
46 | 46 | @pytest.fixture() |
|
47 | 47 | def http_auth(): |
|
48 | 48 | return {'HTTP_AUTHORIZATION': "Basic XXXXX"} |
|
49 | 49 | |
|
50 | 50 | |
|
51 | 51 | class TestLFSApplication: |
|
52 | 52 | |
|
53 | 53 | def test_app_wrong_path(self, git_lfs_app): |
|
54 | 54 | git_lfs_app.get('/repo/info/lfs/xxx', status=404) |
|
55 | 55 | |
|
56 | 56 | def test_app_deprecated_endpoint(self, git_lfs_app): |
|
57 | 57 | response = git_lfs_app.post('/repo/info/lfs/objects', status=501) |
|
58 | 58 | assert response.status_code == 501 |
|
59 | 59 | assert json.loads(response.text) == {'message': 'LFS: v1 api not supported'} |
|
60 | 60 | |
|
61 | 61 | def test_app_lock_verify_api_not_available(self, git_lfs_app): |
|
62 | 62 | response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501) |
|
63 | 63 | assert response.status_code == 501 |
|
64 | 64 | assert json.loads(response.text) == { |
|
65 | 65 | 'message': 'GIT LFS locking api not supported'} |
|
66 | 66 | |
|
67 | 67 | def test_app_lock_api_not_available(self, git_lfs_app): |
|
68 | 68 | response = git_lfs_app.post('/repo/info/lfs/locks', status=501) |
|
69 | 69 | assert response.status_code == 501 |
|
70 | 70 | assert json.loads(response.text) == { |
|
71 | 71 | 'message': 'GIT LFS locking api not supported'} |
|
72 | 72 | |
|
73 | 73 | def test_app_batch_api_missing_auth(self, git_lfs_app): |
|
74 | 74 | git_lfs_app.post_json( |
|
75 | 75 | '/repo/info/lfs/objects/batch', params={}, status=403) |
|
76 | 76 | |
|
77 | 77 | def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth): |
|
78 | 78 | response = git_lfs_app.post_json( |
|
79 | 79 | '/repo/info/lfs/objects/batch', params={}, status=400, |
|
80 | 80 | extra_environ=http_auth) |
|
81 | 81 | assert json.loads(response.text) == { |
|
82 | 82 | 'message': 'unsupported operation mode: `None`'} |
|
83 | 83 | |
|
84 | 84 | def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth): |
|
85 | 85 | response = git_lfs_app.post_json( |
|
86 | 86 | '/repo/info/lfs/objects/batch', params={'operation': 'download'}, |
|
87 | 87 | status=400, extra_environ=http_auth) |
|
88 | 88 | assert json.loads(response.text) == { |
|
89 | 89 | 'message': 'missing objects data'} |
|
90 | 90 | |
|
91 | 91 | def test_app_batch_api_unsupported_data_in_objects( |
|
92 | 92 | self, git_lfs_app, http_auth): |
|
93 | 93 | params = {'operation': 'download', |
|
94 | 94 | 'objects': [{}]} |
|
95 | 95 | response = git_lfs_app.post_json( |
|
96 | 96 | '/repo/info/lfs/objects/batch', params=params, status=400, |
|
97 | 97 | extra_environ=http_auth) |
|
98 | 98 | assert json.loads(response.text) == { |
|
99 | 99 | 'message': 'unsupported data in objects'} |
|
100 | 100 | |
|
101 | 101 | def test_app_batch_api_download_missing_object( |
|
102 | 102 | self, git_lfs_app, http_auth): |
|
103 | 103 | params = {'operation': 'download', |
|
104 | 104 | 'objects': [{'oid': '123', 'size': '1024'}]} |
|
105 | 105 | response = git_lfs_app.post_json( |
|
106 | 106 | '/repo/info/lfs/objects/batch', params=params, |
|
107 | 107 | extra_environ=http_auth) |
|
108 | 108 | |
|
109 | 109 | expected_objects = [ |
|
110 | 110 | {'authenticated': True, |
|
111 | 111 | 'errors': {'error': { |
|
112 | 112 | 'code': 404, |
|
113 | 113 | 'message': 'object: 123 does not exist in store'}}, |
|
114 | 114 | 'oid': '123', |
|
115 | 115 | 'size': '1024'} |
|
116 | 116 | ] |
|
117 | 117 | assert json.loads(response.text) == { |
|
118 | 118 | 'objects': expected_objects, 'transfer': 'basic'} |
|
119 | 119 | |
|
120 | 120 | def test_app_batch_api_download(self, git_lfs_app, http_auth): |
|
121 | 121 | oid = '456' |
|
122 | 122 | oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path |
|
123 | 123 | if not os.path.isdir(os.path.dirname(oid_path)): |
|
124 | 124 | os.makedirs(os.path.dirname(oid_path)) |
|
125 | 125 | with open(oid_path, 'wb') as f: |
|
126 | 126 | f.write(safe_bytes('OID_CONTENT')) |
|
127 | 127 | |
|
128 | 128 | params = {'operation': 'download', |
|
129 | 129 | 'objects': [{'oid': oid, 'size': '1024'}]} |
|
130 | 130 | response = git_lfs_app.post_json( |
|
131 | 131 | '/repo/info/lfs/objects/batch', params=params, |
|
132 | 132 | extra_environ=http_auth) |
|
133 | 133 | |
|
134 | 134 | expected_objects = [ |
|
135 | 135 | {'authenticated': True, |
|
136 | 136 | 'actions': { |
|
137 | 137 | 'download': { |
|
138 | 138 | 'header': {'Authorization': 'Basic XXXXX'}, |
|
139 | 139 | 'href': 'http://localhost/repo/info/lfs/objects/456'}, |
|
140 | 140 | }, |
|
141 | 141 | 'oid': '456', |
|
142 | 142 | 'size': '1024'} |
|
143 | 143 | ] |
|
144 | 144 | assert json.loads(response.text) == { |
|
145 | 145 | 'objects': expected_objects, 'transfer': 'basic'} |
|
146 | 146 | |
|
147 | 147 | def test_app_batch_api_upload(self, git_lfs_app, http_auth): |
|
148 | 148 | params = {'operation': 'upload', |
|
149 | 149 | 'objects': [{'oid': '123', 'size': '1024'}]} |
|
150 | 150 | response = git_lfs_app.post_json( |
|
151 | 151 | '/repo/info/lfs/objects/batch', params=params, |
|
152 | 152 | extra_environ=http_auth) |
|
153 | 153 | expected_objects = [ |
|
154 | 154 | {'authenticated': True, |
|
155 | 155 | 'actions': { |
|
156 | 156 | 'upload': { |
|
157 | 157 | 'header': {'Authorization': 'Basic XXXXX', |
|
158 | 158 | 'Transfer-Encoding': 'chunked'}, |
|
159 | 159 | 'href': 'http://localhost/repo/info/lfs/objects/123'}, |
|
160 | 160 | 'verify': { |
|
161 | 161 | 'header': {'Authorization': 'Basic XXXXX'}, |
|
162 | 162 | 'href': 'http://localhost/repo/info/lfs/verify'} |
|
163 | 163 | }, |
|
164 | 164 | 'oid': '123', |
|
165 | 165 | 'size': '1024'} |
|
166 | 166 | ] |
|
167 | 167 | assert json.loads(response.text) == { |
|
168 | 168 | 'objects': expected_objects, 'transfer': 'basic'} |
|
169 | 169 | |
|
170 | 170 | def test_app_batch_api_upload_for_https(self, git_lfs_https_app, http_auth): |
|
171 | 171 | params = {'operation': 'upload', |
|
172 | 172 | 'objects': [{'oid': '123', 'size': '1024'}]} |
|
173 | 173 | response = git_lfs_https_app.post_json( |
|
174 | 174 | '/repo/info/lfs/objects/batch', params=params, |
|
175 | 175 | extra_environ=http_auth) |
|
176 | 176 | expected_objects = [ |
|
177 | 177 | {'authenticated': True, |
|
178 | 178 | 'actions': { |
|
179 | 179 | 'upload': { |
|
180 | 180 | 'header': {'Authorization': 'Basic XXXXX', |
|
181 | 181 | 'Transfer-Encoding': 'chunked'}, |
|
182 | 182 | 'href': 'https://localhost/repo/info/lfs/objects/123'}, |
|
183 | 183 | 'verify': { |
|
184 | 184 | 'header': {'Authorization': 'Basic XXXXX'}, |
|
185 | 185 | 'href': 'https://localhost/repo/info/lfs/verify'} |
|
186 | 186 | }, |
|
187 | 187 | 'oid': '123', |
|
188 | 188 | 'size': '1024'} |
|
189 | 189 | ] |
|
190 | 190 | assert json.loads(response.text) == { |
|
191 | 191 | 'objects': expected_objects, 'transfer': 'basic'} |
|
192 | 192 | |
|
193 | 193 | def test_app_verify_api_missing_data(self, git_lfs_app): |
|
194 | 194 | params = {'oid': 'missing'} |
|
195 | 195 | response = git_lfs_app.post_json( |
|
196 | 196 | '/repo/info/lfs/verify', params=params, |
|
197 | 197 | status=400) |
|
198 | 198 | |
|
199 | 199 | assert json.loads(response.text) == { |
|
200 | 200 | 'message': 'missing oid and size in request data'} |
|
201 | 201 | |
|
202 | 202 | def test_app_verify_api_missing_obj(self, git_lfs_app): |
|
203 | 203 | params = {'oid': 'missing', 'size': '1024'} |
|
204 | 204 | response = git_lfs_app.post_json( |
|
205 | 205 | '/repo/info/lfs/verify', params=params, |
|
206 | 206 | status=404) |
|
207 | 207 | |
|
208 | 208 | assert json.loads(response.text) == { |
|
209 | 209 | 'message': 'oid `missing` does not exists in store'} |
|
210 | 210 | |
|
211 | 211 | def test_app_verify_api_size_mismatch(self, git_lfs_app): |
|
212 | 212 | oid = 'existing' |
|
213 | 213 | oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path |
|
214 | 214 | if not os.path.isdir(os.path.dirname(oid_path)): |
|
215 | 215 | os.makedirs(os.path.dirname(oid_path)) |
|
216 | 216 | with open(oid_path, 'wb') as f: |
|
217 | 217 | f.write(safe_bytes('OID_CONTENT')) |
|
218 | 218 | |
|
219 | 219 | params = {'oid': oid, 'size': '1024'} |
|
220 | 220 | response = git_lfs_app.post_json( |
|
221 | 221 | '/repo/info/lfs/verify', params=params, status=422) |
|
222 | 222 | |
|
223 | 223 | assert json.loads(response.text) == { |
|
224 | 224 | 'message': 'requested file size mismatch ' |
|
225 | 225 | 'store size:11 requested:1024'} |
|
226 | 226 | |
|
227 | 227 | def test_app_verify_api(self, git_lfs_app): |
|
228 | 228 | oid = 'existing' |
|
229 | 229 | oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path |
|
230 | 230 | if not os.path.isdir(os.path.dirname(oid_path)): |
|
231 | 231 | os.makedirs(os.path.dirname(oid_path)) |
|
232 | 232 | with open(oid_path, 'wb') as f: |
|
233 | 233 | f.write(safe_bytes('OID_CONTENT')) |
|
234 | 234 | |
|
235 | 235 | params = {'oid': oid, 'size': 11} |
|
236 | 236 | response = git_lfs_app.post_json( |
|
237 | 237 | '/repo/info/lfs/verify', params=params) |
|
238 | 238 | |
|
239 | 239 | assert json.loads(response.text) == { |
|
240 | 240 | 'message': {'size': 'ok', 'in_store': 'ok'}} |
|
241 | 241 | |
|
242 | 242 | def test_app_download_api_oid_not_existing(self, git_lfs_app): |
|
243 | 243 | oid = 'missing' |
|
244 | 244 | |
|
245 | 245 | response = git_lfs_app.get( |
|
246 | 246 | '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404) |
|
247 | 247 | |
|
248 | 248 | assert json.loads(response.text) == { |
|
249 | 249 | 'message': 'requested file with oid `missing` not found in store'} |
|
250 | 250 | |
|
251 | 251 | def test_app_download_api(self, git_lfs_app): |
|
252 | 252 | oid = 'existing' |
|
253 | 253 | oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path |
|
254 | 254 | if not os.path.isdir(os.path.dirname(oid_path)): |
|
255 | 255 | os.makedirs(os.path.dirname(oid_path)) |
|
256 | 256 | with open(oid_path, 'wb') as f: |
|
257 | 257 | f.write(safe_bytes('OID_CONTENT')) |
|
258 | 258 | |
|
259 | 259 | response = git_lfs_app.get( |
|
260 | 260 | '/repo/info/lfs/objects/{oid}'.format(oid=oid)) |
|
261 | 261 | assert response |
|
262 | 262 | |
|
263 | 263 | def test_app_upload(self, git_lfs_app): |
|
264 | 264 | oid = 'uploaded' |
|
265 | 265 | |
|
266 | 266 | response = git_lfs_app.put( |
|
267 | 267 | '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT') |
|
268 | 268 | |
|
269 | 269 | assert json.loads(response.text) == {'upload': 'ok'} |
|
270 | 270 | |
|
271 | 271 | # verify that we actually wrote that OID |
|
272 | 272 | oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path |
|
273 | 273 | assert os.path.isfile(oid_path) |
|
274 | 274 | assert 'CONTENT' == open(oid_path).read() |
@@ -1,832 +1,832 b'' | |||
|
1 | 1 | # RhodeCode VCSServer provides access to different vcs backends via network. |
|
2 | 2 | # Copyright (C) 2014-2023 RhodeCode GmbH |
|
3 | 3 | # |
|
4 | 4 | # This program is free software; you can redistribute it and/or modify |
|
5 | 5 | # it under the terms of the GNU General Public License as published by |
|
6 | 6 | # the Free Software Foundation; either version 3 of the License, or |
|
7 | 7 | # (at your option) any later version. |
|
8 | 8 | # |
|
9 | 9 | # This program is distributed in the hope that it will be useful, |
|
10 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 | 12 | # GNU General Public License for more details. |
|
13 | 13 | # |
|
14 | 14 | # You should have received a copy of the GNU General Public License |
|
15 | 15 | # along with this program; if not, write to the Free Software Foundation, |
|
16 | 16 | # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
|
17 | 17 | |
|
18 | 18 | import io |
|
19 | 19 | import os |
|
20 | 20 | import sys |
|
21 | 21 | import logging |
|
22 | 22 | import collections |
|
23 | 23 | import base64 |
|
24 | 24 | import msgpack |
|
25 | 25 | import dataclasses |
|
26 | 26 | import pygit2 |
|
27 | 27 | |
|
28 | 28 | import http.client |
|
29 | 29 | from celery import Celery |
|
30 | 30 | |
|
31 | 31 | import mercurial.scmutil |
|
32 | 32 | import mercurial.node |
|
33 | 33 | |
|
34 |
from vcsserver.lib. |
|
|
34 | from vcsserver.lib.ext_json import json | |
|
35 | 35 | from vcsserver import exceptions, subprocessio, settings |
|
36 | 36 | from vcsserver.str_utils import ascii_str, safe_str |
|
37 | 37 | from vcsserver.remote.git_remote import Repository |
|
38 | 38 | |
|
39 | 39 | celery_app = Celery('__vcsserver__') |
|
40 | 40 | log = logging.getLogger(__name__) |
|
41 | 41 | |
|
42 | 42 | |
|
43 | 43 | class HooksHttpClient: |
|
44 | 44 | proto = 'msgpack.v1' |
|
45 | 45 | connection = None |
|
46 | 46 | |
|
47 | 47 | def __init__(self, hooks_uri): |
|
48 | 48 | self.hooks_uri = hooks_uri |
|
49 | 49 | |
|
50 | 50 | def __repr__(self): |
|
51 | 51 | return f'{self.__class__}(hook_uri={self.hooks_uri}, proto={self.proto})' |
|
52 | 52 | |
|
53 | 53 | def __call__(self, method, extras): |
|
54 | 54 | connection = http.client.HTTPConnection(self.hooks_uri) |
|
55 | 55 | # binary msgpack body |
|
56 | 56 | headers, body = self._serialize(method, extras) |
|
57 | 57 | log.debug('Doing a new hooks call using HTTPConnection to %s', self.hooks_uri) |
|
58 | 58 | |
|
59 | 59 | try: |
|
60 | 60 | try: |
|
61 | 61 | connection.request('POST', '/', body, headers) |
|
62 | 62 | except Exception as error: |
|
63 | 63 | log.error('Hooks calling Connection failed on %s, org error: %s', connection.__dict__, error) |
|
64 | 64 | raise |
|
65 | 65 | |
|
66 | 66 | response = connection.getresponse() |
|
67 | 67 | try: |
|
68 | 68 | return msgpack.load(response) |
|
69 | 69 | except Exception: |
|
70 | 70 | response_data = response.read() |
|
71 | 71 | log.exception('Failed to decode hook response json data. ' |
|
72 | 72 | 'response_code:%s, raw_data:%s', |
|
73 | 73 | response.status, response_data) |
|
74 | 74 | raise |
|
75 | 75 | finally: |
|
76 | 76 | connection.close() |
|
77 | 77 | |
|
78 | 78 | @classmethod |
|
79 | 79 | def _serialize(cls, hook_name, extras): |
|
80 | 80 | data = { |
|
81 | 81 | 'method': hook_name, |
|
82 | 82 | 'extras': extras |
|
83 | 83 | } |
|
84 | 84 | headers = { |
|
85 | 85 | "rc-hooks-protocol": cls.proto, |
|
86 | 86 | "Connection": "keep-alive" |
|
87 | 87 | } |
|
88 | 88 | return headers, msgpack.packb(data) |
|
89 | 89 | |
|
90 | 90 | |
|
91 | 91 | class HooksCeleryClient: |
|
92 | 92 | TASK_TIMEOUT = 60 # time in seconds |
|
93 | 93 | |
|
94 | 94 | def __init__(self, queue, backend): |
|
95 | 95 | celery_app.config_from_object({ |
|
96 | 96 | 'broker_url': queue, 'result_backend': backend, |
|
97 | 97 | 'broker_connection_retry_on_startup': True, |
|
98 | 98 | 'task_serializer': 'json', |
|
99 | 99 | 'accept_content': ['json', 'msgpack'], |
|
100 | 100 | 'result_serializer': 'json', |
|
101 | 101 | 'result_accept_content': ['json', 'msgpack'] |
|
102 | 102 | }) |
|
103 | 103 | self.celery_app = celery_app |
|
104 | 104 | |
|
105 | 105 | def __call__(self, method, extras): |
|
106 | 106 | inquired_task = self.celery_app.signature( |
|
107 | 107 | f'rhodecode.lib.celerylib.tasks.{method}' |
|
108 | 108 | ) |
|
109 | 109 | return inquired_task.delay(extras).get(timeout=self.TASK_TIMEOUT) |
|
110 | 110 | |
|
111 | 111 | |
|
112 | 112 | class HooksShadowRepoClient: |
|
113 | 113 | |
|
114 | 114 | def __call__(self, hook_name, extras): |
|
115 | 115 | return {'output': '', 'status': 0} |
|
116 | 116 | |
|
117 | 117 | |
|
118 | 118 | class RemoteMessageWriter: |
|
119 | 119 | """Writer base class.""" |
|
120 | 120 | def write(self, message): |
|
121 | 121 | raise NotImplementedError() |
|
122 | 122 | |
|
123 | 123 | |
|
124 | 124 | class HgMessageWriter(RemoteMessageWriter): |
|
125 | 125 | """Writer that knows how to send messages to mercurial clients.""" |
|
126 | 126 | |
|
127 | 127 | def __init__(self, ui): |
|
128 | 128 | self.ui = ui |
|
129 | 129 | |
|
130 | 130 | def write(self, message: str): |
|
131 | 131 | # TODO: Check why the quiet flag is set by default. |
|
132 | 132 | old = self.ui.quiet |
|
133 | 133 | self.ui.quiet = False |
|
134 | 134 | self.ui.status(message.encode('utf-8')) |
|
135 | 135 | self.ui.quiet = old |
|
136 | 136 | |
|
137 | 137 | |
|
138 | 138 | class GitMessageWriter(RemoteMessageWriter): |
|
139 | 139 | """Writer that knows how to send messages to git clients.""" |
|
140 | 140 | |
|
141 | 141 | def __init__(self, stdout=None): |
|
142 | 142 | self.stdout = stdout or sys.stdout |
|
143 | 143 | |
|
144 | 144 | def write(self, message: str): |
|
145 | 145 | self.stdout.write(message) |
|
146 | 146 | |
|
147 | 147 | |
|
148 | 148 | class SvnMessageWriter(RemoteMessageWriter): |
|
149 | 149 | """Writer that knows how to send messages to svn clients.""" |
|
150 | 150 | |
|
151 | 151 | def __init__(self, stderr=None): |
|
152 | 152 | # SVN needs data sent to stderr for back-to-client messaging |
|
153 | 153 | self.stderr = stderr or sys.stderr |
|
154 | 154 | |
|
155 | 155 | def write(self, message): |
|
156 | 156 | self.stderr.write(message) |
|
157 | 157 | |
|
158 | 158 | |
|
159 | 159 | def _handle_exception(result): |
|
160 | 160 | exception_class = result.get('exception') |
|
161 | 161 | exception_traceback = result.get('exception_traceback') |
|
162 | 162 | log.debug('Handling hook-call exception: %s', exception_class) |
|
163 | 163 | |
|
164 | 164 | if exception_traceback: |
|
165 | 165 | log.error('Got traceback from remote call:%s', exception_traceback) |
|
166 | 166 | |
|
167 | 167 | if exception_class == 'HTTPLockedRC': |
|
168 | 168 | raise exceptions.RepositoryLockedException()(*result['exception_args']) |
|
169 | 169 | elif exception_class == 'HTTPBranchProtected': |
|
170 | 170 | raise exceptions.RepositoryBranchProtectedException()(*result['exception_args']) |
|
171 | 171 | elif exception_class == 'RepositoryError': |
|
172 | 172 | raise exceptions.VcsException()(*result['exception_args']) |
|
173 | 173 | elif exception_class: |
|
174 | 174 | raise Exception( |
|
175 | 175 | f"""Got remote exception "{exception_class}" with args "{result['exception_args']}" """ |
|
176 | 176 | ) |
|
177 | 177 | |
|
178 | 178 | |
|
179 | 179 | def _get_hooks_client(extras): |
|
180 | 180 | hooks_uri = extras.get('hooks_uri') |
|
181 | 181 | task_queue = extras.get('task_queue') |
|
182 | 182 | task_backend = extras.get('task_backend') |
|
183 | 183 | is_shadow_repo = extras.get('is_shadow_repo') |
|
184 | 184 | |
|
185 | 185 | if hooks_uri: |
|
186 | 186 | return HooksHttpClient(hooks_uri) |
|
187 | 187 | elif task_queue and task_backend: |
|
188 | 188 | return HooksCeleryClient(task_queue, task_backend) |
|
189 | 189 | elif is_shadow_repo: |
|
190 | 190 | return HooksShadowRepoClient() |
|
191 | 191 | else: |
|
192 | 192 | raise Exception("Hooks client not found!") |
|
193 | 193 | |
|
194 | 194 | |
|
195 | 195 | def _call_hook(hook_name, extras, writer): |
|
196 | 196 | hooks_client = _get_hooks_client(extras) |
|
197 | 197 | log.debug('Hooks, using client:%s', hooks_client) |
|
198 | 198 | result = hooks_client(hook_name, extras) |
|
199 | 199 | log.debug('Hooks got result: %s', result) |
|
200 | 200 | _handle_exception(result) |
|
201 | 201 | writer.write(result['output']) |
|
202 | 202 | |
|
203 | 203 | return result['status'] |
|
204 | 204 | |
|
205 | 205 | |
|
206 | 206 | def _extras_from_ui(ui): |
|
207 | 207 | hook_data = ui.config(b'rhodecode', b'RC_SCM_DATA') |
|
208 | 208 | if not hook_data: |
|
209 | 209 | # maybe it's inside environ ? |
|
210 | 210 | env_hook_data = os.environ.get('RC_SCM_DATA') |
|
211 | 211 | if env_hook_data: |
|
212 | 212 | hook_data = env_hook_data |
|
213 | 213 | |
|
214 | 214 | extras = {} |
|
215 | 215 | if hook_data: |
|
216 | 216 | extras = json.loads(hook_data) |
|
217 | 217 | return extras |
|
218 | 218 | |
|
219 | 219 | |
|
220 | 220 | def _rev_range_hash(repo, node, check_heads=False): |
|
221 | 221 | from vcsserver.hgcompat import get_ctx |
|
222 | 222 | |
|
223 | 223 | commits = [] |
|
224 | 224 | revs = [] |
|
225 | 225 | start = get_ctx(repo, node).rev() |
|
226 | 226 | end = len(repo) |
|
227 | 227 | for rev in range(start, end): |
|
228 | 228 | revs.append(rev) |
|
229 | 229 | ctx = get_ctx(repo, rev) |
|
230 | 230 | commit_id = ascii_str(mercurial.node.hex(ctx.node())) |
|
231 | 231 | branch = safe_str(ctx.branch()) |
|
232 | 232 | commits.append((commit_id, branch)) |
|
233 | 233 | |
|
234 | 234 | parent_heads = [] |
|
235 | 235 | if check_heads: |
|
236 | 236 | parent_heads = _check_heads(repo, start, end, revs) |
|
237 | 237 | return commits, parent_heads |
|
238 | 238 | |
|
239 | 239 | |
|
240 | 240 | def _check_heads(repo, start, end, commits): |
|
241 | 241 | from vcsserver.hgcompat import get_ctx |
|
242 | 242 | changelog = repo.changelog |
|
243 | 243 | parents = set() |
|
244 | 244 | |
|
245 | 245 | for new_rev in commits: |
|
246 | 246 | for p in changelog.parentrevs(new_rev): |
|
247 | 247 | if p == mercurial.node.nullrev: |
|
248 | 248 | continue |
|
249 | 249 | if p < start: |
|
250 | 250 | parents.add(p) |
|
251 | 251 | |
|
252 | 252 | for p in parents: |
|
253 | 253 | branch = get_ctx(repo, p).branch() |
|
254 | 254 | # The heads descending from that parent, on the same branch |
|
255 | 255 | parent_heads = {p} |
|
256 | 256 | reachable = {p} |
|
257 | 257 | for x in range(p + 1, end): |
|
258 | 258 | if get_ctx(repo, x).branch() != branch: |
|
259 | 259 | continue |
|
260 | 260 | for pp in changelog.parentrevs(x): |
|
261 | 261 | if pp in reachable: |
|
262 | 262 | reachable.add(x) |
|
263 | 263 | parent_heads.discard(pp) |
|
264 | 264 | parent_heads.add(x) |
|
265 | 265 | # More than one head? Suggest merging |
|
266 | 266 | if len(parent_heads) > 1: |
|
267 | 267 | return list(parent_heads) |
|
268 | 268 | |
|
269 | 269 | return [] |
|
270 | 270 | |
|
271 | 271 | |
|
272 | 272 | def _get_git_env(): |
|
273 | 273 | env = {} |
|
274 | 274 | for k, v in os.environ.items(): |
|
275 | 275 | if k.startswith('GIT'): |
|
276 | 276 | env[k] = v |
|
277 | 277 | |
|
278 | 278 | # serialized version |
|
279 | 279 | return [(k, v) for k, v in env.items()] |
|
280 | 280 | |
|
281 | 281 | |
|
282 | 282 | def _get_hg_env(old_rev, new_rev, txnid, repo_path): |
|
283 | 283 | env = {} |
|
284 | 284 | for k, v in os.environ.items(): |
|
285 | 285 | if k.startswith('HG'): |
|
286 | 286 | env[k] = v |
|
287 | 287 | |
|
288 | 288 | env['HG_NODE'] = old_rev |
|
289 | 289 | env['HG_NODE_LAST'] = new_rev |
|
290 | 290 | env['HG_TXNID'] = txnid |
|
291 | 291 | env['HG_PENDING'] = repo_path |
|
292 | 292 | |
|
293 | 293 | return [(k, v) for k, v in env.items()] |
|
294 | 294 | |
|
295 | 295 | |
|
296 | 296 | def _fix_hooks_executables(ini_path=''): |
|
297 | 297 | """ |
|
298 | 298 | This is a trick to set proper settings.EXECUTABLE paths for certain execution patterns |
|
299 | 299 | especially for subversion where hooks strip entire env, and calling just 'svn' command will most likely fail |
|
300 | 300 | because svn is not on PATH |
|
301 | 301 | """ |
|
302 | 302 | from vcsserver.http_main import sanitize_settings_and_apply_defaults |
|
303 | 303 | from vcsserver.lib.config_utils import get_app_config_lightweight |
|
304 | 304 | |
|
305 | 305 | core_binary_dir = settings.BINARY_DIR or '/usr/local/bin/rhodecode_bin/vcs_bin' |
|
306 | 306 | if ini_path: |
|
307 | 307 | |
|
308 | 308 | ini_settings = get_app_config_lightweight(ini_path) |
|
309 | 309 | ini_settings = sanitize_settings_and_apply_defaults({'__file__': ini_path}, ini_settings) |
|
310 | 310 | core_binary_dir = ini_settings['core.binary_dir'] |
|
311 | 311 | |
|
312 | 312 | settings.BINARY_DIR = core_binary_dir |
|
313 | 313 | |
|
314 | 314 | |
|
315 | 315 | def repo_size(ui, repo, **kwargs): |
|
316 | 316 | extras = _extras_from_ui(ui) |
|
317 | 317 | return _call_hook('repo_size', extras, HgMessageWriter(ui)) |
|
318 | 318 | |
|
319 | 319 | |
|
320 | 320 | def pre_pull(ui, repo, **kwargs): |
|
321 | 321 | extras = _extras_from_ui(ui) |
|
322 | 322 | return _call_hook('pre_pull', extras, HgMessageWriter(ui)) |
|
323 | 323 | |
|
324 | 324 | |
|
325 | 325 | def pre_pull_ssh(ui, repo, **kwargs): |
|
326 | 326 | extras = _extras_from_ui(ui) |
|
327 | 327 | if extras and extras.get('SSH'): |
|
328 | 328 | return pre_pull(ui, repo, **kwargs) |
|
329 | 329 | return 0 |
|
330 | 330 | |
|
331 | 331 | |
|
332 | 332 | def post_pull(ui, repo, **kwargs): |
|
333 | 333 | extras = _extras_from_ui(ui) |
|
334 | 334 | return _call_hook('post_pull', extras, HgMessageWriter(ui)) |
|
335 | 335 | |
|
336 | 336 | |
|
337 | 337 | def post_pull_ssh(ui, repo, **kwargs): |
|
338 | 338 | extras = _extras_from_ui(ui) |
|
339 | 339 | if extras and extras.get('SSH'): |
|
340 | 340 | return post_pull(ui, repo, **kwargs) |
|
341 | 341 | return 0 |
|
342 | 342 | |
|
343 | 343 | |
|
344 | 344 | def pre_push(ui, repo, node=None, **kwargs): |
|
345 | 345 | """ |
|
346 | 346 | Mercurial pre_push hook |
|
347 | 347 | """ |
|
348 | 348 | extras = _extras_from_ui(ui) |
|
349 | 349 | detect_force_push = extras.get('detect_force_push') |
|
350 | 350 | |
|
351 | 351 | rev_data = [] |
|
352 | 352 | hook_type: str = safe_str(kwargs.get('hooktype')) |
|
353 | 353 | |
|
354 | 354 | if node and hook_type == 'pretxnchangegroup': |
|
355 | 355 | branches = collections.defaultdict(list) |
|
356 | 356 | commits, _heads = _rev_range_hash(repo, node, check_heads=detect_force_push) |
|
357 | 357 | for commit_id, branch in commits: |
|
358 | 358 | branches[branch].append(commit_id) |
|
359 | 359 | |
|
360 | 360 | for branch, commits in branches.items(): |
|
361 | 361 | old_rev = ascii_str(kwargs.get('node_last')) or commits[0] |
|
362 | 362 | rev_data.append({ |
|
363 | 363 | 'total_commits': len(commits), |
|
364 | 364 | 'old_rev': old_rev, |
|
365 | 365 | 'new_rev': commits[-1], |
|
366 | 366 | 'ref': '', |
|
367 | 367 | 'type': 'branch', |
|
368 | 368 | 'name': branch, |
|
369 | 369 | }) |
|
370 | 370 | |
|
371 | 371 | for push_ref in rev_data: |
|
372 | 372 | push_ref['multiple_heads'] = _heads |
|
373 | 373 | |
|
374 | 374 | repo_path = os.path.join( |
|
375 | 375 | extras.get('repo_store', ''), extras.get('repository', '')) |
|
376 | 376 | push_ref['hg_env'] = _get_hg_env( |
|
377 | 377 | old_rev=push_ref['old_rev'], |
|
378 | 378 | new_rev=push_ref['new_rev'], txnid=ascii_str(kwargs.get('txnid')), |
|
379 | 379 | repo_path=repo_path) |
|
380 | 380 | |
|
381 | 381 | extras['hook_type'] = hook_type or 'pre_push' |
|
382 | 382 | extras['commit_ids'] = rev_data |
|
383 | 383 | |
|
384 | 384 | return _call_hook('pre_push', extras, HgMessageWriter(ui)) |
|
385 | 385 | |
|
386 | 386 | |
|
387 | 387 | def pre_push_ssh(ui, repo, node=None, **kwargs): |
|
388 | 388 | extras = _extras_from_ui(ui) |
|
389 | 389 | if extras.get('SSH'): |
|
390 | 390 | return pre_push(ui, repo, node, **kwargs) |
|
391 | 391 | |
|
392 | 392 | return 0 |
|
393 | 393 | |
|
394 | 394 | |
|
395 | 395 | def pre_push_ssh_auth(ui, repo, node=None, **kwargs): |
|
396 | 396 | """ |
|
397 | 397 | Mercurial pre_push hook for SSH |
|
398 | 398 | """ |
|
399 | 399 | extras = _extras_from_ui(ui) |
|
400 | 400 | if extras.get('SSH'): |
|
401 | 401 | permission = extras['SSH_PERMISSIONS'] |
|
402 | 402 | |
|
403 | 403 | if 'repository.write' == permission or 'repository.admin' == permission: |
|
404 | 404 | return 0 |
|
405 | 405 | |
|
406 | 406 | # non-zero ret code |
|
407 | 407 | return 1 |
|
408 | 408 | |
|
409 | 409 | return 0 |
|
410 | 410 | |
|
411 | 411 | |
|
412 | 412 | def post_push(ui, repo, node, **kwargs): |
|
413 | 413 | """ |
|
414 | 414 | Mercurial post_push hook |
|
415 | 415 | """ |
|
416 | 416 | extras = _extras_from_ui(ui) |
|
417 | 417 | |
|
418 | 418 | commit_ids = [] |
|
419 | 419 | branches = [] |
|
420 | 420 | bookmarks = [] |
|
421 | 421 | tags = [] |
|
422 | 422 | hook_type: str = safe_str(kwargs.get('hooktype')) |
|
423 | 423 | |
|
424 | 424 | commits, _heads = _rev_range_hash(repo, node) |
|
425 | 425 | for commit_id, branch in commits: |
|
426 | 426 | commit_ids.append(commit_id) |
|
427 | 427 | if branch not in branches: |
|
428 | 428 | branches.append(branch) |
|
429 | 429 | |
|
430 | 430 | if hasattr(ui, '_rc_pushkey_bookmarks'): |
|
431 | 431 | bookmarks = ui._rc_pushkey_bookmarks |
|
432 | 432 | |
|
433 | 433 | extras['hook_type'] = hook_type or 'post_push' |
|
434 | 434 | extras['commit_ids'] = commit_ids |
|
435 | 435 | |
|
436 | 436 | extras['new_refs'] = { |
|
437 | 437 | 'branches': branches, |
|
438 | 438 | 'bookmarks': bookmarks, |
|
439 | 439 | 'tags': tags |
|
440 | 440 | } |
|
441 | 441 | |
|
442 | 442 | return _call_hook('post_push', extras, HgMessageWriter(ui)) |
|
443 | 443 | |
|
444 | 444 | |
|
445 | 445 | def post_push_ssh(ui, repo, node, **kwargs): |
|
446 | 446 | """ |
|
447 | 447 | Mercurial post_push hook for SSH |
|
448 | 448 | """ |
|
449 | 449 | if _extras_from_ui(ui).get('SSH'): |
|
450 | 450 | return post_push(ui, repo, node, **kwargs) |
|
451 | 451 | return 0 |
|
452 | 452 | |
|
453 | 453 | |
|
454 | 454 | def key_push(ui, repo, **kwargs): |
|
455 | 455 | from vcsserver.hgcompat import get_ctx |
|
456 | 456 | |
|
457 | 457 | if kwargs['new'] != b'0' and kwargs['namespace'] == b'bookmarks': |
|
458 | 458 | # store new bookmarks in our UI object propagated later to post_push |
|
459 | 459 | ui._rc_pushkey_bookmarks = get_ctx(repo, kwargs['key']).bookmarks() |
|
460 | 460 | return |
|
461 | 461 | |
|
462 | 462 | |
|
463 | 463 | # backward compat |
|
464 | 464 | log_pull_action = post_pull |
|
465 | 465 | |
|
466 | 466 | # backward compat |
|
467 | 467 | log_push_action = post_push |
|
468 | 468 | |
|
469 | 469 | |
|
470 | 470 | def handle_git_pre_receive(unused_repo_path, unused_revs, unused_env): |
|
471 | 471 | """ |
|
472 | 472 | Old hook name: keep here for backward compatibility. |
|
473 | 473 | |
|
474 | 474 | This is only required when the installed git hooks are not upgraded. |
|
475 | 475 | """ |
|
476 | 476 | pass |
|
477 | 477 | |
|
478 | 478 | |
|
479 | 479 | def handle_git_post_receive(unused_repo_path, unused_revs, unused_env): |
|
480 | 480 | """ |
|
481 | 481 | Old hook name: keep here for backward compatibility. |
|
482 | 482 | |
|
483 | 483 | This is only required when the installed git hooks are not upgraded. |
|
484 | 484 | """ |
|
485 | 485 | pass |
|
486 | 486 | |
|
487 | 487 | |
|
488 | 488 | @dataclasses.dataclass |
|
489 | 489 | class HookResponse: |
|
490 | 490 | status: int |
|
491 | 491 | output: str |
|
492 | 492 | |
|
493 | 493 | |
|
494 | 494 | def git_pre_pull(extras) -> HookResponse: |
|
495 | 495 | """ |
|
496 | 496 | Pre pull hook. |
|
497 | 497 | |
|
498 | 498 | :param extras: dictionary containing the keys defined in simplevcs |
|
499 | 499 | :type extras: dict |
|
500 | 500 | |
|
501 | 501 | :return: status code of the hook. 0 for success. |
|
502 | 502 | :rtype: int |
|
503 | 503 | """ |
|
504 | 504 | |
|
505 | 505 | if 'pull' not in extras['hooks']: |
|
506 | 506 | return HookResponse(0, '') |
|
507 | 507 | |
|
508 | 508 | stdout = io.StringIO() |
|
509 | 509 | try: |
|
510 | 510 | status_code = _call_hook('pre_pull', extras, GitMessageWriter(stdout)) |
|
511 | 511 | |
|
512 | 512 | except Exception as error: |
|
513 | 513 | log.exception('Failed to call pre_pull hook') |
|
514 | 514 | status_code = 128 |
|
515 | 515 | stdout.write(f'ERROR: {error}\n') |
|
516 | 516 | |
|
517 | 517 | return HookResponse(status_code, stdout.getvalue()) |
|
518 | 518 | |
|
519 | 519 | |
|
520 | 520 | def git_post_pull(extras) -> HookResponse: |
|
521 | 521 | """ |
|
522 | 522 | Post pull hook. |
|
523 | 523 | |
|
524 | 524 | :param extras: dictionary containing the keys defined in simplevcs |
|
525 | 525 | :type extras: dict |
|
526 | 526 | |
|
527 | 527 | :return: status code of the hook. 0 for success. |
|
528 | 528 | :rtype: int |
|
529 | 529 | """ |
|
530 | 530 | if 'pull' not in extras['hooks']: |
|
531 | 531 | return HookResponse(0, '') |
|
532 | 532 | |
|
533 | 533 | stdout = io.StringIO() |
|
534 | 534 | try: |
|
535 | 535 | status = _call_hook('post_pull', extras, GitMessageWriter(stdout)) |
|
536 | 536 | except Exception as error: |
|
537 | 537 | status = 128 |
|
538 | 538 | stdout.write(f'ERROR: {error}\n') |
|
539 | 539 | |
|
540 | 540 | return HookResponse(status, stdout.getvalue()) |
|
541 | 541 | |
|
542 | 542 | |
|
543 | 543 | def _parse_git_ref_lines(revision_lines): |
|
544 | 544 | rev_data = [] |
|
545 | 545 | for revision_line in revision_lines or []: |
|
546 | 546 | old_rev, new_rev, ref = revision_line.strip().split(' ') |
|
547 | 547 | ref_data = ref.split('/', 2) |
|
548 | 548 | if ref_data[1] in ('tags', 'heads'): |
|
549 | 549 | rev_data.append({ |
|
550 | 550 | # NOTE(marcink): |
|
551 | 551 | # we're unable to tell total_commits for git at this point |
|
552 | 552 | # but we set the variable for consistency with GIT |
|
553 | 553 | 'total_commits': -1, |
|
554 | 554 | 'old_rev': old_rev, |
|
555 | 555 | 'new_rev': new_rev, |
|
556 | 556 | 'ref': ref, |
|
557 | 557 | 'type': ref_data[1], |
|
558 | 558 | 'name': ref_data[2], |
|
559 | 559 | }) |
|
560 | 560 | return rev_data |
|
561 | 561 | |
|
562 | 562 | |
|
563 | 563 | def git_pre_receive(unused_repo_path, revision_lines, env) -> int: |
|
564 | 564 | """ |
|
565 | 565 | Pre push hook. |
|
566 | 566 | |
|
567 | 567 | :return: status code of the hook. 0 for success. |
|
568 | 568 | """ |
|
569 | 569 | extras = json.loads(env['RC_SCM_DATA']) |
|
570 | 570 | rev_data = _parse_git_ref_lines(revision_lines) |
|
571 | 571 | if 'push' not in extras['hooks']: |
|
572 | 572 | return 0 |
|
573 | 573 | _fix_hooks_executables() |
|
574 | 574 | |
|
575 | 575 | empty_commit_id = '0' * 40 |
|
576 | 576 | |
|
577 | 577 | detect_force_push = extras.get('detect_force_push') |
|
578 | 578 | |
|
579 | 579 | for push_ref in rev_data: |
|
580 | 580 | # store our git-env which holds the temp store |
|
581 | 581 | push_ref['git_env'] = _get_git_env() |
|
582 | 582 | push_ref['pruned_sha'] = '' |
|
583 | 583 | if not detect_force_push: |
|
584 | 584 | # don't check for forced-push when we don't need to |
|
585 | 585 | continue |
|
586 | 586 | |
|
587 | 587 | type_ = push_ref['type'] |
|
588 | 588 | new_branch = push_ref['old_rev'] == empty_commit_id |
|
589 | 589 | delete_branch = push_ref['new_rev'] == empty_commit_id |
|
590 | 590 | if type_ == 'heads' and not (new_branch or delete_branch): |
|
591 | 591 | old_rev = push_ref['old_rev'] |
|
592 | 592 | new_rev = push_ref['new_rev'] |
|
593 | 593 | cmd = [settings.GIT_EXECUTABLE(), 'rev-list', old_rev, f'^{new_rev}'] |
|
594 | 594 | stdout, stderr = subprocessio.run_command( |
|
595 | 595 | cmd, env=os.environ.copy()) |
|
596 | 596 | # means we're having some non-reachable objects, this forced push was used |
|
597 | 597 | if stdout: |
|
598 | 598 | push_ref['pruned_sha'] = stdout.splitlines() |
|
599 | 599 | |
|
600 | 600 | extras['hook_type'] = 'pre_receive' |
|
601 | 601 | extras['commit_ids'] = rev_data |
|
602 | 602 | |
|
603 | 603 | stdout = sys.stdout |
|
604 | 604 | status_code = _call_hook('pre_push', extras, GitMessageWriter(stdout)) |
|
605 | 605 | |
|
606 | 606 | return status_code |
|
607 | 607 | |
|
608 | 608 | |
|
609 | 609 | def git_post_receive(unused_repo_path, revision_lines, env) -> int: |
|
610 | 610 | """ |
|
611 | 611 | Post push hook. |
|
612 | 612 | |
|
613 | 613 | :return: status code of the hook. 0 for success. |
|
614 | 614 | """ |
|
615 | 615 | extras = json.loads(env['RC_SCM_DATA']) |
|
616 | 616 | if 'push' not in extras['hooks']: |
|
617 | 617 | return 0 |
|
618 | 618 | |
|
619 | 619 | _fix_hooks_executables() |
|
620 | 620 | |
|
621 | 621 | rev_data = _parse_git_ref_lines(revision_lines) |
|
622 | 622 | |
|
623 | 623 | git_revs = [] |
|
624 | 624 | |
|
625 | 625 | # N.B.(skreft): it is ok to just call git, as git before calling a |
|
626 | 626 | # subcommand sets the PATH environment variable so that it point to the |
|
627 | 627 | # correct version of the git executable. |
|
628 | 628 | empty_commit_id = '0' * 40 |
|
629 | 629 | branches = [] |
|
630 | 630 | tags = [] |
|
631 | 631 | for push_ref in rev_data: |
|
632 | 632 | type_ = push_ref['type'] |
|
633 | 633 | |
|
634 | 634 | if type_ == 'heads': |
|
635 | 635 | # starting new branch case |
|
636 | 636 | if push_ref['old_rev'] == empty_commit_id: |
|
637 | 637 | push_ref_name = push_ref['name'] |
|
638 | 638 | |
|
639 | 639 | if push_ref_name not in branches: |
|
640 | 640 | branches.append(push_ref_name) |
|
641 | 641 | |
|
642 | 642 | need_head_set = '' |
|
643 | 643 | with Repository(os.getcwd()) as repo: |
|
644 | 644 | try: |
|
645 | 645 | repo.head |
|
646 | 646 | except pygit2.GitError: |
|
647 | 647 | need_head_set = f'refs/heads/{push_ref_name}' |
|
648 | 648 | |
|
649 | 649 | if need_head_set: |
|
650 | 650 | repo.set_head(need_head_set) |
|
651 | 651 | print(f"Setting default branch to {push_ref_name}") |
|
652 | 652 | |
|
653 | 653 | cmd = [settings.GIT_EXECUTABLE(), 'for-each-ref', '--format=%(refname)', 'refs/heads/*'] |
|
654 | 654 | stdout, stderr = subprocessio.run_command( |
|
655 | 655 | cmd, env=os.environ.copy()) |
|
656 | 656 | heads = safe_str(stdout) |
|
657 | 657 | heads = heads.replace(push_ref['ref'], '') |
|
658 | 658 | heads = ' '.join(head for head |
|
659 | 659 | in heads.splitlines() if head) or '.' |
|
660 | 660 | cmd = [settings.GIT_EXECUTABLE(), 'log', '--reverse', |
|
661 | 661 | '--pretty=format:%H', '--', push_ref['new_rev'], |
|
662 | 662 | '--not', heads] |
|
663 | 663 | stdout, stderr = subprocessio.run_command( |
|
664 | 664 | cmd, env=os.environ.copy()) |
|
665 | 665 | git_revs.extend(list(map(ascii_str, stdout.splitlines()))) |
|
666 | 666 | |
|
667 | 667 | # delete branch case |
|
668 | 668 | elif push_ref['new_rev'] == empty_commit_id: |
|
669 | 669 | git_revs.append(f'delete_branch=>{push_ref["name"]}') |
|
670 | 670 | else: |
|
671 | 671 | if push_ref['name'] not in branches: |
|
672 | 672 | branches.append(push_ref['name']) |
|
673 | 673 | |
|
674 | 674 | cmd = [settings.GIT_EXECUTABLE(), 'log', |
|
675 | 675 | f'{push_ref["old_rev"]}..{push_ref["new_rev"]}', |
|
676 | 676 | '--reverse', '--pretty=format:%H'] |
|
677 | 677 | stdout, stderr = subprocessio.run_command( |
|
678 | 678 | cmd, env=os.environ.copy()) |
|
679 | 679 | # we get bytes from stdout, we need str to be consistent |
|
680 | 680 | log_revs = list(map(ascii_str, stdout.splitlines())) |
|
681 | 681 | git_revs.extend(log_revs) |
|
682 | 682 | |
|
683 | 683 | # Pure pygit2 impl. but still 2-3x slower :/ |
|
684 | 684 | # results = [] |
|
685 | 685 | # |
|
686 | 686 | # with Repository(os.getcwd()) as repo: |
|
687 | 687 | # repo_new_rev = repo[push_ref['new_rev']] |
|
688 | 688 | # repo_old_rev = repo[push_ref['old_rev']] |
|
689 | 689 | # walker = repo.walk(repo_new_rev.id, pygit2.GIT_SORT_TOPOLOGICAL) |
|
690 | 690 | # |
|
691 | 691 | # for commit in walker: |
|
692 | 692 | # if commit.id == repo_old_rev.id: |
|
693 | 693 | # break |
|
694 | 694 | # results.append(commit.id.hex) |
|
695 | 695 | # # reverse the order, can't use GIT_SORT_REVERSE |
|
696 | 696 | # log_revs = results[::-1] |
|
697 | 697 | |
|
698 | 698 | elif type_ == 'tags': |
|
699 | 699 | if push_ref['name'] not in tags: |
|
700 | 700 | tags.append(push_ref['name']) |
|
701 | 701 | git_revs.append(f'tag=>{push_ref["name"]}') |
|
702 | 702 | |
|
703 | 703 | extras['hook_type'] = 'post_receive' |
|
704 | 704 | extras['commit_ids'] = git_revs |
|
705 | 705 | extras['new_refs'] = { |
|
706 | 706 | 'branches': branches, |
|
707 | 707 | 'bookmarks': [], |
|
708 | 708 | 'tags': tags, |
|
709 | 709 | } |
|
710 | 710 | |
|
711 | 711 | stdout = sys.stdout |
|
712 | 712 | |
|
713 | 713 | if 'repo_size' in extras['hooks']: |
|
714 | 714 | try: |
|
715 | 715 | _call_hook('repo_size', extras, GitMessageWriter(stdout)) |
|
716 | 716 | except Exception: |
|
717 | 717 | pass |
|
718 | 718 | |
|
719 | 719 | status_code = _call_hook('post_push', extras, GitMessageWriter(stdout)) |
|
720 | 720 | return status_code |
|
721 | 721 | |
|
722 | 722 | |
|
723 | 723 | def _get_extras_from_txn_id(path, txn_id): |
|
724 | 724 | _fix_hooks_executables() |
|
725 | 725 | |
|
726 | 726 | extras = {} |
|
727 | 727 | try: |
|
728 | 728 | cmd = [settings.SVNLOOK_EXECUTABLE(), 'pget', |
|
729 | 729 | '-t', txn_id, |
|
730 | 730 | '--revprop', path, 'rc-scm-extras'] |
|
731 | 731 | stdout, stderr = subprocessio.run_command( |
|
732 | 732 | cmd, env=os.environ.copy()) |
|
733 | 733 | extras = json.loads(base64.urlsafe_b64decode(stdout)) |
|
734 | 734 | except Exception: |
|
735 | 735 | log.exception('Failed to extract extras info from txn_id') |
|
736 | 736 | |
|
737 | 737 | return extras |
|
738 | 738 | |
|
739 | 739 | |
|
740 | 740 | def _get_extras_from_commit_id(commit_id, path): |
|
741 | 741 | _fix_hooks_executables() |
|
742 | 742 | |
|
743 | 743 | extras = {} |
|
744 | 744 | try: |
|
745 | 745 | cmd = [settings.SVNLOOK_EXECUTABLE(), 'pget', |
|
746 | 746 | '-r', commit_id, |
|
747 | 747 | '--revprop', path, 'rc-scm-extras'] |
|
748 | 748 | stdout, stderr = subprocessio.run_command( |
|
749 | 749 | cmd, env=os.environ.copy()) |
|
750 | 750 | extras = json.loads(base64.urlsafe_b64decode(stdout)) |
|
751 | 751 | except Exception: |
|
752 | 752 | log.exception('Failed to extract extras info from commit_id') |
|
753 | 753 | |
|
754 | 754 | return extras |
|
755 | 755 | |
|
756 | 756 | |
|
757 | 757 | def svn_pre_commit(repo_path, commit_data, env): |
|
758 | 758 | |
|
759 | 759 | path, txn_id = commit_data |
|
760 | 760 | branches = [] |
|
761 | 761 | tags = [] |
|
762 | 762 | |
|
763 | 763 | if env.get('RC_SCM_DATA'): |
|
764 | 764 | extras = json.loads(env['RC_SCM_DATA']) |
|
765 | 765 | else: |
|
766 | 766 | # fallback method to read from TXN-ID stored data |
|
767 | 767 | extras = _get_extras_from_txn_id(path, txn_id) |
|
768 | 768 | |
|
769 | 769 | if not extras: |
|
770 | 770 | #TODO: temporary fix until svn txn-id changes are merged |
|
771 | 771 | return 0 |
|
772 | 772 | raise ValueError('Failed to extract context data called extras for hook execution') |
|
773 | 773 | |
|
774 | 774 | extras['hook_type'] = 'pre_commit' |
|
775 | 775 | extras['commit_ids'] = [txn_id] |
|
776 | 776 | extras['txn_id'] = txn_id |
|
777 | 777 | extras['new_refs'] = { |
|
778 | 778 | 'total_commits': 1, |
|
779 | 779 | 'branches': branches, |
|
780 | 780 | 'bookmarks': [], |
|
781 | 781 | 'tags': tags, |
|
782 | 782 | } |
|
783 | 783 | |
|
784 | 784 | return _call_hook('pre_push', extras, SvnMessageWriter()) |
|
785 | 785 | |
|
786 | 786 | |
|
787 | 787 | def svn_post_commit(repo_path, commit_data, env): |
|
788 | 788 | """ |
|
789 | 789 | commit_data is path, rev, txn_id |
|
790 | 790 | """ |
|
791 | 791 | |
|
792 | 792 | if len(commit_data) == 3: |
|
793 | 793 | path, commit_id, txn_id = commit_data |
|
794 | 794 | elif len(commit_data) == 2: |
|
795 | 795 | log.error('Failed to extract txn_id from commit_data using legacy method. ' |
|
796 | 796 | 'Some functionality might be limited') |
|
797 | 797 | path, commit_id = commit_data |
|
798 | 798 | txn_id = None |
|
799 | 799 | else: |
|
800 | 800 | return 0 |
|
801 | 801 | |
|
802 | 802 | branches = [] |
|
803 | 803 | tags = [] |
|
804 | 804 | |
|
805 | 805 | if env.get('RC_SCM_DATA'): |
|
806 | 806 | extras = json.loads(env['RC_SCM_DATA']) |
|
807 | 807 | else: |
|
808 | 808 | # fallback method to read from TXN-ID stored data |
|
809 | 809 | extras = _get_extras_from_commit_id(commit_id, path) |
|
810 | 810 | |
|
811 | 811 | if not extras: |
|
812 | 812 | #TODO: temporary fix until svn txn-id changes are merged |
|
813 | 813 | return 0 |
|
814 | 814 | raise ValueError('Failed to extract context data called extras for hook execution') |
|
815 | 815 | |
|
816 | 816 | extras['hook_type'] = 'post_commit' |
|
817 | 817 | extras['commit_ids'] = [commit_id] |
|
818 | 818 | extras['txn_id'] = txn_id |
|
819 | 819 | extras['new_refs'] = { |
|
820 | 820 | 'branches': branches, |
|
821 | 821 | 'bookmarks': [], |
|
822 | 822 | 'tags': tags, |
|
823 | 823 | 'total_commits': 1, |
|
824 | 824 | } |
|
825 | 825 | |
|
826 | 826 | if 'repo_size' in extras['hooks']: |
|
827 | 827 | try: |
|
828 | 828 | _call_hook('repo_size', extras, SvnMessageWriter()) |
|
829 | 829 | except Exception: |
|
830 | 830 | pass |
|
831 | 831 | |
|
832 | 832 | return _call_hook('post_push', extras, SvnMessageWriter()) |
@@ -1,774 +1,774 b'' | |||
|
1 | 1 | # RhodeCode VCSServer provides access to different vcs backends via network. |
|
2 | 2 | # Copyright (C) 2014-2023 RhodeCode GmbH |
|
3 | 3 | # |
|
4 | 4 | # This program is free software; you can redistribute it and/or modify |
|
5 | 5 | # it under the terms of the GNU General Public License as published by |
|
6 | 6 | # the Free Software Foundation; either version 3 of the License, or |
|
7 | 7 | # (at your option) any later version. |
|
8 | 8 | # |
|
9 | 9 | # This program is distributed in the hope that it will be useful, |
|
10 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 | 12 | # GNU General Public License for more details. |
|
13 | 13 | # |
|
14 | 14 | # You should have received a copy of the GNU General Public License |
|
15 | 15 | # along with this program; if not, write to the Free Software Foundation, |
|
16 | 16 | # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
|
17 | 17 | |
|
18 | 18 | import io |
|
19 | 19 | import os |
|
20 | 20 | import platform |
|
21 | 21 | import sys |
|
22 | 22 | import locale |
|
23 | 23 | import logging |
|
24 | 24 | import uuid |
|
25 | 25 | import time |
|
26 | 26 | import wsgiref.util |
|
27 | 27 | import tempfile |
|
28 | 28 | import psutil |
|
29 | 29 | |
|
30 | 30 | from itertools import chain |
|
31 | 31 | |
|
32 | 32 | import msgpack |
|
33 | 33 | import configparser |
|
34 | 34 | |
|
35 | 35 | from pyramid.config import Configurator |
|
36 | 36 | from pyramid.wsgi import wsgiapp |
|
37 | 37 | from pyramid.response import Response |
|
38 | 38 | |
|
39 | 39 | from vcsserver.base import BytesEnvelope, BinaryEnvelope |
|
40 |
from vcsserver.lib. |
|
|
40 | from vcsserver.lib.ext_json import json | |
|
41 | 41 | from vcsserver.config.settings_maker import SettingsMaker |
|
42 | 42 | from vcsserver.str_utils import safe_int |
|
43 | 43 | from vcsserver.lib.statsd_client import StatsdClient |
|
44 | 44 | from vcsserver.tweens.request_wrapper import get_headers_call_context |
|
45 | 45 | |
|
46 | 46 | import vcsserver |
|
47 | 47 | from vcsserver import remote_wsgi, scm_app, settings, hgpatches |
|
48 | 48 | from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT |
|
49 | 49 | from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub |
|
50 | 50 | from vcsserver.echo_stub.echo_app import EchoApp |
|
51 | 51 | from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected |
|
52 | 52 | from vcsserver.lib.exc_tracking import store_exception, format_exc |
|
53 | 53 | from vcsserver.server import VcsServer |
|
54 | 54 | |
|
55 | 55 | strict_vcs = True |
|
56 | 56 | |
|
57 | 57 | git_import_err = None |
|
58 | 58 | try: |
|
59 | 59 | from vcsserver.remote.git_remote import GitFactory, GitRemote |
|
60 | 60 | except ImportError as e: |
|
61 | 61 | GitFactory = None |
|
62 | 62 | GitRemote = None |
|
63 | 63 | git_import_err = e |
|
64 | 64 | if strict_vcs: |
|
65 | 65 | raise |
|
66 | 66 | |
|
67 | 67 | |
|
68 | 68 | hg_import_err = None |
|
69 | 69 | try: |
|
70 | 70 | from vcsserver.remote.hg_remote import MercurialFactory, HgRemote |
|
71 | 71 | except ImportError as e: |
|
72 | 72 | MercurialFactory = None |
|
73 | 73 | HgRemote = None |
|
74 | 74 | hg_import_err = e |
|
75 | 75 | if strict_vcs: |
|
76 | 76 | raise |
|
77 | 77 | |
|
78 | 78 | |
|
79 | 79 | svn_import_err = None |
|
80 | 80 | try: |
|
81 | 81 | from vcsserver.remote.svn_remote import SubversionFactory, SvnRemote |
|
82 | 82 | except ImportError as e: |
|
83 | 83 | SubversionFactory = None |
|
84 | 84 | SvnRemote = None |
|
85 | 85 | svn_import_err = e |
|
86 | 86 | if strict_vcs: |
|
87 | 87 | raise |
|
88 | 88 | |
|
89 | 89 | log = logging.getLogger(__name__) |
|
90 | 90 | |
|
91 | 91 | # due to Mercurial/glibc2.27 problems we need to detect if locale settings are |
|
92 | 92 | # causing problems and "fix" it in case they do and fallback to LC_ALL = C |
|
93 | 93 | |
|
94 | 94 | try: |
|
95 | 95 | locale.setlocale(locale.LC_ALL, '') |
|
96 | 96 | except locale.Error as e: |
|
97 | 97 | log.error( |
|
98 | 98 | 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e) |
|
99 | 99 | os.environ['LC_ALL'] = 'C' |
|
100 | 100 | |
|
101 | 101 | |
|
102 | 102 | def _is_request_chunked(environ): |
|
103 | 103 | stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked' |
|
104 | 104 | return stream |
|
105 | 105 | |
|
106 | 106 | |
|
107 | 107 | def log_max_fd(): |
|
108 | 108 | try: |
|
109 | 109 | maxfd = psutil.Process().rlimit(psutil.RLIMIT_NOFILE)[1] |
|
110 | 110 | log.info('Max file descriptors value: %s', maxfd) |
|
111 | 111 | except Exception: |
|
112 | 112 | pass |
|
113 | 113 | |
|
114 | 114 | |
|
115 | 115 | class VCS: |
|
116 | 116 | def __init__(self, locale_conf=None, cache_config=None): |
|
117 | 117 | self.locale = locale_conf |
|
118 | 118 | self.cache_config = cache_config |
|
119 | 119 | self._configure_locale() |
|
120 | 120 | |
|
121 | 121 | log_max_fd() |
|
122 | 122 | |
|
123 | 123 | if GitFactory and GitRemote: |
|
124 | 124 | git_factory = GitFactory() |
|
125 | 125 | self._git_remote = GitRemote(git_factory) |
|
126 | 126 | else: |
|
127 | 127 | log.error("Git client import failed: %s", git_import_err) |
|
128 | 128 | |
|
129 | 129 | if MercurialFactory and HgRemote: |
|
130 | 130 | hg_factory = MercurialFactory() |
|
131 | 131 | self._hg_remote = HgRemote(hg_factory) |
|
132 | 132 | else: |
|
133 | 133 | log.error("Mercurial client import failed: %s", hg_import_err) |
|
134 | 134 | |
|
135 | 135 | if SubversionFactory and SvnRemote: |
|
136 | 136 | svn_factory = SubversionFactory() |
|
137 | 137 | |
|
138 | 138 | # hg factory is used for svn url validation |
|
139 | 139 | hg_factory = MercurialFactory() |
|
140 | 140 | self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory) |
|
141 | 141 | else: |
|
142 | 142 | log.error("Subversion client import failed: %s", svn_import_err) |
|
143 | 143 | |
|
144 | 144 | self._vcsserver = VcsServer() |
|
145 | 145 | |
|
146 | 146 | def _configure_locale(self): |
|
147 | 147 | if self.locale: |
|
148 | 148 | log.info('Settings locale: `LC_ALL` to %s', self.locale) |
|
149 | 149 | else: |
|
150 | 150 | log.info('Configuring locale subsystem based on environment variables') |
|
151 | 151 | try: |
|
152 | 152 | # If self.locale is the empty string, then the locale |
|
153 | 153 | # module will use the environment variables. See the |
|
154 | 154 | # documentation of the package `locale`. |
|
155 | 155 | locale.setlocale(locale.LC_ALL, self.locale) |
|
156 | 156 | |
|
157 | 157 | language_code, encoding = locale.getlocale() |
|
158 | 158 | log.info( |
|
159 | 159 | 'Locale set to language code "%s" with encoding "%s".', |
|
160 | 160 | language_code, encoding) |
|
161 | 161 | except locale.Error: |
|
162 | 162 | log.exception('Cannot set locale, not configuring the locale system') |
|
163 | 163 | |
|
164 | 164 | |
|
165 | 165 | class WsgiProxy: |
|
166 | 166 | def __init__(self, wsgi): |
|
167 | 167 | self.wsgi = wsgi |
|
168 | 168 | |
|
169 | 169 | def __call__(self, environ, start_response): |
|
170 | 170 | input_data = environ['wsgi.input'].read() |
|
171 | 171 | input_data = msgpack.unpackb(input_data) |
|
172 | 172 | |
|
173 | 173 | error = None |
|
174 | 174 | try: |
|
175 | 175 | data, status, headers = self.wsgi.handle( |
|
176 | 176 | input_data['environment'], input_data['input_data'], |
|
177 | 177 | *input_data['args'], **input_data['kwargs']) |
|
178 | 178 | except Exception as e: |
|
179 | 179 | data, status, headers = [], None, None |
|
180 | 180 | error = { |
|
181 | 181 | 'message': str(e), |
|
182 | 182 | '_vcs_kind': getattr(e, '_vcs_kind', None) |
|
183 | 183 | } |
|
184 | 184 | |
|
185 | 185 | start_response(200, {}) |
|
186 | 186 | return self._iterator(error, status, headers, data) |
|
187 | 187 | |
|
188 | 188 | def _iterator(self, error, status, headers, data): |
|
189 | 189 | initial_data = [ |
|
190 | 190 | error, |
|
191 | 191 | status, |
|
192 | 192 | headers, |
|
193 | 193 | ] |
|
194 | 194 | |
|
195 | 195 | for d in chain(initial_data, data): |
|
196 | 196 | yield msgpack.packb(d) |
|
197 | 197 | |
|
198 | 198 | |
|
199 | 199 | def not_found(request): |
|
200 | 200 | return {'status': '404 NOT FOUND'} |
|
201 | 201 | |
|
202 | 202 | |
|
203 | 203 | class VCSViewPredicate: |
|
204 | 204 | def __init__(self, val, config): |
|
205 | 205 | self.remotes = val |
|
206 | 206 | |
|
207 | 207 | def text(self): |
|
208 | 208 | return f'vcs view method = {list(self.remotes.keys())}' |
|
209 | 209 | |
|
210 | 210 | phash = text |
|
211 | 211 | |
|
212 | 212 | def __call__(self, context, request): |
|
213 | 213 | """ |
|
214 | 214 | View predicate that returns true if given backend is supported by |
|
215 | 215 | defined remotes. |
|
216 | 216 | """ |
|
217 | 217 | backend = request.matchdict.get('backend') |
|
218 | 218 | return backend in self.remotes |
|
219 | 219 | |
|
220 | 220 | |
|
221 | 221 | class HTTPApplication: |
|
222 | 222 | ALLOWED_EXCEPTIONS = ('KeyError', 'URLError') |
|
223 | 223 | |
|
224 | 224 | remote_wsgi = remote_wsgi |
|
225 | 225 | _use_echo_app = False |
|
226 | 226 | |
|
227 | 227 | def __init__(self, settings=None, global_config=None): |
|
228 | 228 | |
|
229 | 229 | self.config = Configurator(settings=settings) |
|
230 | 230 | # Init our statsd at very start |
|
231 | 231 | self.config.registry.statsd = StatsdClient.statsd |
|
232 | 232 | self.config.registry.vcs_call_context = {} |
|
233 | 233 | |
|
234 | 234 | self.global_config = global_config |
|
235 | 235 | self.config.include('vcsserver.lib.rc_cache') |
|
236 | 236 | self.config.include('vcsserver.lib.rc_cache.archive_cache') |
|
237 | 237 | |
|
238 | 238 | settings_locale = settings.get('locale', '') or 'en_US.UTF-8' |
|
239 | 239 | vcs = VCS(locale_conf=settings_locale, cache_config=settings) |
|
240 | 240 | self._remotes = { |
|
241 | 241 | 'hg': vcs._hg_remote, |
|
242 | 242 | 'git': vcs._git_remote, |
|
243 | 243 | 'svn': vcs._svn_remote, |
|
244 | 244 | 'server': vcs._vcsserver, |
|
245 | 245 | } |
|
246 | 246 | if settings.get('dev.use_echo_app', 'false').lower() == 'true': |
|
247 | 247 | self._use_echo_app = True |
|
248 | 248 | log.warning("Using EchoApp for VCS operations.") |
|
249 | 249 | self.remote_wsgi = remote_wsgi_stub |
|
250 | 250 | |
|
251 | 251 | self._configure_settings(global_config, settings) |
|
252 | 252 | |
|
253 | 253 | self._configure() |
|
254 | 254 | |
|
255 | 255 | def _configure_settings(self, global_config, app_settings): |
|
256 | 256 | """ |
|
257 | 257 | Configure the settings module. |
|
258 | 258 | """ |
|
259 | 259 | settings_merged = global_config.copy() |
|
260 | 260 | settings_merged.update(app_settings) |
|
261 | 261 | |
|
262 | 262 | binary_dir = app_settings['core.binary_dir'] |
|
263 | 263 | |
|
264 | 264 | settings.BINARY_DIR = binary_dir |
|
265 | 265 | |
|
266 | 266 | # Store the settings to make them available to other modules. |
|
267 | 267 | vcsserver.PYRAMID_SETTINGS = settings_merged |
|
268 | 268 | vcsserver.CONFIG = settings_merged |
|
269 | 269 | |
|
270 | 270 | def _configure(self): |
|
271 | 271 | self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory) |
|
272 | 272 | |
|
273 | 273 | self.config.add_route('service', '/_service') |
|
274 | 274 | self.config.add_route('status', '/status') |
|
275 | 275 | self.config.add_route('hg_proxy', '/proxy/hg') |
|
276 | 276 | self.config.add_route('git_proxy', '/proxy/git') |
|
277 | 277 | |
|
278 | 278 | # rpc methods |
|
279 | 279 | self.config.add_route('vcs', '/{backend}') |
|
280 | 280 | |
|
281 | 281 | # streaming rpc remote methods |
|
282 | 282 | self.config.add_route('vcs_stream', '/{backend}/stream') |
|
283 | 283 | |
|
284 | 284 | # vcs operations clone/push as streaming |
|
285 | 285 | self.config.add_route('stream_git', '/stream/git/*repo_name') |
|
286 | 286 | self.config.add_route('stream_hg', '/stream/hg/*repo_name') |
|
287 | 287 | |
|
288 | 288 | self.config.add_view(self.status_view, route_name='status', renderer='json') |
|
289 | 289 | self.config.add_view(self.service_view, route_name='service', renderer='msgpack') |
|
290 | 290 | |
|
291 | 291 | self.config.add_view(self.hg_proxy(), route_name='hg_proxy') |
|
292 | 292 | self.config.add_view(self.git_proxy(), route_name='git_proxy') |
|
293 | 293 | self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack', |
|
294 | 294 | vcs_view=self._remotes) |
|
295 | 295 | self.config.add_view(self.vcs_stream_view, route_name='vcs_stream', |
|
296 | 296 | vcs_view=self._remotes) |
|
297 | 297 | |
|
298 | 298 | self.config.add_view(self.hg_stream(), route_name='stream_hg') |
|
299 | 299 | self.config.add_view(self.git_stream(), route_name='stream_git') |
|
300 | 300 | |
|
301 | 301 | self.config.add_view_predicate('vcs_view', VCSViewPredicate) |
|
302 | 302 | |
|
303 | 303 | self.config.add_notfound_view(not_found, renderer='json') |
|
304 | 304 | |
|
305 | 305 | self.config.add_view(self.handle_vcs_exception, context=Exception) |
|
306 | 306 | |
|
307 | 307 | self.config.add_tween( |
|
308 | 308 | 'vcsserver.tweens.request_wrapper.RequestWrapperTween', |
|
309 | 309 | ) |
|
310 | 310 | self.config.add_request_method( |
|
311 | 311 | 'vcsserver.lib.request_counter.get_request_counter', |
|
312 | 312 | 'request_count') |
|
313 | 313 | |
|
314 | 314 | def wsgi_app(self): |
|
315 | 315 | return self.config.make_wsgi_app() |
|
316 | 316 | |
|
317 | 317 | def _vcs_view_params(self, request): |
|
318 | 318 | remote = self._remotes[request.matchdict['backend']] |
|
319 | 319 | payload = msgpack.unpackb(request.body, use_list=True) |
|
320 | 320 | |
|
321 | 321 | method = payload.get('method') |
|
322 | 322 | params = payload['params'] |
|
323 | 323 | wire = params.get('wire') |
|
324 | 324 | args = params.get('args') |
|
325 | 325 | kwargs = params.get('kwargs') |
|
326 | 326 | context_uid = None |
|
327 | 327 | |
|
328 | 328 | request.registry.vcs_call_context = { |
|
329 | 329 | 'method': method, |
|
330 | 330 | 'repo_name': payload.get('_repo_name'), |
|
331 | 331 | } |
|
332 | 332 | |
|
333 | 333 | if wire: |
|
334 | 334 | try: |
|
335 | 335 | wire['context'] = context_uid = uuid.UUID(wire['context']) |
|
336 | 336 | except KeyError: |
|
337 | 337 | pass |
|
338 | 338 | args.insert(0, wire) |
|
339 | 339 | repo_state_uid = wire.get('repo_state_uid') if wire else None |
|
340 | 340 | |
|
341 | 341 | # NOTE(marcink): trading complexity for slight performance |
|
342 | 342 | if log.isEnabledFor(logging.DEBUG): |
|
343 | 343 | # also we SKIP printing out any of those methods args since they maybe excessive |
|
344 | 344 | just_args_methods = { |
|
345 | 345 | 'commitctx': ('content', 'removed', 'updated'), |
|
346 | 346 | 'commit': ('content', 'removed', 'updated') |
|
347 | 347 | } |
|
348 | 348 | if method in just_args_methods: |
|
349 | 349 | skip_args = just_args_methods[method] |
|
350 | 350 | call_args = '' |
|
351 | 351 | call_kwargs = {} |
|
352 | 352 | for k in kwargs: |
|
353 | 353 | if k in skip_args: |
|
354 | 354 | # replace our skip key with dummy |
|
355 | 355 | call_kwargs[k] = f'RemovedParam({k})' |
|
356 | 356 | else: |
|
357 | 357 | call_kwargs[k] = kwargs[k] |
|
358 | 358 | else: |
|
359 | 359 | call_args = args[1:] |
|
360 | 360 | call_kwargs = kwargs |
|
361 | 361 | |
|
362 | 362 | log.debug('Method requested:`%s` with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s', |
|
363 | 363 | method, call_args, call_kwargs, context_uid, repo_state_uid) |
|
364 | 364 | |
|
365 | 365 | statsd = request.registry.statsd |
|
366 | 366 | if statsd: |
|
367 | 367 | statsd.incr( |
|
368 | 368 | 'vcsserver_method_total', tags=[ |
|
369 | 369 | f"method:{method}", |
|
370 | 370 | ]) |
|
371 | 371 | return payload, remote, method, args, kwargs |
|
372 | 372 | |
|
373 | 373 | def vcs_view(self, request): |
|
374 | 374 | |
|
375 | 375 | payload, remote, method, args, kwargs = self._vcs_view_params(request) |
|
376 | 376 | payload_id = payload.get('id') |
|
377 | 377 | |
|
378 | 378 | try: |
|
379 | 379 | resp = getattr(remote, method)(*args, **kwargs) |
|
380 | 380 | except Exception as e: |
|
381 | 381 | exc_info = list(sys.exc_info()) |
|
382 | 382 | exc_type, exc_value, exc_traceback = exc_info |
|
383 | 383 | |
|
384 | 384 | org_exc = getattr(e, '_org_exc', None) |
|
385 | 385 | org_exc_name = None |
|
386 | 386 | org_exc_tb = '' |
|
387 | 387 | if org_exc: |
|
388 | 388 | org_exc_name = org_exc.__class__.__name__ |
|
389 | 389 | org_exc_tb = getattr(e, '_org_exc_tb', '') |
|
390 | 390 | # replace our "faked" exception with our org |
|
391 | 391 | exc_info[0] = org_exc.__class__ |
|
392 | 392 | exc_info[1] = org_exc |
|
393 | 393 | |
|
394 | 394 | should_store_exc = True |
|
395 | 395 | if org_exc: |
|
396 | 396 | def get_exc_fqn(_exc_obj): |
|
397 | 397 | module_name = getattr(org_exc.__class__, '__module__', 'UNKNOWN') |
|
398 | 398 | return module_name + '.' + org_exc_name |
|
399 | 399 | |
|
400 | 400 | exc_fqn = get_exc_fqn(org_exc) |
|
401 | 401 | |
|
402 | 402 | if exc_fqn in ['mercurial.error.RepoLookupError', |
|
403 | 403 | 'vcsserver.exceptions.RefNotFoundException']: |
|
404 | 404 | should_store_exc = False |
|
405 | 405 | |
|
406 | 406 | if should_store_exc: |
|
407 | 407 | store_exception(id(exc_info), exc_info, request_path=request.path) |
|
408 | 408 | |
|
409 | 409 | tb_info = format_exc(exc_info) |
|
410 | 410 | |
|
411 | 411 | type_ = e.__class__.__name__ |
|
412 | 412 | if type_ not in self.ALLOWED_EXCEPTIONS: |
|
413 | 413 | type_ = None |
|
414 | 414 | |
|
415 | 415 | resp = { |
|
416 | 416 | 'id': payload_id, |
|
417 | 417 | 'error': { |
|
418 | 418 | 'message': str(e), |
|
419 | 419 | 'traceback': tb_info, |
|
420 | 420 | 'org_exc': org_exc_name, |
|
421 | 421 | 'org_exc_tb': org_exc_tb, |
|
422 | 422 | 'type': type_ |
|
423 | 423 | } |
|
424 | 424 | } |
|
425 | 425 | |
|
426 | 426 | try: |
|
427 | 427 | resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None) |
|
428 | 428 | except AttributeError: |
|
429 | 429 | pass |
|
430 | 430 | else: |
|
431 | 431 | resp = { |
|
432 | 432 | 'id': payload_id, |
|
433 | 433 | 'result': resp |
|
434 | 434 | } |
|
435 | 435 | log.debug('Serving data for method %s', method) |
|
436 | 436 | return resp |
|
437 | 437 | |
|
438 | 438 | def vcs_stream_view(self, request): |
|
439 | 439 | payload, remote, method, args, kwargs = self._vcs_view_params(request) |
|
440 | 440 | # this method has a stream: marker we remove it here |
|
441 | 441 | method = method.split('stream:')[-1] |
|
442 | 442 | chunk_size = safe_int(payload.get('chunk_size')) or 4096 |
|
443 | 443 | |
|
444 | 444 | resp = getattr(remote, method)(*args, **kwargs) |
|
445 | 445 | |
|
446 | 446 | def get_chunked_data(method_resp): |
|
447 | 447 | stream = io.BytesIO(method_resp) |
|
448 | 448 | while 1: |
|
449 | 449 | chunk = stream.read(chunk_size) |
|
450 | 450 | if not chunk: |
|
451 | 451 | break |
|
452 | 452 | yield chunk |
|
453 | 453 | |
|
454 | 454 | response = Response(app_iter=get_chunked_data(resp)) |
|
455 | 455 | response.content_type = 'application/octet-stream' |
|
456 | 456 | |
|
457 | 457 | return response |
|
458 | 458 | |
|
459 | 459 | def status_view(self, request): |
|
460 | 460 | import vcsserver |
|
461 | 461 | _platform_id = platform.uname()[1] or 'instance' |
|
462 | 462 | |
|
463 | 463 | return { |
|
464 | 464 | "status": "OK", |
|
465 | 465 | "vcsserver_version": vcsserver.get_version(), |
|
466 | 466 | "platform": _platform_id, |
|
467 | 467 | "pid": os.getpid(), |
|
468 | 468 | } |
|
469 | 469 | |
|
470 | 470 | def service_view(self, request): |
|
471 | 471 | import vcsserver |
|
472 | 472 | |
|
473 | 473 | payload = msgpack.unpackb(request.body, use_list=True) |
|
474 | 474 | server_config, app_config = {}, {} |
|
475 | 475 | |
|
476 | 476 | try: |
|
477 | 477 | path = self.global_config['__file__'] |
|
478 | 478 | config = configparser.RawConfigParser() |
|
479 | 479 | |
|
480 | 480 | config.read(path) |
|
481 | 481 | |
|
482 | 482 | if config.has_section('server:main'): |
|
483 | 483 | server_config = dict(config.items('server:main')) |
|
484 | 484 | if config.has_section('app:main'): |
|
485 | 485 | app_config = dict(config.items('app:main')) |
|
486 | 486 | |
|
487 | 487 | except Exception: |
|
488 | 488 | log.exception('Failed to read .ini file for display') |
|
489 | 489 | |
|
490 | 490 | environ = list(os.environ.items()) |
|
491 | 491 | |
|
492 | 492 | resp = { |
|
493 | 493 | 'id': payload.get('id'), |
|
494 | 494 | 'result': dict( |
|
495 | 495 | version=vcsserver.get_version(), |
|
496 | 496 | config=server_config, |
|
497 | 497 | app_config=app_config, |
|
498 | 498 | environ=environ, |
|
499 | 499 | payload=payload, |
|
500 | 500 | ) |
|
501 | 501 | } |
|
502 | 502 | return resp |
|
503 | 503 | |
|
504 | 504 | def _msgpack_renderer_factory(self, info): |
|
505 | 505 | |
|
506 | 506 | def _render(value, system): |
|
507 | 507 | bin_type = False |
|
508 | 508 | res = value.get('result') |
|
509 | 509 | if isinstance(res, BytesEnvelope): |
|
510 | 510 | log.debug('Result is wrapped in BytesEnvelope type') |
|
511 | 511 | bin_type = True |
|
512 | 512 | elif isinstance(res, BinaryEnvelope): |
|
513 | 513 | log.debug('Result is wrapped in BinaryEnvelope type') |
|
514 | 514 | value['result'] = res.val |
|
515 | 515 | bin_type = True |
|
516 | 516 | |
|
517 | 517 | request = system.get('request') |
|
518 | 518 | if request is not None: |
|
519 | 519 | response = request.response |
|
520 | 520 | ct = response.content_type |
|
521 | 521 | if ct == response.default_content_type: |
|
522 | 522 | response.content_type = 'application/x-msgpack' |
|
523 | 523 | if bin_type: |
|
524 | 524 | response.content_type = 'application/x-msgpack-bin' |
|
525 | 525 | |
|
526 | 526 | return msgpack.packb(value, use_bin_type=bin_type) |
|
527 | 527 | return _render |
|
528 | 528 | |
|
529 | 529 | def set_env_from_config(self, environ, config): |
|
530 | 530 | dict_conf = {} |
|
531 | 531 | try: |
|
532 | 532 | for elem in config: |
|
533 | 533 | if elem[0] == 'rhodecode': |
|
534 | 534 | dict_conf = json.loads(elem[2]) |
|
535 | 535 | break |
|
536 | 536 | except Exception: |
|
537 | 537 | log.exception('Failed to fetch SCM CONFIG') |
|
538 | 538 | return |
|
539 | 539 | |
|
540 | 540 | username = dict_conf.get('username') |
|
541 | 541 | if username: |
|
542 | 542 | environ['REMOTE_USER'] = username |
|
543 | 543 | # mercurial specific, some extension api rely on this |
|
544 | 544 | environ['HGUSER'] = username |
|
545 | 545 | |
|
546 | 546 | ip = dict_conf.get('ip') |
|
547 | 547 | if ip: |
|
548 | 548 | environ['REMOTE_HOST'] = ip |
|
549 | 549 | |
|
550 | 550 | if _is_request_chunked(environ): |
|
551 | 551 | # set the compatibility flag for webob |
|
552 | 552 | environ['wsgi.input_terminated'] = True |
|
553 | 553 | |
|
554 | 554 | def hg_proxy(self): |
|
555 | 555 | @wsgiapp |
|
556 | 556 | def _hg_proxy(environ, start_response): |
|
557 | 557 | app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi()) |
|
558 | 558 | return app(environ, start_response) |
|
559 | 559 | return _hg_proxy |
|
560 | 560 | |
|
561 | 561 | def git_proxy(self): |
|
562 | 562 | @wsgiapp |
|
563 | 563 | def _git_proxy(environ, start_response): |
|
564 | 564 | app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi()) |
|
565 | 565 | return app(environ, start_response) |
|
566 | 566 | return _git_proxy |
|
567 | 567 | |
|
568 | 568 | def hg_stream(self): |
|
569 | 569 | if self._use_echo_app: |
|
570 | 570 | @wsgiapp |
|
571 | 571 | def _hg_stream(environ, start_response): |
|
572 | 572 | app = EchoApp('fake_path', 'fake_name', None) |
|
573 | 573 | return app(environ, start_response) |
|
574 | 574 | return _hg_stream |
|
575 | 575 | else: |
|
576 | 576 | @wsgiapp |
|
577 | 577 | def _hg_stream(environ, start_response): |
|
578 | 578 | log.debug('http-app: handling hg stream') |
|
579 | 579 | call_context = get_headers_call_context(environ) |
|
580 | 580 | |
|
581 | 581 | repo_path = call_context['repo_path'] |
|
582 | 582 | repo_name = call_context['repo_name'] |
|
583 | 583 | config = call_context['repo_config'] |
|
584 | 584 | |
|
585 | 585 | app = scm_app.create_hg_wsgi_app( |
|
586 | 586 | repo_path, repo_name, config) |
|
587 | 587 | |
|
588 | 588 | # Consistent path information for hgweb |
|
589 | 589 | environ['PATH_INFO'] = call_context['path_info'] |
|
590 | 590 | environ['REPO_NAME'] = repo_name |
|
591 | 591 | self.set_env_from_config(environ, config) |
|
592 | 592 | |
|
593 | 593 | log.debug('http-app: starting app handler ' |
|
594 | 594 | 'with %s and process request', app) |
|
595 | 595 | return app(environ, ResponseFilter(start_response)) |
|
596 | 596 | return _hg_stream |
|
597 | 597 | |
|
598 | 598 | def git_stream(self): |
|
599 | 599 | if self._use_echo_app: |
|
600 | 600 | @wsgiapp |
|
601 | 601 | def _git_stream(environ, start_response): |
|
602 | 602 | app = EchoApp('fake_path', 'fake_name', None) |
|
603 | 603 | return app(environ, start_response) |
|
604 | 604 | return _git_stream |
|
605 | 605 | else: |
|
606 | 606 | @wsgiapp |
|
607 | 607 | def _git_stream(environ, start_response): |
|
608 | 608 | log.debug('http-app: handling git stream') |
|
609 | 609 | |
|
610 | 610 | call_context = get_headers_call_context(environ) |
|
611 | 611 | |
|
612 | 612 | repo_path = call_context['repo_path'] |
|
613 | 613 | repo_name = call_context['repo_name'] |
|
614 | 614 | config = call_context['repo_config'] |
|
615 | 615 | |
|
616 | 616 | environ['PATH_INFO'] = call_context['path_info'] |
|
617 | 617 | self.set_env_from_config(environ, config) |
|
618 | 618 | |
|
619 | 619 | content_type = environ.get('CONTENT_TYPE', '') |
|
620 | 620 | |
|
621 | 621 | path = environ['PATH_INFO'] |
|
622 | 622 | is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type |
|
623 | 623 | log.debug( |
|
624 | 624 | 'LFS: Detecting if request `%s` is LFS server path based ' |
|
625 | 625 | 'on content type:`%s`, is_lfs:%s', |
|
626 | 626 | path, content_type, is_lfs_request) |
|
627 | 627 | |
|
628 | 628 | if not is_lfs_request: |
|
629 | 629 | # fallback detection by path |
|
630 | 630 | if GIT_LFS_PROTO_PAT.match(path): |
|
631 | 631 | is_lfs_request = True |
|
632 | 632 | log.debug( |
|
633 | 633 | 'LFS: fallback detection by path of: `%s`, is_lfs:%s', |
|
634 | 634 | path, is_lfs_request) |
|
635 | 635 | |
|
636 | 636 | if is_lfs_request: |
|
637 | 637 | app = scm_app.create_git_lfs_wsgi_app( |
|
638 | 638 | repo_path, repo_name, config) |
|
639 | 639 | else: |
|
640 | 640 | app = scm_app.create_git_wsgi_app( |
|
641 | 641 | repo_path, repo_name, config) |
|
642 | 642 | |
|
643 | 643 | log.debug('http-app: starting app handler ' |
|
644 | 644 | 'with %s and process request', app) |
|
645 | 645 | |
|
646 | 646 | return app(environ, start_response) |
|
647 | 647 | |
|
648 | 648 | return _git_stream |
|
649 | 649 | |
|
650 | 650 | def handle_vcs_exception(self, exception, request): |
|
651 | 651 | _vcs_kind = getattr(exception, '_vcs_kind', '') |
|
652 | 652 | |
|
653 | 653 | if _vcs_kind == 'repo_locked': |
|
654 | 654 | headers_call_context = get_headers_call_context(request.environ) |
|
655 | 655 | status_code = safe_int(headers_call_context['locked_status_code']) |
|
656 | 656 | |
|
657 | 657 | return HTTPRepoLocked( |
|
658 | 658 | title=str(exception), status_code=status_code, headers=[('X-Rc-Locked', '1')]) |
|
659 | 659 | |
|
660 | 660 | elif _vcs_kind == 'repo_branch_protected': |
|
661 | 661 | # Get custom repo-branch-protected status code if present. |
|
662 | 662 | return HTTPRepoBranchProtected( |
|
663 | 663 | title=str(exception), headers=[('X-Rc-Branch-Protection', '1')]) |
|
664 | 664 | |
|
665 | 665 | exc_info = request.exc_info |
|
666 | 666 | store_exception(id(exc_info), exc_info) |
|
667 | 667 | |
|
668 | 668 | traceback_info = 'unavailable' |
|
669 | 669 | if request.exc_info: |
|
670 | 670 | traceback_info = format_exc(request.exc_info) |
|
671 | 671 | |
|
672 | 672 | log.error( |
|
673 | 673 | 'error occurred handling this request for path: %s, \n%s', |
|
674 | 674 | request.path, traceback_info) |
|
675 | 675 | |
|
676 | 676 | statsd = request.registry.statsd |
|
677 | 677 | if statsd: |
|
678 | 678 | exc_type = f"{exception.__class__.__module__}.{exception.__class__.__name__}" |
|
679 | 679 | statsd.incr('vcsserver_exception_total', |
|
680 | 680 | tags=[f"type:{exc_type}"]) |
|
681 | 681 | raise exception |
|
682 | 682 | |
|
683 | 683 | |
|
684 | 684 | class ResponseFilter: |
|
685 | 685 | |
|
686 | 686 | def __init__(self, start_response): |
|
687 | 687 | self._start_response = start_response |
|
688 | 688 | |
|
689 | 689 | def __call__(self, status, response_headers, exc_info=None): |
|
690 | 690 | headers = tuple( |
|
691 | 691 | (h, v) for h, v in response_headers |
|
692 | 692 | if not wsgiref.util.is_hop_by_hop(h)) |
|
693 | 693 | return self._start_response(status, headers, exc_info) |
|
694 | 694 | |
|
695 | 695 | |
|
696 | 696 | def sanitize_settings_and_apply_defaults(global_config, settings): |
|
697 | 697 | _global_settings_maker = SettingsMaker(global_config) |
|
698 | 698 | settings_maker = SettingsMaker(settings) |
|
699 | 699 | |
|
700 | 700 | settings_maker.make_setting('logging.autoconfigure', False, parser='bool') |
|
701 | 701 | |
|
702 | 702 | logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini') |
|
703 | 703 | settings_maker.enable_logging(logging_conf) |
|
704 | 704 | |
|
705 | 705 | # Default includes, possible to change as a user |
|
706 | 706 | pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline') |
|
707 | 707 | log.debug("Using the following pyramid.includes: %s", pyramid_includes) |
|
708 | 708 | |
|
709 | 709 | settings_maker.make_setting('__file__', global_config.get('__file__')) |
|
710 | 710 | |
|
711 | 711 | settings_maker.make_setting('pyramid.default_locale_name', 'en') |
|
712 | 712 | settings_maker.make_setting('locale', 'en_US.UTF-8') |
|
713 | 713 | |
|
714 | 714 | settings_maker.make_setting( |
|
715 | 715 | 'core.binary_dir', '/usr/local/bin/rhodecode_bin/vcs_bin', |
|
716 | 716 | default_when_empty=True, parser='string:noquote') |
|
717 | 717 | |
|
718 | 718 | temp_store = tempfile.gettempdir() |
|
719 | 719 | default_cache_dir = os.path.join(temp_store, 'rc_cache') |
|
720 | 720 | # save default, cache dir, and use it for all backends later. |
|
721 | 721 | default_cache_dir = settings_maker.make_setting( |
|
722 | 722 | 'cache_dir', |
|
723 | 723 | default=default_cache_dir, default_when_empty=True, |
|
724 | 724 | parser='dir:ensured') |
|
725 | 725 | |
|
726 | 726 | # exception store cache |
|
727 | 727 | settings_maker.make_setting( |
|
728 | 728 | 'exception_tracker.store_path', |
|
729 | 729 | default=os.path.join(default_cache_dir, 'exc_store'), default_when_empty=True, |
|
730 | 730 | parser='dir:ensured' |
|
731 | 731 | ) |
|
732 | 732 | |
|
733 | 733 | # repo_object cache defaults |
|
734 | 734 | settings_maker.make_setting( |
|
735 | 735 | 'rc_cache.repo_object.backend', |
|
736 | 736 | default='dogpile.cache.rc.file_namespace', |
|
737 | 737 | parser='string') |
|
738 | 738 | settings_maker.make_setting( |
|
739 | 739 | 'rc_cache.repo_object.expiration_time', |
|
740 | 740 | default=30 * 24 * 60 * 60, # 30days |
|
741 | 741 | parser='int') |
|
742 | 742 | settings_maker.make_setting( |
|
743 | 743 | 'rc_cache.repo_object.arguments.filename', |
|
744 | 744 | default=os.path.join(default_cache_dir, 'vcsserver_cache_repo_object.db'), |
|
745 | 745 | parser='string') |
|
746 | 746 | |
|
747 | 747 | # statsd |
|
748 | 748 | settings_maker.make_setting('statsd.enabled', False, parser='bool') |
|
749 | 749 | settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string') |
|
750 | 750 | settings_maker.make_setting('statsd.statsd_port', 9125, parser='int') |
|
751 | 751 | settings_maker.make_setting('statsd.statsd_prefix', '') |
|
752 | 752 | settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool') |
|
753 | 753 | |
|
754 | 754 | settings_maker.env_expand() |
|
755 | 755 | |
|
756 | 756 | |
|
757 | 757 | def main(global_config, **settings): |
|
758 | 758 | start_time = time.time() |
|
759 | 759 | log.info('Pyramid app config starting') |
|
760 | 760 | |
|
761 | 761 | if MercurialFactory: |
|
762 | 762 | hgpatches.patch_largefiles_capabilities() |
|
763 | 763 | hgpatches.patch_subrepo_type_mapping() |
|
764 | 764 | |
|
765 | 765 | # Fill in and sanitize the defaults & do ENV expansion |
|
766 | 766 | sanitize_settings_and_apply_defaults(global_config, settings) |
|
767 | 767 | |
|
768 | 768 | # init and bootstrap StatsdClient |
|
769 | 769 | StatsdClient.setup(settings) |
|
770 | 770 | |
|
771 | 771 | pyramid_app = HTTPApplication(settings=settings, global_config=global_config).wsgi_app() |
|
772 | 772 | total_time = time.time() - start_time |
|
773 | 773 | log.info('Pyramid app created and configured in %.2fs', total_time) |
|
774 | 774 | return pyramid_app |
|
1 | NO CONTENT: file renamed from vcsserver/lib/rc_json.py to vcsserver/lib/ext_json.py |
@@ -1,417 +1,417 b'' | |||
|
1 | 1 | # RhodeCode VCSServer provides access to different vcs backends via network. |
|
2 | 2 | # Copyright (C) 2014-2023 RhodeCode GmbH |
|
3 | 3 | # |
|
4 | 4 | # This program is free software; you can redistribute it and/or modify |
|
5 | 5 | # it under the terms of the GNU General Public License as published by |
|
6 | 6 | # the Free Software Foundation; either version 3 of the License, or |
|
7 | 7 | # (at your option) any later version. |
|
8 | 8 | # |
|
9 | 9 | # This program is distributed in the hope that it will be useful, |
|
10 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 | 12 | # GNU General Public License for more details. |
|
13 | 13 | # |
|
14 | 14 | # You should have received a copy of the GNU General Public License |
|
15 | 15 | # along with this program; if not, write to the Free Software Foundation, |
|
16 | 16 | # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
|
17 | 17 | |
|
18 | 18 | """Handles the Git smart protocol.""" |
|
19 | 19 | |
|
20 | 20 | import os |
|
21 | 21 | import socket |
|
22 | 22 | import logging |
|
23 | 23 | |
|
24 | 24 | import dulwich.protocol |
|
25 | 25 | from dulwich.protocol import CAPABILITY_SIDE_BAND, CAPABILITY_SIDE_BAND_64K |
|
26 | 26 | from webob import Request, Response, exc |
|
27 | 27 | |
|
28 |
from vcsserver.lib. |
|
|
28 | from vcsserver.lib.ext_json import json | |
|
29 | 29 | from vcsserver import hooks, subprocessio |
|
30 | 30 | from vcsserver.str_utils import ascii_bytes |
|
31 | 31 | |
|
32 | 32 | |
|
33 | 33 | log = logging.getLogger(__name__) |
|
34 | 34 | |
|
35 | 35 | |
|
36 | 36 | class FileWrapper: |
|
37 | 37 | """File wrapper that ensures how much data is read from it.""" |
|
38 | 38 | |
|
39 | 39 | def __init__(self, fd, content_length): |
|
40 | 40 | self.fd = fd |
|
41 | 41 | self.content_length = content_length |
|
42 | 42 | self.remain = content_length |
|
43 | 43 | |
|
44 | 44 | def read(self, size): |
|
45 | 45 | if size <= self.remain: |
|
46 | 46 | try: |
|
47 | 47 | data = self.fd.read(size) |
|
48 | 48 | except socket.error: |
|
49 | 49 | raise IOError(self) |
|
50 | 50 | self.remain -= size |
|
51 | 51 | elif self.remain: |
|
52 | 52 | data = self.fd.read(self.remain) |
|
53 | 53 | self.remain = 0 |
|
54 | 54 | else: |
|
55 | 55 | data = None |
|
56 | 56 | return data |
|
57 | 57 | |
|
58 | 58 | def __repr__(self): |
|
59 | 59 | return '<FileWrapper {} len: {}, read: {}>'.format( |
|
60 | 60 | self.fd, self.content_length, self.content_length - self.remain |
|
61 | 61 | ) |
|
62 | 62 | |
|
63 | 63 | |
|
64 | 64 | class GitRepository: |
|
65 | 65 | """WSGI app for handling Git smart protocol endpoints.""" |
|
66 | 66 | |
|
67 | 67 | git_folder_signature = frozenset(('config', 'head', 'info', 'objects', 'refs')) |
|
68 | 68 | commands = frozenset(('git-upload-pack', 'git-receive-pack')) |
|
69 | 69 | valid_accepts = frozenset(f'application/x-{c}-result' for c in commands) |
|
70 | 70 | |
|
71 | 71 | # The last bytes are the SHA1 of the first 12 bytes. |
|
72 | 72 | EMPTY_PACK = ( |
|
73 | 73 | b'PACK\x00\x00\x00\x02\x00\x00\x00\x00\x02\x9d\x08' + |
|
74 | 74 | b'\x82;\xd8\xa8\xea\xb5\x10\xadj\xc7\\\x82<\xfd>\xd3\x1e' |
|
75 | 75 | ) |
|
76 | 76 | FLUSH_PACKET = b"0000" |
|
77 | 77 | |
|
78 | 78 | SIDE_BAND_CAPS = frozenset((CAPABILITY_SIDE_BAND, CAPABILITY_SIDE_BAND_64K)) |
|
79 | 79 | |
|
80 | 80 | def __init__(self, repo_name, content_path, git_path, update_server_info, extras): |
|
81 | 81 | files = frozenset(f.lower() for f in os.listdir(content_path)) |
|
82 | 82 | valid_dir_signature = self.git_folder_signature.issubset(files) |
|
83 | 83 | |
|
84 | 84 | if not valid_dir_signature: |
|
85 | 85 | raise OSError(f'{content_path} missing git signature') |
|
86 | 86 | |
|
87 | 87 | self.content_path = content_path |
|
88 | 88 | self.repo_name = repo_name |
|
89 | 89 | self.extras = extras |
|
90 | 90 | self.git_path = git_path |
|
91 | 91 | self.update_server_info = update_server_info |
|
92 | 92 | |
|
93 | 93 | def _get_fixedpath(self, path): |
|
94 | 94 | """ |
|
95 | 95 | Small fix for repo_path |
|
96 | 96 | |
|
97 | 97 | :param path: |
|
98 | 98 | """ |
|
99 | 99 | path = path.split(self.repo_name, 1)[-1] |
|
100 | 100 | if path.startswith('.git'): |
|
101 | 101 | # for bare repos we still get the .git prefix inside, we skip it |
|
102 | 102 | # here, and remove from the service command |
|
103 | 103 | path = path[4:] |
|
104 | 104 | |
|
105 | 105 | return path.strip('/') |
|
106 | 106 | |
|
107 | 107 | def inforefs(self, request, unused_environ): |
|
108 | 108 | """ |
|
109 | 109 | WSGI Response producer for HTTP GET Git Smart |
|
110 | 110 | HTTP /info/refs request. |
|
111 | 111 | """ |
|
112 | 112 | |
|
113 | 113 | git_command = request.GET.get('service') |
|
114 | 114 | if git_command not in self.commands: |
|
115 | 115 | log.debug('command %s not allowed', git_command) |
|
116 | 116 | return exc.HTTPForbidden() |
|
117 | 117 | |
|
118 | 118 | # please, resist the urge to add '\n' to git capture and increment |
|
119 | 119 | # line count by 1. |
|
120 | 120 | # by git docs: Documentation/technical/http-protocol.txt#L214 \n is |
|
121 | 121 | # a part of protocol. |
|
122 | 122 | # The code in Git client not only does NOT need '\n', but actually |
|
123 | 123 | # blows up if you sprinkle "flush" (0000) as "0001\n". |
|
124 | 124 | # It reads binary, per number of bytes specified. |
|
125 | 125 | # if you do add '\n' as part of data, count it. |
|
126 | 126 | server_advert = f'# service={git_command}\n' |
|
127 | 127 | packet_len = hex(len(server_advert) + 4)[2:].rjust(4, '0').lower() |
|
128 | 128 | try: |
|
129 | 129 | gitenv = dict(os.environ) |
|
130 | 130 | # forget all configs |
|
131 | 131 | gitenv['RC_SCM_DATA'] = json.dumps(self.extras) |
|
132 | 132 | command = [self.git_path, git_command[4:], '--stateless-rpc', |
|
133 | 133 | '--advertise-refs', self.content_path] |
|
134 | 134 | out = subprocessio.SubprocessIOChunker( |
|
135 | 135 | command, |
|
136 | 136 | env=gitenv, |
|
137 | 137 | starting_values=[ascii_bytes(packet_len + server_advert) + self.FLUSH_PACKET], |
|
138 | 138 | shell=False |
|
139 | 139 | ) |
|
140 | 140 | except OSError: |
|
141 | 141 | log.exception('Error processing command') |
|
142 | 142 | raise exc.HTTPExpectationFailed() |
|
143 | 143 | |
|
144 | 144 | resp = Response() |
|
145 | 145 | resp.content_type = f'application/x-{git_command}-advertisement' |
|
146 | 146 | resp.charset = None |
|
147 | 147 | resp.app_iter = out |
|
148 | 148 | |
|
149 | 149 | return resp |
|
150 | 150 | |
|
151 | 151 | def _get_want_capabilities(self, request): |
|
152 | 152 | """Read the capabilities found in the first want line of the request.""" |
|
153 | 153 | pos = request.body_file_seekable.tell() |
|
154 | 154 | first_line = request.body_file_seekable.readline() |
|
155 | 155 | request.body_file_seekable.seek(pos) |
|
156 | 156 | |
|
157 | 157 | return frozenset( |
|
158 | 158 | dulwich.protocol.extract_want_line_capabilities(first_line)[1]) |
|
159 | 159 | |
|
160 | 160 | def _build_failed_pre_pull_response(self, capabilities, pre_pull_messages): |
|
161 | 161 | """ |
|
162 | 162 | Construct a response with an empty PACK file. |
|
163 | 163 | |
|
164 | 164 | We use an empty PACK file, as that would trigger the failure of the pull |
|
165 | 165 | or clone command. |
|
166 | 166 | |
|
167 | 167 | We also print in the error output a message explaining why the command |
|
168 | 168 | was aborted. |
|
169 | 169 | |
|
170 | 170 | If additionally, the user is accepting messages we send them the output |
|
171 | 171 | of the pre-pull hook. |
|
172 | 172 | |
|
173 | 173 | Note that for clients not supporting side-band we just send them the |
|
174 | 174 | emtpy PACK file. |
|
175 | 175 | """ |
|
176 | 176 | |
|
177 | 177 | if self.SIDE_BAND_CAPS.intersection(capabilities): |
|
178 | 178 | response = [] |
|
179 | 179 | proto = dulwich.protocol.Protocol(None, response.append) |
|
180 | 180 | proto.write_pkt_line(dulwich.protocol.NAK_LINE) |
|
181 | 181 | |
|
182 | 182 | self._write_sideband_to_proto(proto, ascii_bytes(pre_pull_messages, allow_bytes=True), capabilities) |
|
183 | 183 | # N.B.(skreft): Do not change the sideband channel to 3, as that |
|
184 | 184 | # produces a fatal error in the client: |
|
185 | 185 | # fatal: error in sideband demultiplexer |
|
186 | 186 | proto.write_sideband( |
|
187 | 187 | dulwich.protocol.SIDE_BAND_CHANNEL_PROGRESS, |
|
188 | 188 | ascii_bytes('Pre pull hook failed: aborting\n', allow_bytes=True)) |
|
189 | 189 | proto.write_sideband( |
|
190 | 190 | dulwich.protocol.SIDE_BAND_CHANNEL_DATA, |
|
191 | 191 | ascii_bytes(self.EMPTY_PACK, allow_bytes=True)) |
|
192 | 192 | |
|
193 | 193 | # writes b"0000" as default |
|
194 | 194 | proto.write_pkt_line(None) |
|
195 | 195 | |
|
196 | 196 | return response |
|
197 | 197 | else: |
|
198 | 198 | return [ascii_bytes(self.EMPTY_PACK, allow_bytes=True)] |
|
199 | 199 | |
|
200 | 200 | def _build_post_pull_response(self, response, capabilities, start_message, end_message): |
|
201 | 201 | """ |
|
202 | 202 | Given a list response we inject the post-pull messages. |
|
203 | 203 | |
|
204 | 204 | We only inject the messages if the client supports sideband, and the |
|
205 | 205 | response has the format: |
|
206 | 206 | 0008NAK\n...0000 |
|
207 | 207 | |
|
208 | 208 | Note that we do not check the no-progress capability as by default, git |
|
209 | 209 | sends it, which effectively would block all messages. |
|
210 | 210 | """ |
|
211 | 211 | |
|
212 | 212 | if not self.SIDE_BAND_CAPS.intersection(capabilities): |
|
213 | 213 | return response |
|
214 | 214 | |
|
215 | 215 | if not start_message and not end_message: |
|
216 | 216 | return response |
|
217 | 217 | |
|
218 | 218 | try: |
|
219 | 219 | iter(response) |
|
220 | 220 | # iterator probably will work, we continue |
|
221 | 221 | except TypeError: |
|
222 | 222 | raise TypeError(f'response must be an iterator: got {type(response)}') |
|
223 | 223 | if isinstance(response, (list, tuple)): |
|
224 | 224 | raise TypeError(f'response must be an iterator: got {type(response)}') |
|
225 | 225 | |
|
226 | 226 | def injected_response(): |
|
227 | 227 | |
|
228 | 228 | do_loop = 1 |
|
229 | 229 | header_injected = 0 |
|
230 | 230 | next_item = None |
|
231 | 231 | has_item = False |
|
232 | 232 | item = b'' |
|
233 | 233 | |
|
234 | 234 | while do_loop: |
|
235 | 235 | |
|
236 | 236 | try: |
|
237 | 237 | next_item = next(response) |
|
238 | 238 | except StopIteration: |
|
239 | 239 | do_loop = 0 |
|
240 | 240 | |
|
241 | 241 | if has_item: |
|
242 | 242 | # last item ! alter it now |
|
243 | 243 | if do_loop == 0 and item.endswith(self.FLUSH_PACKET): |
|
244 | 244 | new_response = [item[:-4]] |
|
245 | 245 | new_response.extend(self._get_messages(end_message, capabilities)) |
|
246 | 246 | new_response.append(self.FLUSH_PACKET) |
|
247 | 247 | item = b''.join(new_response) |
|
248 | 248 | |
|
249 | 249 | yield item |
|
250 | 250 | |
|
251 | 251 | has_item = True |
|
252 | 252 | item = next_item |
|
253 | 253 | |
|
254 | 254 | # alter item if it's the initial chunk |
|
255 | 255 | if not header_injected and item.startswith(b'0008NAK\n'): |
|
256 | 256 | new_response = [b'0008NAK\n'] |
|
257 | 257 | new_response.extend(self._get_messages(start_message, capabilities)) |
|
258 | 258 | new_response.append(item[8:]) |
|
259 | 259 | item = b''.join(new_response) |
|
260 | 260 | header_injected = 1 |
|
261 | 261 | |
|
262 | 262 | return injected_response() |
|
263 | 263 | |
|
264 | 264 | def _write_sideband_to_proto(self, proto, data, capabilities): |
|
265 | 265 | """ |
|
266 | 266 | Write the data to the proto's sideband number 2 == SIDE_BAND_CHANNEL_PROGRESS |
|
267 | 267 | |
|
268 | 268 | We do not use dulwich's write_sideband directly as it only supports |
|
269 | 269 | side-band-64k. |
|
270 | 270 | """ |
|
271 | 271 | if not data: |
|
272 | 272 | return |
|
273 | 273 | |
|
274 | 274 | # N.B.(skreft): The values below are explained in the pack protocol |
|
275 | 275 | # documentation, section Packfile Data. |
|
276 | 276 | # https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt |
|
277 | 277 | if CAPABILITY_SIDE_BAND_64K in capabilities: |
|
278 | 278 | chunk_size = 65515 |
|
279 | 279 | elif CAPABILITY_SIDE_BAND in capabilities: |
|
280 | 280 | chunk_size = 995 |
|
281 | 281 | else: |
|
282 | 282 | return |
|
283 | 283 | |
|
284 | 284 | chunker = (data[i:i + chunk_size] for i in range(0, len(data), chunk_size)) |
|
285 | 285 | |
|
286 | 286 | for chunk in chunker: |
|
287 | 287 | proto.write_sideband(dulwich.protocol.SIDE_BAND_CHANNEL_PROGRESS, ascii_bytes(chunk, allow_bytes=True)) |
|
288 | 288 | |
|
289 | 289 | def _get_messages(self, data, capabilities): |
|
290 | 290 | """Return a list with packets for sending data in sideband number 2.""" |
|
291 | 291 | response = [] |
|
292 | 292 | proto = dulwich.protocol.Protocol(None, response.append) |
|
293 | 293 | |
|
294 | 294 | self._write_sideband_to_proto(proto, data, capabilities) |
|
295 | 295 | |
|
296 | 296 | return response |
|
297 | 297 | |
|
298 | 298 | def backend(self, request, environ): |
|
299 | 299 | """ |
|
300 | 300 | WSGI Response producer for HTTP POST Git Smart HTTP requests. |
|
301 | 301 | Reads commands and data from HTTP POST's body. |
|
302 | 302 | returns an iterator obj with contents of git command's |
|
303 | 303 | response to stdout |
|
304 | 304 | """ |
|
305 | 305 | # TODO(skreft): think how we could detect an HTTPLockedException, as |
|
306 | 306 | # we probably want to have the same mechanism used by mercurial and |
|
307 | 307 | # simplevcs. |
|
308 | 308 | # For that we would need to parse the output of the command looking for |
|
309 | 309 | # some signs of the HTTPLockedError, parse the data and reraise it in |
|
310 | 310 | # pygrack. However, that would interfere with the streaming. |
|
311 | 311 | # |
|
312 | 312 | # Now the output of a blocked push is: |
|
313 | 313 | # Pushing to http://test_regular:test12@127.0.0.1:5001/vcs_test_git |
|
314 | 314 | # POST git-receive-pack (1047 bytes) |
|
315 | 315 | # remote: ERROR: Repository `vcs_test_git` locked by user `test_admin`. Reason:`lock_auto` |
|
316 | 316 | # To http://test_regular:test12@127.0.0.1:5001/vcs_test_git |
|
317 | 317 | # ! [remote rejected] master -> master (pre-receive hook declined) |
|
318 | 318 | # error: failed to push some refs to 'http://test_regular:test12@127.0.0.1:5001/vcs_test_git' |
|
319 | 319 | |
|
320 | 320 | git_command = self._get_fixedpath(request.path_info) |
|
321 | 321 | if git_command not in self.commands: |
|
322 | 322 | log.debug('command %s not allowed', git_command) |
|
323 | 323 | return exc.HTTPForbidden() |
|
324 | 324 | |
|
325 | 325 | capabilities = None |
|
326 | 326 | if git_command == 'git-upload-pack': |
|
327 | 327 | capabilities = self._get_want_capabilities(request) |
|
328 | 328 | |
|
329 | 329 | if 'CONTENT_LENGTH' in environ: |
|
330 | 330 | inputstream = FileWrapper(request.body_file_seekable, |
|
331 | 331 | request.content_length) |
|
332 | 332 | else: |
|
333 | 333 | inputstream = request.body_file_seekable |
|
334 | 334 | |
|
335 | 335 | resp = Response() |
|
336 | 336 | resp.content_type = f'application/x-{git_command}-result' |
|
337 | 337 | resp.charset = None |
|
338 | 338 | |
|
339 | 339 | pre_pull_messages = '' |
|
340 | 340 | # Upload-pack == clone |
|
341 | 341 | if git_command == 'git-upload-pack': |
|
342 | 342 | hook_response = hooks.git_pre_pull(self.extras) |
|
343 | 343 | if hook_response.status != 0: |
|
344 | 344 | pre_pull_messages = hook_response.output |
|
345 | 345 | resp.app_iter = self._build_failed_pre_pull_response( |
|
346 | 346 | capabilities, pre_pull_messages) |
|
347 | 347 | return resp |
|
348 | 348 | |
|
349 | 349 | gitenv = dict(os.environ) |
|
350 | 350 | # forget all configs |
|
351 | 351 | gitenv['GIT_CONFIG_NOGLOBAL'] = '1' |
|
352 | 352 | gitenv['RC_SCM_DATA'] = json.dumps(self.extras) |
|
353 | 353 | cmd = [self.git_path, git_command[4:], '--stateless-rpc', |
|
354 | 354 | self.content_path] |
|
355 | 355 | log.debug('handling cmd %s', cmd) |
|
356 | 356 | |
|
357 | 357 | out = subprocessio.SubprocessIOChunker( |
|
358 | 358 | cmd, |
|
359 | 359 | input_stream=inputstream, |
|
360 | 360 | env=gitenv, |
|
361 | 361 | cwd=self.content_path, |
|
362 | 362 | shell=False, |
|
363 | 363 | fail_on_stderr=False, |
|
364 | 364 | fail_on_return_code=False |
|
365 | 365 | ) |
|
366 | 366 | |
|
367 | 367 | if self.update_server_info and git_command == 'git-receive-pack': |
|
368 | 368 | # We need to fully consume the iterator here, as the |
|
369 | 369 | # update-server-info command needs to be run after the push. |
|
370 | 370 | out = list(out) |
|
371 | 371 | |
|
372 | 372 | # Updating refs manually after each push. |
|
373 | 373 | # This is required as some clients are exposing Git repos internally |
|
374 | 374 | # with the dumb protocol. |
|
375 | 375 | cmd = [self.git_path, 'update-server-info'] |
|
376 | 376 | log.debug('handling cmd %s', cmd) |
|
377 | 377 | output = subprocessio.SubprocessIOChunker( |
|
378 | 378 | cmd, |
|
379 | 379 | input_stream=inputstream, |
|
380 | 380 | env=gitenv, |
|
381 | 381 | cwd=self.content_path, |
|
382 | 382 | shell=False, |
|
383 | 383 | fail_on_stderr=False, |
|
384 | 384 | fail_on_return_code=False |
|
385 | 385 | ) |
|
386 | 386 | # Consume all the output so the subprocess finishes |
|
387 | 387 | for _ in output: |
|
388 | 388 | pass |
|
389 | 389 | |
|
390 | 390 | # Upload-pack == clone |
|
391 | 391 | if git_command == 'git-upload-pack': |
|
392 | 392 | hook_response = hooks.git_post_pull(self.extras) |
|
393 | 393 | post_pull_messages = hook_response.output |
|
394 | 394 | resp.app_iter = self._build_post_pull_response(out, capabilities, pre_pull_messages, post_pull_messages) |
|
395 | 395 | else: |
|
396 | 396 | resp.app_iter = out |
|
397 | 397 | |
|
398 | 398 | return resp |
|
399 | 399 | |
|
400 | 400 | def __call__(self, environ, start_response): |
|
401 | 401 | request = Request(environ) |
|
402 | 402 | _path = self._get_fixedpath(request.path_info) |
|
403 | 403 | if _path.startswith('info/refs'): |
|
404 | 404 | app = self.inforefs |
|
405 | 405 | else: |
|
406 | 406 | app = self.backend |
|
407 | 407 | |
|
408 | 408 | try: |
|
409 | 409 | resp = app(request, environ) |
|
410 | 410 | except exc.HTTPException as error: |
|
411 | 411 | log.exception('HTTP Error') |
|
412 | 412 | resp = error |
|
413 | 413 | except Exception: |
|
414 | 414 | log.exception('Unknown error') |
|
415 | 415 | resp = exc.HTTPInternalServerError() |
|
416 | 416 | |
|
417 | 417 | return resp(environ, start_response) |
@@ -1,257 +1,257 b'' | |||
|
1 | 1 | # RhodeCode VCSServer provides access to different vcs backends via network. |
|
2 | 2 | # Copyright (C) 2014-2023 RhodeCode GmbH |
|
3 | 3 | # |
|
4 | 4 | # This program is free software; you can redistribute it and/or modify |
|
5 | 5 | # it under the terms of the GNU General Public License as published by |
|
6 | 6 | # the Free Software Foundation; either version 3 of the License, or |
|
7 | 7 | # (at your option) any later version. |
|
8 | 8 | # |
|
9 | 9 | # This program is distributed in the hope that it will be useful, |
|
10 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 | 12 | # GNU General Public License for more details. |
|
13 | 13 | # |
|
14 | 14 | # You should have received a copy of the GNU General Public License |
|
15 | 15 | # along with this program; if not, write to the Free Software Foundation, |
|
16 | 16 | # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
|
17 | 17 | |
|
18 | 18 | import threading |
|
19 | 19 | import msgpack |
|
20 | 20 | |
|
21 | 21 | from http.server import BaseHTTPRequestHandler |
|
22 | 22 | from socketserver import TCPServer |
|
23 | 23 | |
|
24 | 24 | import mercurial.ui |
|
25 | 25 | import mock |
|
26 | 26 | import pytest |
|
27 | 27 | |
|
28 | 28 | from vcsserver.hooks import HooksHttpClient |
|
29 |
from vcsserver.lib. |
|
|
29 | from vcsserver.lib.ext_json import json | |
|
30 | 30 | from vcsserver import hooks |
|
31 | 31 | |
|
32 | 32 | |
|
33 | 33 | def get_hg_ui(extras=None): |
|
34 | 34 | """Create a Config object with a valid RC_SCM_DATA entry.""" |
|
35 | 35 | extras = extras or {} |
|
36 | 36 | required_extras = { |
|
37 | 37 | 'username': '', |
|
38 | 38 | 'repository': '', |
|
39 | 39 | 'locked_by': '', |
|
40 | 40 | 'scm': '', |
|
41 | 41 | 'make_lock': '', |
|
42 | 42 | 'action': '', |
|
43 | 43 | 'ip': '', |
|
44 | 44 | 'hooks_uri': 'fake_hooks_uri', |
|
45 | 45 | } |
|
46 | 46 | required_extras.update(extras) |
|
47 | 47 | hg_ui = mercurial.ui.ui() |
|
48 | 48 | hg_ui.setconfig(b'rhodecode', b'RC_SCM_DATA', json.dumps(required_extras)) |
|
49 | 49 | |
|
50 | 50 | return hg_ui |
|
51 | 51 | |
|
52 | 52 | |
|
53 | 53 | def test_git_pre_receive_is_disabled(): |
|
54 | 54 | extras = {'hooks': ['pull']} |
|
55 | 55 | response = hooks.git_pre_receive(None, None, |
|
56 | 56 | {'RC_SCM_DATA': json.dumps(extras)}) |
|
57 | 57 | |
|
58 | 58 | assert response == 0 |
|
59 | 59 | |
|
60 | 60 | |
|
61 | 61 | def test_git_post_receive_is_disabled(): |
|
62 | 62 | extras = {'hooks': ['pull']} |
|
63 | 63 | response = hooks.git_post_receive(None, '', |
|
64 | 64 | {'RC_SCM_DATA': json.dumps(extras)}) |
|
65 | 65 | |
|
66 | 66 | assert response == 0 |
|
67 | 67 | |
|
68 | 68 | |
|
69 | 69 | def test_git_post_receive_calls_repo_size(): |
|
70 | 70 | extras = {'hooks': ['push', 'repo_size']} |
|
71 | 71 | |
|
72 | 72 | with mock.patch.object(hooks, '_call_hook') as call_hook_mock: |
|
73 | 73 | hooks.git_post_receive( |
|
74 | 74 | None, '', {'RC_SCM_DATA': json.dumps(extras)}) |
|
75 | 75 | extras.update({'commit_ids': [], 'hook_type': 'post_receive', |
|
76 | 76 | 'new_refs': {'bookmarks': [], 'branches': [], 'tags': []}}) |
|
77 | 77 | expected_calls = [ |
|
78 | 78 | mock.call('repo_size', extras, mock.ANY), |
|
79 | 79 | mock.call('post_push', extras, mock.ANY), |
|
80 | 80 | ] |
|
81 | 81 | assert call_hook_mock.call_args_list == expected_calls |
|
82 | 82 | |
|
83 | 83 | |
|
84 | 84 | def test_git_post_receive_does_not_call_disabled_repo_size(): |
|
85 | 85 | extras = {'hooks': ['push']} |
|
86 | 86 | |
|
87 | 87 | with mock.patch.object(hooks, '_call_hook') as call_hook_mock: |
|
88 | 88 | hooks.git_post_receive( |
|
89 | 89 | None, '', {'RC_SCM_DATA': json.dumps(extras)}) |
|
90 | 90 | extras.update({'commit_ids': [], 'hook_type': 'post_receive', |
|
91 | 91 | 'new_refs': {'bookmarks': [], 'branches': [], 'tags': []}}) |
|
92 | 92 | expected_calls = [ |
|
93 | 93 | mock.call('post_push', extras, mock.ANY) |
|
94 | 94 | ] |
|
95 | 95 | assert call_hook_mock.call_args_list == expected_calls |
|
96 | 96 | |
|
97 | 97 | |
|
98 | 98 | def test_repo_size_exception_does_not_affect_git_post_receive(): |
|
99 | 99 | extras = {'hooks': ['push', 'repo_size']} |
|
100 | 100 | status = 0 |
|
101 | 101 | |
|
102 | 102 | def side_effect(name, *args, **kwargs): |
|
103 | 103 | if name == 'repo_size': |
|
104 | 104 | raise Exception('Fake exception') |
|
105 | 105 | else: |
|
106 | 106 | return status |
|
107 | 107 | |
|
108 | 108 | with mock.patch.object(hooks, '_call_hook') as call_hook_mock: |
|
109 | 109 | call_hook_mock.side_effect = side_effect |
|
110 | 110 | result = hooks.git_post_receive( |
|
111 | 111 | None, '', {'RC_SCM_DATA': json.dumps(extras)}) |
|
112 | 112 | assert result == status |
|
113 | 113 | |
|
114 | 114 | |
|
115 | 115 | def test_git_pre_pull_is_disabled(): |
|
116 | 116 | assert hooks.git_pre_pull({'hooks': ['push']}) == hooks.HookResponse(0, '') |
|
117 | 117 | |
|
118 | 118 | |
|
119 | 119 | def test_git_post_pull_is_disabled(): |
|
120 | 120 | assert ( |
|
121 | 121 | hooks.git_post_pull({'hooks': ['push']}) == hooks.HookResponse(0, '')) |
|
122 | 122 | |
|
123 | 123 | |
|
124 | 124 | class TestGetHooksClient: |
|
125 | 125 | |
|
126 | 126 | def test_returns_http_client_when_protocol_matches(self): |
|
127 | 127 | hooks_uri = 'localhost:8000' |
|
128 | 128 | result = hooks._get_hooks_client({ |
|
129 | 129 | 'hooks_uri': hooks_uri, |
|
130 | 130 | 'hooks_protocol': 'http' |
|
131 | 131 | }) |
|
132 | 132 | assert isinstance(result, hooks.HooksHttpClient) |
|
133 | 133 | assert result.hooks_uri == hooks_uri |
|
134 | 134 | |
|
135 | 135 | def test_return_celery_client_when_queue_and_backend_provided(self): |
|
136 | 136 | task_queue = 'redis://task_queue:0' |
|
137 | 137 | task_backend = task_queue |
|
138 | 138 | result = hooks._get_hooks_client({ |
|
139 | 139 | 'task_queue': task_queue, |
|
140 | 140 | 'task_backend': task_backend |
|
141 | 141 | }) |
|
142 | 142 | assert isinstance(result, hooks.HooksCeleryClient) |
|
143 | 143 | |
|
144 | 144 | |
|
145 | 145 | class TestHooksHttpClient: |
|
146 | 146 | def test_init_sets_hooks_uri(self): |
|
147 | 147 | uri = 'localhost:3000' |
|
148 | 148 | client = hooks.HooksHttpClient(uri) |
|
149 | 149 | assert client.hooks_uri == uri |
|
150 | 150 | |
|
151 | 151 | def test_serialize_returns_serialized_string(self): |
|
152 | 152 | client = hooks.HooksHttpClient('localhost:3000') |
|
153 | 153 | hook_name = 'test' |
|
154 | 154 | extras = { |
|
155 | 155 | 'first': 1, |
|
156 | 156 | 'second': 'two' |
|
157 | 157 | } |
|
158 | 158 | hooks_proto, result = client._serialize(hook_name, extras) |
|
159 | 159 | expected_result = msgpack.packb({ |
|
160 | 160 | 'method': hook_name, |
|
161 | 161 | 'extras': extras, |
|
162 | 162 | }) |
|
163 | 163 | assert hooks_proto == {'rc-hooks-protocol': 'msgpack.v1', 'Connection': 'keep-alive'} |
|
164 | 164 | assert result == expected_result |
|
165 | 165 | |
|
166 | 166 | def test_call_queries_http_server(self, http_mirror): |
|
167 | 167 | client = hooks.HooksHttpClient(http_mirror.uri) |
|
168 | 168 | hook_name = 'test' |
|
169 | 169 | extras = { |
|
170 | 170 | 'first': 1, |
|
171 | 171 | 'second': 'two' |
|
172 | 172 | } |
|
173 | 173 | result = client(hook_name, extras) |
|
174 | 174 | expected_result = msgpack.unpackb(msgpack.packb({ |
|
175 | 175 | 'method': hook_name, |
|
176 | 176 | 'extras': extras |
|
177 | 177 | }), raw=False) |
|
178 | 178 | assert result == expected_result |
|
179 | 179 | |
|
180 | 180 | |
|
181 | 181 | @pytest.fixture |
|
182 | 182 | def http_mirror(request): |
|
183 | 183 | server = MirrorHttpServer() |
|
184 | 184 | request.addfinalizer(server.stop) |
|
185 | 185 | return server |
|
186 | 186 | |
|
187 | 187 | |
|
188 | 188 | class MirrorHttpHandler(BaseHTTPRequestHandler): |
|
189 | 189 | |
|
190 | 190 | def do_POST(self): |
|
191 | 191 | length = int(self.headers['Content-Length']) |
|
192 | 192 | body = self.rfile.read(length) |
|
193 | 193 | self.send_response(200) |
|
194 | 194 | self.end_headers() |
|
195 | 195 | self.wfile.write(body) |
|
196 | 196 | |
|
197 | 197 | |
|
198 | 198 | class MirrorHttpServer: |
|
199 | 199 | ip_address = '127.0.0.1' |
|
200 | 200 | port = 0 |
|
201 | 201 | |
|
202 | 202 | def __init__(self): |
|
203 | 203 | self._daemon = TCPServer((self.ip_address, 0), MirrorHttpHandler) |
|
204 | 204 | _, self.port = self._daemon.server_address |
|
205 | 205 | self._thread = threading.Thread(target=self._daemon.serve_forever) |
|
206 | 206 | self._thread.daemon = True |
|
207 | 207 | self._thread.start() |
|
208 | 208 | |
|
209 | 209 | def stop(self): |
|
210 | 210 | self._daemon.shutdown() |
|
211 | 211 | self._thread.join() |
|
212 | 212 | self._daemon = None |
|
213 | 213 | self._thread = None |
|
214 | 214 | |
|
215 | 215 | @property |
|
216 | 216 | def uri(self): |
|
217 | 217 | return '{}:{}'.format(self.ip_address, self.port) |
|
218 | 218 | |
|
219 | 219 | |
|
220 | 220 | def test_hooks_http_client_init(): |
|
221 | 221 | hooks_uri = 'http://localhost:8000' |
|
222 | 222 | client = HooksHttpClient(hooks_uri) |
|
223 | 223 | assert client.hooks_uri == hooks_uri |
|
224 | 224 | |
|
225 | 225 | |
|
226 | 226 | def test_hooks_http_client_call(): |
|
227 | 227 | hooks_uri = 'http://localhost:8000' |
|
228 | 228 | |
|
229 | 229 | method = 'test_method' |
|
230 | 230 | extras = {'key': 'value'} |
|
231 | 231 | |
|
232 | 232 | with \ |
|
233 | 233 | mock.patch('http.client.HTTPConnection') as mock_connection,\ |
|
234 | 234 | mock.patch('msgpack.load') as mock_load: |
|
235 | 235 | |
|
236 | 236 | client = HooksHttpClient(hooks_uri) |
|
237 | 237 | |
|
238 | 238 | mock_load.return_value = {'result': 'success'} |
|
239 | 239 | response = mock.MagicMock() |
|
240 | 240 | response.status = 200 |
|
241 | 241 | mock_connection.request.side_effect = None |
|
242 | 242 | mock_connection.getresponse.return_value = response |
|
243 | 243 | |
|
244 | 244 | result = client(method, extras) |
|
245 | 245 | |
|
246 | 246 | mock_connection.assert_called_with(hooks_uri) |
|
247 | 247 | mock_connection.return_value.request.assert_called_once() |
|
248 | 248 | assert result == {'result': 'success'} |
|
249 | 249 | |
|
250 | 250 | |
|
251 | 251 | def test_hooks_http_client_serialize(): |
|
252 | 252 | method = 'test_method' |
|
253 | 253 | extras = {'key': 'value'} |
|
254 | 254 | headers, body = HooksHttpClient._serialize(method, extras) |
|
255 | 255 | |
|
256 | 256 | assert headers == {'rc-hooks-protocol': HooksHttpClient.proto, 'Connection': 'keep-alive'} |
|
257 | 257 | assert msgpack.unpackb(body) == {'method': method, 'extras': extras} |
General Comments 0
You need to be logged in to leave comments.
Login now