Show More
@@ -1,287 +1,292 b'' | |||
|
1 | 1 | # RhodeCode VCSServer provides access to different vcs backends via network. |
|
2 | 2 | # Copyright (C) 2014-2019 RhodeCode GmbH |
|
3 | 3 | # |
|
4 | 4 | # This program is free software; you can redistribute it and/or modify |
|
5 | 5 | # it under the terms of the GNU General Public License as published by |
|
6 | 6 | # the Free Software Foundation; either version 3 of the License, or |
|
7 | 7 | # (at your option) any later version. |
|
8 | 8 | # |
|
9 | 9 | # This program is distributed in the hope that it will be useful, |
|
10 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 | 12 | # GNU General Public License for more details. |
|
13 | 13 | # |
|
14 | 14 | # You should have received a copy of the GNU General Public License |
|
15 | 15 | # along with this program; if not, write to the Free Software Foundation, |
|
16 | 16 | # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
|
17 | 17 | |
|
18 | 18 | import re |
|
19 | 19 | import logging |
|
20 | 20 | from wsgiref.util import FileWrapper |
|
21 | 21 | |
|
22 | 22 | import simplejson as json |
|
23 | 23 | from pyramid.config import Configurator |
|
24 | 24 | from pyramid.response import Response, FileIter |
|
25 | 25 | from pyramid.httpexceptions import ( |
|
26 | 26 | HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden, |
|
27 | 27 | HTTPUnprocessableEntity) |
|
28 | 28 | |
|
29 | 29 | from vcsserver.git_lfs.lib import OidHandler, LFSOidStore |
|
30 | 30 | from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator |
|
31 | 31 | from vcsserver.utils import safe_int |
|
32 | 32 | |
|
33 | 33 | log = logging.getLogger(__name__) |
|
34 | 34 | |
|
35 | 35 | |
|
36 | 36 | GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs' #+json ? |
|
37 | 37 | GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))') |
|
38 | 38 | |
|
39 | 39 | |
|
40 | 40 | def write_response_error(http_exception, text=None): |
|
41 | 41 | content_type = GIT_LFS_CONTENT_TYPE + '+json' |
|
42 | 42 | _exception = http_exception(content_type=content_type) |
|
43 | 43 | _exception.content_type = content_type |
|
44 | 44 | if text: |
|
45 | 45 | _exception.body = json.dumps({'message': text}) |
|
46 | 46 | log.debug('LFS: writing response of type %s to client with text:%s', |
|
47 | 47 | http_exception, text) |
|
48 | 48 | return _exception |
|
49 | 49 | |
|
50 | 50 | |
|
51 | 51 | class AuthHeaderRequired(object): |
|
52 | 52 | """ |
|
53 | 53 | Decorator to check if request has proper auth-header |
|
54 | 54 | """ |
|
55 | 55 | |
|
56 | 56 | def __call__(self, func): |
|
57 | 57 | return get_cython_compat_decorator(self.__wrapper, func) |
|
58 | 58 | |
|
59 | 59 | def __wrapper(self, func, *fargs, **fkwargs): |
|
60 | 60 | request = fargs[1] |
|
61 | 61 | auth = request.authorization |
|
62 | 62 | if not auth: |
|
63 | 63 | return write_response_error(HTTPForbidden) |
|
64 | 64 | return func(*fargs[1:], **fkwargs) |
|
65 | 65 | |
|
66 | 66 | |
|
67 | 67 | # views |
|
68 | 68 | |
|
69 | 69 | def lfs_objects(request): |
|
70 | 70 | # indicate not supported, V1 API |
|
71 | 71 | log.warning('LFS: v1 api not supported, reporting it back to client') |
|
72 | 72 | return write_response_error(HTTPNotImplemented, 'LFS: v1 api not supported') |
|
73 | 73 | |
|
74 | 74 | |
|
75 | 75 | @AuthHeaderRequired() |
|
76 | 76 | def lfs_objects_batch(request): |
|
77 | 77 | """ |
|
78 | 78 | The client sends the following information to the Batch endpoint to transfer some objects: |
|
79 | 79 | |
|
80 | 80 | operation - Should be download or upload. |
|
81 | 81 | transfers - An optional Array of String identifiers for transfer |
|
82 | 82 | adapters that the client has configured. If omitted, the basic |
|
83 | 83 | transfer adapter MUST be assumed by the server. |
|
84 | 84 | objects - An Array of objects to download. |
|
85 | 85 | oid - String OID of the LFS object. |
|
86 | 86 | size - Integer byte size of the LFS object. Must be at least zero. |
|
87 | 87 | """ |
|
88 | 88 | request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json' |
|
89 | 89 | auth = request.authorization |
|
90 | 90 | repo = request.matchdict.get('repo') |
|
91 | 91 | data = request.json |
|
92 | 92 | operation = data.get('operation') |
|
93 | http_scheme = request.registry.git_lfs_http_scheme | |
|
94 | ||
|
93 | 95 | if operation not in ('download', 'upload'): |
|
94 | 96 | log.debug('LFS: unsupported operation:%s', operation) |
|
95 | 97 | return write_response_error( |
|
96 | 98 | HTTPBadRequest, 'unsupported operation mode: `%s`' % operation) |
|
97 | 99 | |
|
98 | 100 | if 'objects' not in data: |
|
99 | 101 | log.debug('LFS: missing objects data') |
|
100 | 102 | return write_response_error( |
|
101 | 103 | HTTPBadRequest, 'missing objects data') |
|
102 | 104 | |
|
103 | 105 | log.debug('LFS: handling operation of type: %s', operation) |
|
104 | 106 | |
|
105 | 107 | objects = [] |
|
106 | 108 | for o in data['objects']: |
|
107 | 109 | try: |
|
108 | 110 | oid = o['oid'] |
|
109 | 111 | obj_size = o['size'] |
|
110 | 112 | except KeyError: |
|
111 | 113 | log.exception('LFS, failed to extract data') |
|
112 | 114 | return write_response_error( |
|
113 | 115 | HTTPBadRequest, 'unsupported data in objects') |
|
114 | 116 | |
|
115 | 117 | obj_data = {'oid': oid} |
|
116 | 118 | |
|
117 |
obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid |
|
|
118 | obj_verify_href = request.route_url('lfs_objects_verify', repo=repo) | |
|
119 | obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid, | |
|
120 | _scheme=http_scheme) | |
|
121 | obj_verify_href = request.route_url('lfs_objects_verify', repo=repo, | |
|
122 | _scheme=http_scheme) | |
|
119 | 123 | store = LFSOidStore( |
|
120 | 124 | oid, repo, store_location=request.registry.git_lfs_store_path) |
|
121 | 125 | handler = OidHandler( |
|
122 | 126 | store, repo, auth, oid, obj_size, obj_data, |
|
123 | 127 | obj_href, obj_verify_href) |
|
124 | 128 | |
|
125 | 129 | # this verifies also OIDs |
|
126 | 130 | actions, errors = handler.exec_operation(operation) |
|
127 | 131 | if errors: |
|
128 | 132 | log.warning('LFS: got following errors: %s', errors) |
|
129 | 133 | obj_data['errors'] = errors |
|
130 | 134 | |
|
131 | 135 | if actions: |
|
132 | 136 | obj_data['actions'] = actions |
|
133 | 137 | |
|
134 | 138 | obj_data['size'] = obj_size |
|
135 | 139 | obj_data['authenticated'] = True |
|
136 | 140 | objects.append(obj_data) |
|
137 | 141 | |
|
138 | 142 | result = {'objects': objects, 'transfer': 'basic'} |
|
139 | 143 | log.debug('LFS Response %s', safe_result(result)) |
|
140 | 144 | |
|
141 | 145 | return result |
|
142 | 146 | |
|
143 | 147 | |
|
144 | 148 | def lfs_objects_oid_upload(request): |
|
145 | 149 | request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json' |
|
146 | 150 | repo = request.matchdict.get('repo') |
|
147 | 151 | oid = request.matchdict.get('oid') |
|
148 | 152 | store = LFSOidStore( |
|
149 | 153 | oid, repo, store_location=request.registry.git_lfs_store_path) |
|
150 | 154 | engine = store.get_engine(mode='wb') |
|
151 | 155 | log.debug('LFS: starting chunked write of LFS oid: %s to storage', oid) |
|
152 | 156 | |
|
153 | 157 | body = request.environ['wsgi.input'] |
|
154 | 158 | |
|
155 | 159 | with engine as f: |
|
156 | 160 | blksize = 64 * 1024 # 64kb |
|
157 | 161 | while True: |
|
158 | 162 | # read in chunks as stream comes in from Gunicorn |
|
159 | 163 | # this is a specific Gunicorn support function. |
|
160 | 164 | # might work differently on waitress |
|
161 | 165 | chunk = body.read(blksize) |
|
162 | 166 | if not chunk: |
|
163 | 167 | break |
|
164 | 168 | f.write(chunk) |
|
165 | 169 | |
|
166 | 170 | return {'upload': 'ok'} |
|
167 | 171 | |
|
168 | 172 | |
|
169 | 173 | def lfs_objects_oid_download(request): |
|
170 | 174 | repo = request.matchdict.get('repo') |
|
171 | 175 | oid = request.matchdict.get('oid') |
|
172 | 176 | |
|
173 | 177 | store = LFSOidStore( |
|
174 | 178 | oid, repo, store_location=request.registry.git_lfs_store_path) |
|
175 | 179 | if not store.has_oid(): |
|
176 | 180 | log.debug('LFS: oid %s does not exists in store', oid) |
|
177 | 181 | return write_response_error( |
|
178 | 182 | HTTPNotFound, 'requested file with oid `%s` not found in store' % oid) |
|
179 | 183 | |
|
180 | 184 | # TODO(marcink): support range header ? |
|
181 | 185 | # Range: bytes=0-, `bytes=(\d+)\-.*` |
|
182 | 186 | |
|
183 | 187 | f = open(store.oid_path, 'rb') |
|
184 | 188 | response = Response( |
|
185 | 189 | content_type='application/octet-stream', app_iter=FileIter(f)) |
|
186 | 190 | response.headers.add('X-RC-LFS-Response-Oid', str(oid)) |
|
187 | 191 | return response |
|
188 | 192 | |
|
189 | 193 | |
|
190 | 194 | def lfs_objects_verify(request): |
|
191 | 195 | request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json' |
|
192 | 196 | repo = request.matchdict.get('repo') |
|
193 | 197 | |
|
194 | 198 | data = request.json |
|
195 | 199 | oid = data.get('oid') |
|
196 | 200 | size = safe_int(data.get('size')) |
|
197 | 201 | |
|
198 | 202 | if not (oid and size): |
|
199 | 203 | return write_response_error( |
|
200 | 204 | HTTPBadRequest, 'missing oid and size in request data') |
|
201 | 205 | |
|
202 | 206 | store = LFSOidStore( |
|
203 | 207 | oid, repo, store_location=request.registry.git_lfs_store_path) |
|
204 | 208 | if not store.has_oid(): |
|
205 | 209 | log.debug('LFS: oid %s does not exists in store', oid) |
|
206 | 210 | return write_response_error( |
|
207 | 211 | HTTPNotFound, 'oid `%s` does not exists in store' % oid) |
|
208 | 212 | |
|
209 | 213 | store_size = store.size_oid() |
|
210 | 214 | if store_size != size: |
|
211 | 215 | msg = 'requested file size mismatch store size:%s requested:%s' % ( |
|
212 | 216 | store_size, size) |
|
213 | 217 | return write_response_error( |
|
214 | 218 | HTTPUnprocessableEntity, msg) |
|
215 | 219 | |
|
216 | 220 | return {'message': {'size': 'ok', 'in_store': 'ok'}} |
|
217 | 221 | |
|
218 | 222 | |
|
219 | 223 | def lfs_objects_lock(request): |
|
220 | 224 | return write_response_error( |
|
221 | 225 | HTTPNotImplemented, 'GIT LFS locking api not supported') |
|
222 | 226 | |
|
223 | 227 | |
|
224 | 228 | def not_found(request): |
|
225 | 229 | return write_response_error( |
|
226 | 230 | HTTPNotFound, 'request path not found') |
|
227 | 231 | |
|
228 | 232 | |
|
229 | 233 | def lfs_disabled(request): |
|
230 | 234 | return write_response_error( |
|
231 | 235 | HTTPNotImplemented, 'GIT LFS disabled for this repo') |
|
232 | 236 | |
|
233 | 237 | |
|
234 | 238 | def git_lfs_app(config): |
|
235 | 239 | |
|
236 | 240 | # v1 API deprecation endpoint |
|
237 | 241 | config.add_route('lfs_objects', |
|
238 | 242 | '/{repo:.*?[^/]}/info/lfs/objects') |
|
239 | 243 | config.add_view(lfs_objects, route_name='lfs_objects', |
|
240 | 244 | request_method='POST', renderer='json') |
|
241 | 245 | |
|
242 | 246 | # locking API |
|
243 | 247 | config.add_route('lfs_objects_lock', |
|
244 | 248 | '/{repo:.*?[^/]}/info/lfs/locks') |
|
245 | 249 | config.add_view(lfs_objects_lock, route_name='lfs_objects_lock', |
|
246 | 250 | request_method=('POST', 'GET'), renderer='json') |
|
247 | 251 | |
|
248 | 252 | config.add_route('lfs_objects_lock_verify', |
|
249 | 253 | '/{repo:.*?[^/]}/info/lfs/locks/verify') |
|
250 | 254 | config.add_view(lfs_objects_lock, route_name='lfs_objects_lock_verify', |
|
251 | 255 | request_method=('POST', 'GET'), renderer='json') |
|
252 | 256 | |
|
253 | 257 | # batch API |
|
254 | 258 | config.add_route('lfs_objects_batch', |
|
255 | 259 | '/{repo:.*?[^/]}/info/lfs/objects/batch') |
|
256 | 260 | config.add_view(lfs_objects_batch, route_name='lfs_objects_batch', |
|
257 | 261 | request_method='POST', renderer='json') |
|
258 | 262 | |
|
259 | 263 | # oid upload/download API |
|
260 | 264 | config.add_route('lfs_objects_oid', |
|
261 | 265 | '/{repo:.*?[^/]}/info/lfs/objects/{oid}') |
|
262 | 266 | config.add_view(lfs_objects_oid_upload, route_name='lfs_objects_oid', |
|
263 | 267 | request_method='PUT', renderer='json') |
|
264 | 268 | config.add_view(lfs_objects_oid_download, route_name='lfs_objects_oid', |
|
265 | 269 | request_method='GET', renderer='json') |
|
266 | 270 | |
|
267 | 271 | # verification API |
|
268 | 272 | config.add_route('lfs_objects_verify', |
|
269 | 273 | '/{repo:.*?[^/]}/info/lfs/verify') |
|
270 | 274 | config.add_view(lfs_objects_verify, route_name='lfs_objects_verify', |
|
271 | 275 | request_method='POST', renderer='json') |
|
272 | 276 | |
|
273 | 277 | # not found handler for API |
|
274 | 278 | config.add_notfound_view(not_found, renderer='json') |
|
275 | 279 | |
|
276 | 280 | |
|
277 | def create_app(git_lfs_enabled, git_lfs_store_path): | |
|
281 | def create_app(git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme): | |
|
278 | 282 | config = Configurator() |
|
279 | 283 | if git_lfs_enabled: |
|
280 | 284 | config.include(git_lfs_app) |
|
281 | 285 | config.registry.git_lfs_store_path = git_lfs_store_path |
|
286 | config.registry.git_lfs_http_scheme = git_lfs_http_scheme | |
|
282 | 287 | else: |
|
283 | 288 | # not found handler for API, reporting disabled LFS support |
|
284 | 289 | config.add_notfound_view(lfs_disabled, renderer='json') |
|
285 | 290 | |
|
286 | 291 | app = config.make_wsgi_app() |
|
287 | 292 | return app |
@@ -1,239 +1,272 b'' | |||
|
1 | 1 | # RhodeCode VCSServer provides access to different vcs backends via network. |
|
2 | 2 | # Copyright (C) 2014-2019 RhodeCode GmbH |
|
3 | 3 | # |
|
4 | 4 | # This program is free software; you can redistribute it and/or modify |
|
5 | 5 | # it under the terms of the GNU General Public License as published by |
|
6 | 6 | # the Free Software Foundation; either version 3 of the License, or |
|
7 | 7 | # (at your option) any later version. |
|
8 | 8 | # |
|
9 | 9 | # This program is distributed in the hope that it will be useful, |
|
10 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 | 12 | # GNU General Public License for more details. |
|
13 | 13 | # |
|
14 | 14 | # You should have received a copy of the GNU General Public License |
|
15 | 15 | # along with this program; if not, write to the Free Software Foundation, |
|
16 | 16 | # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
|
17 | 17 | |
|
18 | 18 | import os |
|
19 | 19 | import pytest |
|
20 | 20 | from webtest.app import TestApp as WebObTestApp |
|
21 | 21 | import simplejson as json |
|
22 | 22 | |
|
23 | 23 | from vcsserver.git_lfs.app import create_app |
|
24 | 24 | |
|
25 | 25 | |
|
26 | 26 | @pytest.fixture(scope='function') |
|
27 | 27 | def git_lfs_app(tmpdir): |
|
28 | 28 | custom_app = WebObTestApp(create_app( |
|
29 |
git_lfs_enabled=True, git_lfs_store_path=str(tmpdir) |
|
|
29 | git_lfs_enabled=True, git_lfs_store_path=str(tmpdir), | |
|
30 | git_lfs_http_scheme='http')) | |
|
31 | custom_app._store = str(tmpdir) | |
|
32 | return custom_app | |
|
33 | ||
|
34 | ||
|
35 | @pytest.fixture(scope='function') | |
|
36 | def git_lfs_https_app(tmpdir): | |
|
37 | custom_app = WebObTestApp(create_app( | |
|
38 | git_lfs_enabled=True, git_lfs_store_path=str(tmpdir), | |
|
39 | git_lfs_http_scheme='https')) | |
|
30 | 40 | custom_app._store = str(tmpdir) |
|
31 | 41 | return custom_app |
|
32 | 42 | |
|
33 | 43 | |
|
34 | 44 | @pytest.fixture() |
|
35 | 45 | def http_auth(): |
|
36 | 46 | return {'HTTP_AUTHORIZATION': "Basic XXXXX"} |
|
37 | 47 | |
|
38 | 48 | |
|
39 | 49 | class TestLFSApplication(object): |
|
40 | 50 | |
|
41 | 51 | def test_app_wrong_path(self, git_lfs_app): |
|
42 | 52 | git_lfs_app.get('/repo/info/lfs/xxx', status=404) |
|
43 | 53 | |
|
44 | 54 | def test_app_deprecated_endpoint(self, git_lfs_app): |
|
45 | 55 | response = git_lfs_app.post('/repo/info/lfs/objects', status=501) |
|
46 | 56 | assert response.status_code == 501 |
|
47 | 57 | assert json.loads(response.text) == {u'message': u'LFS: v1 api not supported'} |
|
48 | 58 | |
|
49 | 59 | def test_app_lock_verify_api_not_available(self, git_lfs_app): |
|
50 | 60 | response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501) |
|
51 | 61 | assert response.status_code == 501 |
|
52 | 62 | assert json.loads(response.text) == { |
|
53 | 63 | u'message': u'GIT LFS locking api not supported'} |
|
54 | 64 | |
|
55 | 65 | def test_app_lock_api_not_available(self, git_lfs_app): |
|
56 | 66 | response = git_lfs_app.post('/repo/info/lfs/locks', status=501) |
|
57 | 67 | assert response.status_code == 501 |
|
58 | 68 | assert json.loads(response.text) == { |
|
59 | 69 | u'message': u'GIT LFS locking api not supported'} |
|
60 | 70 | |
|
61 |
def test_app_batch_api_missing_auth(self, git_lfs_app |
|
|
71 | def test_app_batch_api_missing_auth(self, git_lfs_app): | |
|
62 | 72 | git_lfs_app.post_json( |
|
63 | 73 | '/repo/info/lfs/objects/batch', params={}, status=403) |
|
64 | 74 | |
|
65 | 75 | def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth): |
|
66 | 76 | response = git_lfs_app.post_json( |
|
67 | 77 | '/repo/info/lfs/objects/batch', params={}, status=400, |
|
68 | 78 | extra_environ=http_auth) |
|
69 | 79 | assert json.loads(response.text) == { |
|
70 | 80 | u'message': u'unsupported operation mode: `None`'} |
|
71 | 81 | |
|
72 | 82 | def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth): |
|
73 | 83 | response = git_lfs_app.post_json( |
|
74 | 84 | '/repo/info/lfs/objects/batch', params={'operation': 'download'}, |
|
75 | 85 | status=400, extra_environ=http_auth) |
|
76 | 86 | assert json.loads(response.text) == { |
|
77 | 87 | u'message': u'missing objects data'} |
|
78 | 88 | |
|
79 | 89 | def test_app_batch_api_unsupported_data_in_objects( |
|
80 | 90 | self, git_lfs_app, http_auth): |
|
81 | 91 | params = {'operation': 'download', |
|
82 | 92 | 'objects': [{}]} |
|
83 | 93 | response = git_lfs_app.post_json( |
|
84 | 94 | '/repo/info/lfs/objects/batch', params=params, status=400, |
|
85 | 95 | extra_environ=http_auth) |
|
86 | 96 | assert json.loads(response.text) == { |
|
87 | 97 | u'message': u'unsupported data in objects'} |
|
88 | 98 | |
|
89 | 99 | def test_app_batch_api_download_missing_object( |
|
90 | 100 | self, git_lfs_app, http_auth): |
|
91 | 101 | params = {'operation': 'download', |
|
92 | 102 | 'objects': [{'oid': '123', 'size': '1024'}]} |
|
93 | 103 | response = git_lfs_app.post_json( |
|
94 | 104 | '/repo/info/lfs/objects/batch', params=params, |
|
95 | 105 | extra_environ=http_auth) |
|
96 | 106 | |
|
97 | 107 | expected_objects = [ |
|
98 | 108 | {u'authenticated': True, |
|
99 | 109 | u'errors': {u'error': { |
|
100 | 110 | u'code': 404, |
|
101 | 111 | u'message': u'object: 123 does not exist in store'}}, |
|
102 | 112 | u'oid': u'123', |
|
103 | 113 | u'size': u'1024'} |
|
104 | 114 | ] |
|
105 | 115 | assert json.loads(response.text) == { |
|
106 | 116 | 'objects': expected_objects, 'transfer': 'basic'} |
|
107 | 117 | |
|
108 | 118 | def test_app_batch_api_download(self, git_lfs_app, http_auth): |
|
109 | 119 | oid = '456' |
|
110 | 120 | oid_path = os.path.join(git_lfs_app._store, oid) |
|
111 | 121 | if not os.path.isdir(os.path.dirname(oid_path)): |
|
112 | 122 | os.makedirs(os.path.dirname(oid_path)) |
|
113 | 123 | with open(oid_path, 'wb') as f: |
|
114 | 124 | f.write('OID_CONTENT') |
|
115 | 125 | |
|
116 | 126 | params = {'operation': 'download', |
|
117 | 127 | 'objects': [{'oid': oid, 'size': '1024'}]} |
|
118 | 128 | response = git_lfs_app.post_json( |
|
119 | 129 | '/repo/info/lfs/objects/batch', params=params, |
|
120 | 130 | extra_environ=http_auth) |
|
121 | 131 | |
|
122 | 132 | expected_objects = [ |
|
123 | 133 | {u'authenticated': True, |
|
124 | 134 | u'actions': { |
|
125 | 135 | u'download': { |
|
126 | 136 | u'header': {u'Authorization': u'Basic XXXXX'}, |
|
127 | 137 | u'href': u'http://localhost/repo/info/lfs/objects/456'}, |
|
128 | 138 | }, |
|
129 | 139 | u'oid': u'456', |
|
130 | 140 | u'size': u'1024'} |
|
131 | 141 | ] |
|
132 | 142 | assert json.loads(response.text) == { |
|
133 | 143 | 'objects': expected_objects, 'transfer': 'basic'} |
|
134 | 144 | |
|
135 | 145 | def test_app_batch_api_upload(self, git_lfs_app, http_auth): |
|
136 | 146 | params = {'operation': 'upload', |
|
137 | 147 | 'objects': [{'oid': '123', 'size': '1024'}]} |
|
138 | 148 | response = git_lfs_app.post_json( |
|
139 | 149 | '/repo/info/lfs/objects/batch', params=params, |
|
140 | 150 | extra_environ=http_auth) |
|
141 | 151 | expected_objects = [ |
|
142 | 152 | {u'authenticated': True, |
|
143 | 153 | u'actions': { |
|
144 | 154 | u'upload': { |
|
145 | 155 | u'header': {u'Authorization': u'Basic XXXXX', |
|
146 | 156 | u'Transfer-Encoding': u'chunked'}, |
|
147 | 157 | u'href': u'http://localhost/repo/info/lfs/objects/123'}, |
|
148 | 158 | u'verify': { |
|
149 | 159 | u'header': {u'Authorization': u'Basic XXXXX'}, |
|
150 | 160 | u'href': u'http://localhost/repo/info/lfs/verify'} |
|
151 | 161 | }, |
|
152 | 162 | u'oid': u'123', |
|
153 | 163 | u'size': u'1024'} |
|
154 | 164 | ] |
|
155 | 165 | assert json.loads(response.text) == { |
|
156 | 166 | 'objects': expected_objects, 'transfer': 'basic'} |
|
157 | 167 | |
|
168 | def test_app_batch_api_upload_for_https(self, git_lfs_https_app, http_auth): | |
|
169 | params = {'operation': 'upload', | |
|
170 | 'objects': [{'oid': '123', 'size': '1024'}]} | |
|
171 | response = git_lfs_https_app.post_json( | |
|
172 | '/repo/info/lfs/objects/batch', params=params, | |
|
173 | extra_environ=http_auth) | |
|
174 | expected_objects = [ | |
|
175 | {u'authenticated': True, | |
|
176 | u'actions': { | |
|
177 | u'upload': { | |
|
178 | u'header': {u'Authorization': u'Basic XXXXX', | |
|
179 | u'Transfer-Encoding': u'chunked'}, | |
|
180 | u'href': u'https://localhost/repo/info/lfs/objects/123'}, | |
|
181 | u'verify': { | |
|
182 | u'header': {u'Authorization': u'Basic XXXXX'}, | |
|
183 | u'href': u'https://localhost/repo/info/lfs/verify'} | |
|
184 | }, | |
|
185 | u'oid': u'123', | |
|
186 | u'size': u'1024'} | |
|
187 | ] | |
|
188 | assert json.loads(response.text) == { | |
|
189 | 'objects': expected_objects, 'transfer': 'basic'} | |
|
190 | ||
|
158 | 191 | def test_app_verify_api_missing_data(self, git_lfs_app): |
|
159 |
params = {'oid': 'missing' |
|
|
192 | params = {'oid': 'missing'} | |
|
160 | 193 | response = git_lfs_app.post_json( |
|
161 | 194 | '/repo/info/lfs/verify', params=params, |
|
162 | 195 | status=400) |
|
163 | 196 | |
|
164 | 197 | assert json.loads(response.text) == { |
|
165 | 198 | u'message': u'missing oid and size in request data'} |
|
166 | 199 | |
|
167 | 200 | def test_app_verify_api_missing_obj(self, git_lfs_app): |
|
168 | 201 | params = {'oid': 'missing', 'size': '1024'} |
|
169 | 202 | response = git_lfs_app.post_json( |
|
170 | 203 | '/repo/info/lfs/verify', params=params, |
|
171 | 204 | status=404) |
|
172 | 205 | |
|
173 | 206 | assert json.loads(response.text) == { |
|
174 | 207 | u'message': u'oid `missing` does not exists in store'} |
|
175 | 208 | |
|
176 | 209 | def test_app_verify_api_size_mismatch(self, git_lfs_app): |
|
177 | 210 | oid = 'existing' |
|
178 | 211 | oid_path = os.path.join(git_lfs_app._store, oid) |
|
179 | 212 | if not os.path.isdir(os.path.dirname(oid_path)): |
|
180 | 213 | os.makedirs(os.path.dirname(oid_path)) |
|
181 | 214 | with open(oid_path, 'wb') as f: |
|
182 | 215 | f.write('OID_CONTENT') |
|
183 | 216 | |
|
184 | 217 | params = {'oid': oid, 'size': '1024'} |
|
185 | 218 | response = git_lfs_app.post_json( |
|
186 | 219 | '/repo/info/lfs/verify', params=params, status=422) |
|
187 | 220 | |
|
188 | 221 | assert json.loads(response.text) == { |
|
189 | 222 | u'message': u'requested file size mismatch ' |
|
190 | 223 | u'store size:11 requested:1024'} |
|
191 | 224 | |
|
192 | 225 | def test_app_verify_api(self, git_lfs_app): |
|
193 | 226 | oid = 'existing' |
|
194 | 227 | oid_path = os.path.join(git_lfs_app._store, oid) |
|
195 | 228 | if not os.path.isdir(os.path.dirname(oid_path)): |
|
196 | 229 | os.makedirs(os.path.dirname(oid_path)) |
|
197 | 230 | with open(oid_path, 'wb') as f: |
|
198 | 231 | f.write('OID_CONTENT') |
|
199 | 232 | |
|
200 | 233 | params = {'oid': oid, 'size': 11} |
|
201 | 234 | response = git_lfs_app.post_json( |
|
202 | 235 | '/repo/info/lfs/verify', params=params) |
|
203 | 236 | |
|
204 | 237 | assert json.loads(response.text) == { |
|
205 | 238 | u'message': {u'size': u'ok', u'in_store': u'ok'}} |
|
206 | 239 | |
|
207 | 240 | def test_app_download_api_oid_not_existing(self, git_lfs_app): |
|
208 | 241 | oid = 'missing' |
|
209 | 242 | |
|
210 | 243 | response = git_lfs_app.get( |
|
211 | 244 | '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404) |
|
212 | 245 | |
|
213 | 246 | assert json.loads(response.text) == { |
|
214 | 247 | u'message': u'requested file with oid `missing` not found in store'} |
|
215 | 248 | |
|
216 | 249 | def test_app_download_api(self, git_lfs_app): |
|
217 | 250 | oid = 'existing' |
|
218 | 251 | oid_path = os.path.join(git_lfs_app._store, oid) |
|
219 | 252 | if not os.path.isdir(os.path.dirname(oid_path)): |
|
220 | 253 | os.makedirs(os.path.dirname(oid_path)) |
|
221 | 254 | with open(oid_path, 'wb') as f: |
|
222 | 255 | f.write('OID_CONTENT') |
|
223 | 256 | |
|
224 | 257 | response = git_lfs_app.get( |
|
225 | 258 | '/repo/info/lfs/objects/{oid}'.format(oid=oid)) |
|
226 | 259 | assert response |
|
227 | 260 | |
|
228 | 261 | def test_app_upload(self, git_lfs_app): |
|
229 | 262 | oid = 'uploaded' |
|
230 | 263 | |
|
231 | 264 | response = git_lfs_app.put( |
|
232 | 265 | '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT') |
|
233 | 266 | |
|
234 | 267 | assert json.loads(response.text) == {u'upload': u'ok'} |
|
235 | 268 | |
|
236 | 269 | # verify that we actually wrote that OID |
|
237 | 270 | oid_path = os.path.join(git_lfs_app._store, oid) |
|
238 | 271 | assert os.path.isfile(oid_path) |
|
239 | 272 | assert 'CONTENT' == open(oid_path).read() |
@@ -1,234 +1,235 b'' | |||
|
1 | 1 | # RhodeCode VCSServer provides access to different vcs backends via network. |
|
2 | 2 | # Copyright (C) 2014-2019 RhodeCode GmbH |
|
3 | 3 | # |
|
4 | 4 | # This program is free software; you can redistribute it and/or modify |
|
5 | 5 | # it under the terms of the GNU General Public License as published by |
|
6 | 6 | # the Free Software Foundation; either version 3 of the License, or |
|
7 | 7 | # (at your option) any later version. |
|
8 | 8 | # |
|
9 | 9 | # This program is distributed in the hope that it will be useful, |
|
10 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 | 12 | # GNU General Public License for more details. |
|
13 | 13 | # |
|
14 | 14 | # You should have received a copy of the GNU General Public License |
|
15 | 15 | # along with this program; if not, write to the Free Software Foundation, |
|
16 | 16 | # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
|
17 | 17 | |
|
18 | 18 | import os |
|
19 | 19 | import logging |
|
20 | 20 | import itertools |
|
21 | 21 | |
|
22 | 22 | import mercurial |
|
23 | 23 | import mercurial.error |
|
24 | 24 | import mercurial.wireprotoserver |
|
25 | 25 | import mercurial.hgweb.common |
|
26 | 26 | import mercurial.hgweb.hgweb_mod |
|
27 | 27 | import webob.exc |
|
28 | 28 | |
|
29 | 29 | from vcsserver import pygrack, exceptions, settings, git_lfs |
|
30 | 30 | |
|
31 | 31 | |
|
32 | 32 | log = logging.getLogger(__name__) |
|
33 | 33 | |
|
34 | 34 | |
|
35 | 35 | # propagated from mercurial documentation |
|
36 | 36 | HG_UI_SECTIONS = [ |
|
37 | 37 | 'alias', 'auth', 'decode/encode', 'defaults', 'diff', 'email', 'extensions', |
|
38 | 38 | 'format', 'merge-patterns', 'merge-tools', 'hooks', 'http_proxy', 'smtp', |
|
39 | 39 | 'patch', 'paths', 'profiling', 'server', 'trusted', 'ui', 'web', |
|
40 | 40 | ] |
|
41 | 41 | |
|
42 | 42 | |
|
43 | 43 | class HgWeb(mercurial.hgweb.hgweb_mod.hgweb): |
|
44 | 44 | """Extension of hgweb that simplifies some functions.""" |
|
45 | 45 | |
|
46 | 46 | def _get_view(self, repo): |
|
47 | 47 | """Views are not supported.""" |
|
48 | 48 | return repo |
|
49 | 49 | |
|
50 | 50 | def loadsubweb(self): |
|
51 | 51 | """The result is only used in the templater method which is not used.""" |
|
52 | 52 | return None |
|
53 | 53 | |
|
54 | 54 | def run(self): |
|
55 | 55 | """Unused function so raise an exception if accidentally called.""" |
|
56 | 56 | raise NotImplementedError |
|
57 | 57 | |
|
58 | 58 | def templater(self, req): |
|
59 | 59 | """Function used in an unreachable code path. |
|
60 | 60 | |
|
61 | 61 | This code is unreachable because we guarantee that the HTTP request, |
|
62 | 62 | corresponds to a Mercurial command. See the is_hg method. So, we are |
|
63 | 63 | never going to get a user-visible url. |
|
64 | 64 | """ |
|
65 | 65 | raise NotImplementedError |
|
66 | 66 | |
|
67 | 67 | def archivelist(self, nodeid): |
|
68 | 68 | """Unused function so raise an exception if accidentally called.""" |
|
69 | 69 | raise NotImplementedError |
|
70 | 70 | |
|
71 | 71 | def __call__(self, environ, start_response): |
|
72 | 72 | """Run the WSGI application. |
|
73 | 73 | |
|
74 | 74 | This may be called by multiple threads. |
|
75 | 75 | """ |
|
76 | 76 | from mercurial.hgweb import request as requestmod |
|
77 | 77 | req = requestmod.parserequestfromenv(environ) |
|
78 | 78 | res = requestmod.wsgiresponse(req, start_response) |
|
79 | 79 | gen = self.run_wsgi(req, res) |
|
80 | 80 | |
|
81 | 81 | first_chunk = None |
|
82 | 82 | |
|
83 | 83 | try: |
|
84 | 84 | data = gen.next() |
|
85 | 85 | |
|
86 | 86 | def first_chunk(): |
|
87 | 87 | yield data |
|
88 | 88 | except StopIteration: |
|
89 | 89 | pass |
|
90 | 90 | |
|
91 | 91 | if first_chunk: |
|
92 | 92 | return itertools.chain(first_chunk(), gen) |
|
93 | 93 | return gen |
|
94 | 94 | |
|
95 | 95 | def _runwsgi(self, req, res, repo): |
|
96 | 96 | |
|
97 | 97 | cmd = req.qsparams.get('cmd', '') |
|
98 | 98 | if not mercurial.wireprotoserver.iscmd(cmd): |
|
99 | 99 | # NOTE(marcink): for unsupported commands, we return bad request |
|
100 | 100 | # internally from HG |
|
101 | 101 | from mercurial.hgweb.common import statusmessage |
|
102 | 102 | res.status = statusmessage(mercurial.hgweb.common.HTTP_BAD_REQUEST) |
|
103 | 103 | res.setbodybytes('') |
|
104 | 104 | return res.sendresponse() |
|
105 | 105 | |
|
106 | 106 | return super(HgWeb, self)._runwsgi(req, res, repo) |
|
107 | 107 | |
|
108 | 108 | |
|
109 | 109 | def make_hg_ui_from_config(repo_config): |
|
110 | 110 | baseui = mercurial.ui.ui() |
|
111 | 111 | |
|
112 | 112 | # clean the baseui object |
|
113 | 113 | baseui._ocfg = mercurial.config.config() |
|
114 | 114 | baseui._ucfg = mercurial.config.config() |
|
115 | 115 | baseui._tcfg = mercurial.config.config() |
|
116 | 116 | |
|
117 | 117 | for section, option, value in repo_config: |
|
118 | 118 | baseui.setconfig(section, option, value) |
|
119 | 119 | |
|
120 | 120 | # make our hgweb quiet so it doesn't print output |
|
121 | 121 | baseui.setconfig('ui', 'quiet', 'true') |
|
122 | 122 | |
|
123 | 123 | return baseui |
|
124 | 124 | |
|
125 | 125 | |
|
126 | 126 | def update_hg_ui_from_hgrc(baseui, repo_path): |
|
127 | 127 | path = os.path.join(repo_path, '.hg', 'hgrc') |
|
128 | 128 | |
|
129 | 129 | if not os.path.isfile(path): |
|
130 | 130 | log.debug('hgrc file is not present at %s, skipping...', path) |
|
131 | 131 | return |
|
132 | 132 | log.debug('reading hgrc from %s', path) |
|
133 | 133 | cfg = mercurial.config.config() |
|
134 | 134 | cfg.read(path) |
|
135 | 135 | for section in HG_UI_SECTIONS: |
|
136 | 136 | for k, v in cfg.items(section): |
|
137 | 137 | log.debug('settings ui from file: [%s] %s=%s', section, k, v) |
|
138 | 138 | baseui.setconfig(section, k, v) |
|
139 | 139 | |
|
140 | 140 | |
|
141 | 141 | def create_hg_wsgi_app(repo_path, repo_name, config): |
|
142 | 142 | """ |
|
143 | 143 | Prepares a WSGI application to handle Mercurial requests. |
|
144 | 144 | |
|
145 | 145 | :param config: is a list of 3-item tuples representing a ConfigObject |
|
146 | 146 | (it is the serialized version of the config object). |
|
147 | 147 | """ |
|
148 | 148 | log.debug("Creating Mercurial WSGI application") |
|
149 | 149 | |
|
150 | 150 | baseui = make_hg_ui_from_config(config) |
|
151 | 151 | update_hg_ui_from_hgrc(baseui, repo_path) |
|
152 | 152 | |
|
153 | 153 | try: |
|
154 | 154 | return HgWeb(repo_path, name=repo_name, baseui=baseui) |
|
155 | 155 | except mercurial.error.RequirementError as e: |
|
156 | 156 | raise exceptions.RequirementException(e)(e) |
|
157 | 157 | |
|
158 | 158 | |
|
159 | 159 | class GitHandler(object): |
|
160 | 160 | """ |
|
161 | 161 | Handler for Git operations like push/pull etc |
|
162 | 162 | """ |
|
163 | 163 | def __init__(self, repo_location, repo_name, git_path, update_server_info, |
|
164 | 164 | extras): |
|
165 | 165 | if not os.path.isdir(repo_location): |
|
166 | 166 | raise OSError(repo_location) |
|
167 | 167 | self.content_path = repo_location |
|
168 | 168 | self.repo_name = repo_name |
|
169 | 169 | self.repo_location = repo_location |
|
170 | 170 | self.extras = extras |
|
171 | 171 | self.git_path = git_path |
|
172 | 172 | self.update_server_info = update_server_info |
|
173 | 173 | |
|
174 | 174 | def __call__(self, environ, start_response): |
|
175 | 175 | app = webob.exc.HTTPNotFound() |
|
176 | 176 | candidate_paths = ( |
|
177 | 177 | self.content_path, os.path.join(self.content_path, '.git')) |
|
178 | 178 | |
|
179 | 179 | for content_path in candidate_paths: |
|
180 | 180 | try: |
|
181 | 181 | app = pygrack.GitRepository( |
|
182 | 182 | self.repo_name, content_path, self.git_path, |
|
183 | 183 | self.update_server_info, self.extras) |
|
184 | 184 | break |
|
185 | 185 | except OSError: |
|
186 | 186 | continue |
|
187 | 187 | |
|
188 | 188 | return app(environ, start_response) |
|
189 | 189 | |
|
190 | 190 | |
|
191 | 191 | def create_git_wsgi_app(repo_path, repo_name, config): |
|
192 | 192 | """ |
|
193 | 193 | Creates a WSGI application to handle Git requests. |
|
194 | 194 | |
|
195 | 195 | :param config: is a dictionary holding the extras. |
|
196 | 196 | """ |
|
197 | 197 | git_path = settings.GIT_EXECUTABLE |
|
198 | 198 | update_server_info = config.pop('git_update_server_info') |
|
199 | 199 | app = GitHandler( |
|
200 | 200 | repo_path, repo_name, git_path, update_server_info, config) |
|
201 | 201 | |
|
202 | 202 | return app |
|
203 | 203 | |
|
204 | 204 | |
|
205 | 205 | class GitLFSHandler(object): |
|
206 | 206 | """ |
|
207 | 207 | Handler for Git LFS operations |
|
208 | 208 | """ |
|
209 | 209 | |
|
210 | 210 | def __init__(self, repo_location, repo_name, git_path, update_server_info, |
|
211 | 211 | extras): |
|
212 | 212 | if not os.path.isdir(repo_location): |
|
213 | 213 | raise OSError(repo_location) |
|
214 | 214 | self.content_path = repo_location |
|
215 | 215 | self.repo_name = repo_name |
|
216 | 216 | self.repo_location = repo_location |
|
217 | 217 | self.extras = extras |
|
218 | 218 | self.git_path = git_path |
|
219 | 219 | self.update_server_info = update_server_info |
|
220 | 220 | |
|
221 | def get_app(self, git_lfs_enabled, git_lfs_store_path): | |
|
222 | app = git_lfs.create_app(git_lfs_enabled, git_lfs_store_path) | |
|
221 | def get_app(self, git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme): | |
|
222 | app = git_lfs.create_app(git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme) | |
|
223 | 223 | return app |
|
224 | 224 | |
|
225 | 225 | |
|
226 | 226 | def create_git_lfs_wsgi_app(repo_path, repo_name, config): |
|
227 | 227 | git_path = settings.GIT_EXECUTABLE |
|
228 | 228 | update_server_info = config.pop('git_update_server_info') |
|
229 | 229 | git_lfs_enabled = config.pop('git_lfs_enabled') |
|
230 | 230 | git_lfs_store_path = config.pop('git_lfs_store_path') |
|
231 | git_lfs_http_scheme = config.pop('git_lfs_http_scheme', 'http') | |
|
231 | 232 | app = GitLFSHandler( |
|
232 | 233 | repo_path, repo_name, git_path, update_server_info, config) |
|
233 | 234 | |
|
234 | return app.get_app(git_lfs_enabled, git_lfs_store_path) | |
|
235 | return app.get_app(git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme) |
General Comments 0
You need to be logged in to leave comments.
Login now