##// END OF EJS Templates
git-lfs: report returned data as 'application/vnd.git-lfs+json' instead of plain json
marcink -
r204:b0f7b2ca stable
parent child Browse files
Show More
@@ -1,276 +1,277 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import re
19 19 import logging
20 20 from wsgiref.util import FileWrapper
21 21
22 22 import simplejson as json
23 23 from pyramid.config import Configurator
24 24 from pyramid.response import Response, FileIter
25 25 from pyramid.httpexceptions import (
26 26 HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden,
27 27 HTTPUnprocessableEntity)
28 28
29 29 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
30 30 from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator
31 31 from vcsserver.utils import safe_int
32 32
33 33 log = logging.getLogger(__name__)
34 34
35 35
36 36 GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs' #+json ?
37 37 GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))')
38 38
39 39
40 40 def write_response_error(http_exception, text=None):
41 content_type = 'application/json'
41 content_type = GIT_LFS_CONTENT_TYPE + '+json'
42 42 _exception = http_exception(content_type=content_type)
43 43 _exception.content_type = content_type
44 44 if text:
45 45 _exception.body = json.dumps({'message': text})
46 46 log.debug('LFS: writing response of type %s to client with text:%s',
47 47 http_exception, text)
48 48 return _exception
49 49
50 50
51 51 class AuthHeaderRequired(object):
52 52 """
53 53 Decorator to check if request has proper auth-header
54 54 """
55 55
56 56 def __call__(self, func):
57 57 return get_cython_compat_decorator(self.__wrapper, func)
58 58
59 59 def __wrapper(self, func, *fargs, **fkwargs):
60 60 request = fargs[1]
61 61 auth = request.authorization
62 62 if not auth:
63 63 return write_response_error(HTTPForbidden)
64 64 return func(*fargs[1:], **fkwargs)
65 65
66 66
67 67 # views
68 68
69 69 def lfs_objects(request):
70 70 # indicate not supported, V1 API
71 71 log.warning('LFS: v1 api not supported, reporting it back to client')
72 72 return write_response_error(HTTPNotImplemented, 'LFS: v1 api not supported')
73 73
74 74
75 75 @AuthHeaderRequired()
76 76 def lfs_objects_batch(request):
77 77 """
78 78 The client sends the following information to the Batch endpoint to transfer some objects:
79 79
80 80 operation - Should be download or upload.
81 81 transfers - An optional Array of String identifiers for transfer
82 82 adapters that the client has configured. If omitted, the basic
83 83 transfer adapter MUST be assumed by the server.
84 84 objects - An Array of objects to download.
85 85 oid - String OID of the LFS object.
86 86 size - Integer byte size of the LFS object. Must be at least zero.
87 87 """
88 request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
88 89 auth = request.authorization
89
90 90 repo = request.matchdict.get('repo')
91
92 91 data = request.json
93 92 operation = data.get('operation')
94 93 if operation not in ('download', 'upload'):
95 94 log.debug('LFS: unsupported operation:%s', operation)
96 95 return write_response_error(
97 96 HTTPBadRequest, 'unsupported operation mode: `%s`' % operation)
98 97
99 98 if 'objects' not in data:
100 99 log.debug('LFS: missing objects data')
101 100 return write_response_error(
102 101 HTTPBadRequest, 'missing objects data')
103 102
104 103 log.debug('LFS: handling operation of type: %s', operation)
105 104
106 105 objects = []
107 106 for o in data['objects']:
108 107 try:
109 108 oid = o['oid']
110 109 obj_size = o['size']
111 110 except KeyError:
112 111 log.exception('LFS, failed to extract data')
113 112 return write_response_error(
114 113 HTTPBadRequest, 'unsupported data in objects')
115 114
116 115 obj_data = {'oid': oid}
117 116
118 117 obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid)
119 118 obj_verify_href = request.route_url('lfs_objects_verify', repo=repo)
120 119 store = LFSOidStore(
121 120 oid, repo, store_location=request.registry.git_lfs_store_path)
122 121 handler = OidHandler(
123 122 store, repo, auth, oid, obj_size, obj_data,
124 123 obj_href, obj_verify_href)
125 124
126 125 # this verifies also OIDs
127 126 actions, errors = handler.exec_operation(operation)
128 127 if errors:
129 128 log.warning('LFS: got following errors: %s', errors)
130 129 obj_data['errors'] = errors
131 130
132 131 if actions:
133 132 obj_data['actions'] = actions
134 133
135 134 obj_data['size'] = obj_size
136 135 obj_data['authenticated'] = True
137 136 objects.append(obj_data)
138 137
139 138 result = {'objects': objects, 'transfer': 'basic'}
140 139 log.debug('LFS Response %s', safe_result(result))
141 140
142 141 return result
143 142
144 143
145 144 def lfs_objects_oid_upload(request):
145 request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
146 146 repo = request.matchdict.get('repo')
147 147 oid = request.matchdict.get('oid')
148 148 store = LFSOidStore(
149 149 oid, repo, store_location=request.registry.git_lfs_store_path)
150 150 engine = store.get_engine(mode='wb')
151 151 log.debug('LFS: starting chunked write of LFS oid: %s to storage', oid)
152 152 with engine as f:
153 153 for chunk in FileWrapper(request.body_file_seekable, blksize=64 * 1024):
154 154 f.write(chunk)
155 155
156 156 return {'upload': 'ok'}
157 157
158 158
159 159 def lfs_objects_oid_download(request):
160 160 repo = request.matchdict.get('repo')
161 161 oid = request.matchdict.get('oid')
162 162
163 163 store = LFSOidStore(
164 164 oid, repo, store_location=request.registry.git_lfs_store_path)
165 165 if not store.has_oid():
166 166 log.debug('LFS: oid %s does not exists in store', oid)
167 167 return write_response_error(
168 168 HTTPNotFound, 'requested file with oid `%s` not found in store' % oid)
169 169
170 170 # TODO(marcink): support range header ?
171 171 # Range: bytes=0-, `bytes=(\d+)\-.*`
172 172
173 173 f = open(store.oid_path, 'rb')
174 174 response = Response(
175 175 content_type='application/octet-stream', app_iter=FileIter(f))
176 176 response.headers.add('X-RC-LFS-Response-Oid', str(oid))
177 177 return response
178 178
179 179
180 180 def lfs_objects_verify(request):
181 request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
181 182 repo = request.matchdict.get('repo')
182 183
183 184 data = request.json
184 185 oid = data.get('oid')
185 186 size = safe_int(data.get('size'))
186 187
187 188 if not (oid and size):
188 189 return write_response_error(
189 190 HTTPBadRequest, 'missing oid and size in request data')
190 191
191 192 store = LFSOidStore(
192 193 oid, repo, store_location=request.registry.git_lfs_store_path)
193 194 if not store.has_oid():
194 195 log.debug('LFS: oid %s does not exists in store', oid)
195 196 return write_response_error(
196 197 HTTPNotFound, 'oid `%s` does not exists in store' % oid)
197 198
198 199 store_size = store.size_oid()
199 200 if store_size != size:
200 201 msg = 'requested file size mismatch store size:%s requested:%s' % (
201 202 store_size, size)
202 203 return write_response_error(
203 204 HTTPUnprocessableEntity, msg)
204 205
205 206 return {'message': {'size': 'ok', 'in_store': 'ok'}}
206 207
207 208
208 209 def lfs_objects_lock(request):
209 210 return write_response_error(
210 211 HTTPNotImplemented, 'GIT LFS locking api not supported')
211 212
212 213
213 214 def not_found(request):
214 215 return write_response_error(
215 216 HTTPNotFound, 'request path not found')
216 217
217 218
218 219 def lfs_disabled(request):
219 220 return write_response_error(
220 221 HTTPNotImplemented, 'GIT LFS disabled for this repo')
221 222
222 223
223 224 def git_lfs_app(config):
224 225
225 226 # v1 API deprecation endpoint
226 227 config.add_route('lfs_objects',
227 228 '/{repo:.*?[^/]}/info/lfs/objects')
228 229 config.add_view(lfs_objects, route_name='lfs_objects',
229 230 request_method='POST', renderer='json')
230 231
231 232 # locking API
232 233 config.add_route('lfs_objects_lock',
233 234 '/{repo:.*?[^/]}/info/lfs/locks')
234 235 config.add_view(lfs_objects_lock, route_name='lfs_objects_lock',
235 236 request_method=('POST', 'GET'), renderer='json')
236 237
237 238 config.add_route('lfs_objects_lock_verify',
238 239 '/{repo:.*?[^/]}/info/lfs/locks/verify')
239 240 config.add_view(lfs_objects_lock, route_name='lfs_objects_lock_verify',
240 241 request_method=('POST', 'GET'), renderer='json')
241 242
242 243 # batch API
243 244 config.add_route('lfs_objects_batch',
244 245 '/{repo:.*?[^/]}/info/lfs/objects/batch')
245 246 config.add_view(lfs_objects_batch, route_name='lfs_objects_batch',
246 247 request_method='POST', renderer='json')
247 248
248 249 # oid upload/download API
249 250 config.add_route('lfs_objects_oid',
250 251 '/{repo:.*?[^/]}/info/lfs/objects/{oid}')
251 252 config.add_view(lfs_objects_oid_upload, route_name='lfs_objects_oid',
252 253 request_method='PUT', renderer='json')
253 254 config.add_view(lfs_objects_oid_download, route_name='lfs_objects_oid',
254 255 request_method='GET', renderer='json')
255 256
256 257 # verification API
257 258 config.add_route('lfs_objects_verify',
258 259 '/{repo:.*?[^/]}/info/lfs/verify')
259 260 config.add_view(lfs_objects_verify, route_name='lfs_objects_verify',
260 261 request_method='POST', renderer='json')
261 262
262 263 # not found handler for API
263 264 config.add_notfound_view(not_found, renderer='json')
264 265
265 266
266 267 def create_app(git_lfs_enabled, git_lfs_store_path):
267 268 config = Configurator()
268 269 if git_lfs_enabled:
269 270 config.include(git_lfs_app)
270 271 config.registry.git_lfs_store_path = git_lfs_store_path
271 272 else:
272 273 # not found handler for API, reporting disabled LFS support
273 274 config.add_notfound_view(lfs_disabled, renderer='json')
274 275
275 276 app = config.make_wsgi_app()
276 277 return app
@@ -1,237 +1,238 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import os
19 19 import pytest
20 20 from webtest.app import TestApp as WebObTestApp
21 import simplejson as json
21 22
22 23 from vcsserver.git_lfs.app import create_app
23 24
24 25
25 26 @pytest.fixture(scope='function')
26 27 def git_lfs_app(tmpdir):
27 28 custom_app = WebObTestApp(create_app(
28 29 git_lfs_enabled=True, git_lfs_store_path=str(tmpdir)))
29 30 custom_app._store = str(tmpdir)
30 31 return custom_app
31 32
32 33
33 34 @pytest.fixture()
34 35 def http_auth():
35 36 return {'HTTP_AUTHORIZATION': "Basic XXXXX"}
36 37
37 38
38 39 class TestLFSApplication(object):
39 40
40 41 def test_app_wrong_path(self, git_lfs_app):
41 42 git_lfs_app.get('/repo/info/lfs/xxx', status=404)
42 43
43 44 def test_app_deprecated_endpoint(self, git_lfs_app):
44 45 response = git_lfs_app.post('/repo/info/lfs/objects', status=501)
45 46 assert response.status_code == 501
46 assert response.json == {u'message': u'LFS: v1 api not supported'}
47 assert json.loads(response.text) == {u'message': u'LFS: v1 api not supported'}
47 48
48 49 def test_app_lock_verify_api_not_available(self, git_lfs_app):
49 50 response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501)
50 51 assert response.status_code == 501
51 assert response.json == {
52 assert json.loads(response.text) == {
52 53 u'message': u'GIT LFS locking api not supported'}
53 54
54 55 def test_app_lock_api_not_available(self, git_lfs_app):
55 56 response = git_lfs_app.post('/repo/info/lfs/locks', status=501)
56 57 assert response.status_code == 501
57 assert response.json == {
58 assert json.loads(response.text) == {
58 59 u'message': u'GIT LFS locking api not supported'}
59 60
60 61 def test_app_batch_api_missing_auth(self, git_lfs_app,):
61 62 git_lfs_app.post_json(
62 63 '/repo/info/lfs/objects/batch', params={}, status=403)
63 64
64 65 def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth):
65 66 response = git_lfs_app.post_json(
66 67 '/repo/info/lfs/objects/batch', params={}, status=400,
67 68 extra_environ=http_auth)
68 assert response.json == {
69 assert json.loads(response.text) == {
69 70 u'message': u'unsupported operation mode: `None`'}
70 71
71 72 def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth):
72 73 response = git_lfs_app.post_json(
73 74 '/repo/info/lfs/objects/batch', params={'operation': 'download'},
74 75 status=400, extra_environ=http_auth)
75 assert response.json == {
76 assert json.loads(response.text) == {
76 77 u'message': u'missing objects data'}
77 78
78 79 def test_app_batch_api_unsupported_data_in_objects(
79 80 self, git_lfs_app, http_auth):
80 81 params = {'operation': 'download',
81 82 'objects': [{}]}
82 83 response = git_lfs_app.post_json(
83 84 '/repo/info/lfs/objects/batch', params=params, status=400,
84 85 extra_environ=http_auth)
85 assert response.json == {
86 assert json.loads(response.text) == {
86 87 u'message': u'unsupported data in objects'}
87 88
88 89 def test_app_batch_api_download_missing_object(
89 90 self, git_lfs_app, http_auth):
90 91 params = {'operation': 'download',
91 92 'objects': [{'oid': '123', 'size': '1024'}]}
92 93 response = git_lfs_app.post_json(
93 94 '/repo/info/lfs/objects/batch', params=params,
94 95 extra_environ=http_auth)
95 96
96 97 expected_objects = [
97 98 {u'authenticated': True,
98 99 u'errors': {u'error': {
99 100 u'code': 404,
100 101 u'message': u'object: 123 does not exist in store'}},
101 102 u'oid': u'123',
102 103 u'size': u'1024'}
103 104 ]
104 assert response.json == {
105 assert json.loads(response.text) == {
105 106 'objects': expected_objects, 'transfer': 'basic'}
106 107
107 108 def test_app_batch_api_download(self, git_lfs_app, http_auth):
108 109 oid = '456'
109 110 oid_path = os.path.join(git_lfs_app._store, oid)
110 111 if not os.path.isdir(os.path.dirname(oid_path)):
111 112 os.makedirs(os.path.dirname(oid_path))
112 113 with open(oid_path, 'wb') as f:
113 114 f.write('OID_CONTENT')
114 115
115 116 params = {'operation': 'download',
116 117 'objects': [{'oid': oid, 'size': '1024'}]}
117 118 response = git_lfs_app.post_json(
118 119 '/repo/info/lfs/objects/batch', params=params,
119 120 extra_environ=http_auth)
120 121
121 122 expected_objects = [
122 123 {u'authenticated': True,
123 124 u'actions': {
124 125 u'download': {
125 126 u'header': {u'Authorization': u'Basic XXXXX'},
126 127 u'href': u'http://localhost/repo/info/lfs/objects/456'},
127 128 },
128 129 u'oid': u'456',
129 130 u'size': u'1024'}
130 131 ]
131 assert response.json == {
132 assert json.loads(response.text) == {
132 133 'objects': expected_objects, 'transfer': 'basic'}
133 134
134 135 def test_app_batch_api_upload(self, git_lfs_app, http_auth):
135 136 params = {'operation': 'upload',
136 137 'objects': [{'oid': '123', 'size': '1024'}]}
137 138 response = git_lfs_app.post_json(
138 139 '/repo/info/lfs/objects/batch', params=params,
139 140 extra_environ=http_auth)
140 141 expected_objects = [
141 142 {u'authenticated': True,
142 143 u'actions': {
143 144 u'upload': {
144 145 u'header': {u'Authorization': u'Basic XXXXX'},
145 146 u'href': u'http://localhost/repo/info/lfs/objects/123'},
146 147 u'verify': {
147 148 u'header': {u'Authorization': u'Basic XXXXX'},
148 149 u'href': u'http://localhost/repo/info/lfs/verify'}
149 150 },
150 151 u'oid': u'123',
151 152 u'size': u'1024'}
152 153 ]
153 assert response.json == {
154 assert json.loads(response.text) == {
154 155 'objects': expected_objects, 'transfer': 'basic'}
155 156
156 157 def test_app_verify_api_missing_data(self, git_lfs_app):
157 158 params = {'oid': 'missing',}
158 159 response = git_lfs_app.post_json(
159 160 '/repo/info/lfs/verify', params=params,
160 161 status=400)
161 162
162 assert response.json == {
163 assert json.loads(response.text) == {
163 164 u'message': u'missing oid and size in request data'}
164 165
165 166 def test_app_verify_api_missing_obj(self, git_lfs_app):
166 167 params = {'oid': 'missing', 'size': '1024'}
167 168 response = git_lfs_app.post_json(
168 169 '/repo/info/lfs/verify', params=params,
169 170 status=404)
170 171
171 assert response.json == {
172 assert json.loads(response.text) == {
172 173 u'message': u'oid `missing` does not exists in store'}
173 174
174 175 def test_app_verify_api_size_mismatch(self, git_lfs_app):
175 176 oid = 'existing'
176 177 oid_path = os.path.join(git_lfs_app._store, oid)
177 178 if not os.path.isdir(os.path.dirname(oid_path)):
178 179 os.makedirs(os.path.dirname(oid_path))
179 180 with open(oid_path, 'wb') as f:
180 181 f.write('OID_CONTENT')
181 182
182 183 params = {'oid': oid, 'size': '1024'}
183 184 response = git_lfs_app.post_json(
184 185 '/repo/info/lfs/verify', params=params, status=422)
185 186
186 assert response.json == {
187 assert json.loads(response.text) == {
187 188 u'message': u'requested file size mismatch '
188 189 u'store size:11 requested:1024'}
189 190
190 191 def test_app_verify_api(self, git_lfs_app):
191 192 oid = 'existing'
192 193 oid_path = os.path.join(git_lfs_app._store, oid)
193 194 if not os.path.isdir(os.path.dirname(oid_path)):
194 195 os.makedirs(os.path.dirname(oid_path))
195 196 with open(oid_path, 'wb') as f:
196 197 f.write('OID_CONTENT')
197 198
198 199 params = {'oid': oid, 'size': 11}
199 200 response = git_lfs_app.post_json(
200 201 '/repo/info/lfs/verify', params=params)
201 202
202 assert response.json == {
203 assert json.loads(response.text) == {
203 204 u'message': {u'size': u'ok', u'in_store': u'ok'}}
204 205
205 206 def test_app_download_api_oid_not_existing(self, git_lfs_app):
206 207 oid = 'missing'
207 208
208 209 response = git_lfs_app.get(
209 210 '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404)
210 211
211 assert response.json == {
212 assert json.loads(response.text) == {
212 213 u'message': u'requested file with oid `missing` not found in store'}
213 214
214 215 def test_app_download_api(self, git_lfs_app):
215 216 oid = 'existing'
216 217 oid_path = os.path.join(git_lfs_app._store, oid)
217 218 if not os.path.isdir(os.path.dirname(oid_path)):
218 219 os.makedirs(os.path.dirname(oid_path))
219 220 with open(oid_path, 'wb') as f:
220 221 f.write('OID_CONTENT')
221 222
222 223 response = git_lfs_app.get(
223 224 '/repo/info/lfs/objects/{oid}'.format(oid=oid))
224 225 assert response
225 226
226 227 def test_app_upload(self, git_lfs_app):
227 228 oid = 'uploaded'
228 229
229 230 response = git_lfs_app.put(
230 231 '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT')
231 232
232 assert response.json == {u'upload': u'ok'}
233 assert json.loads(response.text) == {u'upload': u'ok'}
233 234
234 235 # verify that we actually wrote that OID
235 236 oid_path = os.path.join(git_lfs_app._store, oid)
236 237 assert os.path.isfile(oid_path)
237 238 assert 'CONTENT' == open(oid_path).read()
General Comments 0
You need to be logged in to leave comments. Login now