##// END OF EJS Templates
git-lfs: fixed bug #5399 git-lfs application failed to generate HTTPS urls properly.
dan -
r700:43af4e52 default
parent child
Show More
@@ -1,287 +1,292
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2019 RhodeCode GmbH
2 # Copyright (C) 2014-2019 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import re
18 import re
19 import logging
19 import logging
20 from wsgiref.util import FileWrapper
20 from wsgiref.util import FileWrapper
21
21
22 import simplejson as json
22 import simplejson as json
23 from pyramid.config import Configurator
23 from pyramid.config import Configurator
24 from pyramid.response import Response, FileIter
24 from pyramid.response import Response, FileIter
25 from pyramid.httpexceptions import (
25 from pyramid.httpexceptions import (
26 HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden,
26 HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden,
27 HTTPUnprocessableEntity)
27 HTTPUnprocessableEntity)
28
28
29 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
29 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
30 from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator
30 from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator
31 from vcsserver.utils import safe_int
31 from vcsserver.utils import safe_int
32
32
33 log = logging.getLogger(__name__)
33 log = logging.getLogger(__name__)
34
34
35
35
36 GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs' #+json ?
36 GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs' #+json ?
37 GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))')
37 GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))')
38
38
39
39
40 def write_response_error(http_exception, text=None):
40 def write_response_error(http_exception, text=None):
41 content_type = GIT_LFS_CONTENT_TYPE + '+json'
41 content_type = GIT_LFS_CONTENT_TYPE + '+json'
42 _exception = http_exception(content_type=content_type)
42 _exception = http_exception(content_type=content_type)
43 _exception.content_type = content_type
43 _exception.content_type = content_type
44 if text:
44 if text:
45 _exception.body = json.dumps({'message': text})
45 _exception.body = json.dumps({'message': text})
46 log.debug('LFS: writing response of type %s to client with text:%s',
46 log.debug('LFS: writing response of type %s to client with text:%s',
47 http_exception, text)
47 http_exception, text)
48 return _exception
48 return _exception
49
49
50
50
51 class AuthHeaderRequired(object):
51 class AuthHeaderRequired(object):
52 """
52 """
53 Decorator to check if request has proper auth-header
53 Decorator to check if request has proper auth-header
54 """
54 """
55
55
56 def __call__(self, func):
56 def __call__(self, func):
57 return get_cython_compat_decorator(self.__wrapper, func)
57 return get_cython_compat_decorator(self.__wrapper, func)
58
58
59 def __wrapper(self, func, *fargs, **fkwargs):
59 def __wrapper(self, func, *fargs, **fkwargs):
60 request = fargs[1]
60 request = fargs[1]
61 auth = request.authorization
61 auth = request.authorization
62 if not auth:
62 if not auth:
63 return write_response_error(HTTPForbidden)
63 return write_response_error(HTTPForbidden)
64 return func(*fargs[1:], **fkwargs)
64 return func(*fargs[1:], **fkwargs)
65
65
66
66
67 # views
67 # views
68
68
69 def lfs_objects(request):
69 def lfs_objects(request):
70 # indicate not supported, V1 API
70 # indicate not supported, V1 API
71 log.warning('LFS: v1 api not supported, reporting it back to client')
71 log.warning('LFS: v1 api not supported, reporting it back to client')
72 return write_response_error(HTTPNotImplemented, 'LFS: v1 api not supported')
72 return write_response_error(HTTPNotImplemented, 'LFS: v1 api not supported')
73
73
74
74
75 @AuthHeaderRequired()
75 @AuthHeaderRequired()
76 def lfs_objects_batch(request):
76 def lfs_objects_batch(request):
77 """
77 """
78 The client sends the following information to the Batch endpoint to transfer some objects:
78 The client sends the following information to the Batch endpoint to transfer some objects:
79
79
80 operation - Should be download or upload.
80 operation - Should be download or upload.
81 transfers - An optional Array of String identifiers for transfer
81 transfers - An optional Array of String identifiers for transfer
82 adapters that the client has configured. If omitted, the basic
82 adapters that the client has configured. If omitted, the basic
83 transfer adapter MUST be assumed by the server.
83 transfer adapter MUST be assumed by the server.
84 objects - An Array of objects to download.
84 objects - An Array of objects to download.
85 oid - String OID of the LFS object.
85 oid - String OID of the LFS object.
86 size - Integer byte size of the LFS object. Must be at least zero.
86 size - Integer byte size of the LFS object. Must be at least zero.
87 """
87 """
88 request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
88 request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
89 auth = request.authorization
89 auth = request.authorization
90 repo = request.matchdict.get('repo')
90 repo = request.matchdict.get('repo')
91 data = request.json
91 data = request.json
92 operation = data.get('operation')
92 operation = data.get('operation')
93 http_scheme = request.registry.git_lfs_http_scheme
94
93 if operation not in ('download', 'upload'):
95 if operation not in ('download', 'upload'):
94 log.debug('LFS: unsupported operation:%s', operation)
96 log.debug('LFS: unsupported operation:%s', operation)
95 return write_response_error(
97 return write_response_error(
96 HTTPBadRequest, 'unsupported operation mode: `%s`' % operation)
98 HTTPBadRequest, 'unsupported operation mode: `%s`' % operation)
97
99
98 if 'objects' not in data:
100 if 'objects' not in data:
99 log.debug('LFS: missing objects data')
101 log.debug('LFS: missing objects data')
100 return write_response_error(
102 return write_response_error(
101 HTTPBadRequest, 'missing objects data')
103 HTTPBadRequest, 'missing objects data')
102
104
103 log.debug('LFS: handling operation of type: %s', operation)
105 log.debug('LFS: handling operation of type: %s', operation)
104
106
105 objects = []
107 objects = []
106 for o in data['objects']:
108 for o in data['objects']:
107 try:
109 try:
108 oid = o['oid']
110 oid = o['oid']
109 obj_size = o['size']
111 obj_size = o['size']
110 except KeyError:
112 except KeyError:
111 log.exception('LFS, failed to extract data')
113 log.exception('LFS, failed to extract data')
112 return write_response_error(
114 return write_response_error(
113 HTTPBadRequest, 'unsupported data in objects')
115 HTTPBadRequest, 'unsupported data in objects')
114
116
115 obj_data = {'oid': oid}
117 obj_data = {'oid': oid}
116
118
117 obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid)
119 obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid,
118 obj_verify_href = request.route_url('lfs_objects_verify', repo=repo)
120 _scheme=http_scheme)
121 obj_verify_href = request.route_url('lfs_objects_verify', repo=repo,
122 _scheme=http_scheme)
119 store = LFSOidStore(
123 store = LFSOidStore(
120 oid, repo, store_location=request.registry.git_lfs_store_path)
124 oid, repo, store_location=request.registry.git_lfs_store_path)
121 handler = OidHandler(
125 handler = OidHandler(
122 store, repo, auth, oid, obj_size, obj_data,
126 store, repo, auth, oid, obj_size, obj_data,
123 obj_href, obj_verify_href)
127 obj_href, obj_verify_href)
124
128
125 # this verifies also OIDs
129 # this verifies also OIDs
126 actions, errors = handler.exec_operation(operation)
130 actions, errors = handler.exec_operation(operation)
127 if errors:
131 if errors:
128 log.warning('LFS: got following errors: %s', errors)
132 log.warning('LFS: got following errors: %s', errors)
129 obj_data['errors'] = errors
133 obj_data['errors'] = errors
130
134
131 if actions:
135 if actions:
132 obj_data['actions'] = actions
136 obj_data['actions'] = actions
133
137
134 obj_data['size'] = obj_size
138 obj_data['size'] = obj_size
135 obj_data['authenticated'] = True
139 obj_data['authenticated'] = True
136 objects.append(obj_data)
140 objects.append(obj_data)
137
141
138 result = {'objects': objects, 'transfer': 'basic'}
142 result = {'objects': objects, 'transfer': 'basic'}
139 log.debug('LFS Response %s', safe_result(result))
143 log.debug('LFS Response %s', safe_result(result))
140
144
141 return result
145 return result
142
146
143
147
144 def lfs_objects_oid_upload(request):
148 def lfs_objects_oid_upload(request):
145 request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
149 request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
146 repo = request.matchdict.get('repo')
150 repo = request.matchdict.get('repo')
147 oid = request.matchdict.get('oid')
151 oid = request.matchdict.get('oid')
148 store = LFSOidStore(
152 store = LFSOidStore(
149 oid, repo, store_location=request.registry.git_lfs_store_path)
153 oid, repo, store_location=request.registry.git_lfs_store_path)
150 engine = store.get_engine(mode='wb')
154 engine = store.get_engine(mode='wb')
151 log.debug('LFS: starting chunked write of LFS oid: %s to storage', oid)
155 log.debug('LFS: starting chunked write of LFS oid: %s to storage', oid)
152
156
153 body = request.environ['wsgi.input']
157 body = request.environ['wsgi.input']
154
158
155 with engine as f:
159 with engine as f:
156 blksize = 64 * 1024 # 64kb
160 blksize = 64 * 1024 # 64kb
157 while True:
161 while True:
158 # read in chunks as stream comes in from Gunicorn
162 # read in chunks as stream comes in from Gunicorn
159 # this is a specific Gunicorn support function.
163 # this is a specific Gunicorn support function.
160 # might work differently on waitress
164 # might work differently on waitress
161 chunk = body.read(blksize)
165 chunk = body.read(blksize)
162 if not chunk:
166 if not chunk:
163 break
167 break
164 f.write(chunk)
168 f.write(chunk)
165
169
166 return {'upload': 'ok'}
170 return {'upload': 'ok'}
167
171
168
172
169 def lfs_objects_oid_download(request):
173 def lfs_objects_oid_download(request):
170 repo = request.matchdict.get('repo')
174 repo = request.matchdict.get('repo')
171 oid = request.matchdict.get('oid')
175 oid = request.matchdict.get('oid')
172
176
173 store = LFSOidStore(
177 store = LFSOidStore(
174 oid, repo, store_location=request.registry.git_lfs_store_path)
178 oid, repo, store_location=request.registry.git_lfs_store_path)
175 if not store.has_oid():
179 if not store.has_oid():
176 log.debug('LFS: oid %s does not exists in store', oid)
180 log.debug('LFS: oid %s does not exists in store', oid)
177 return write_response_error(
181 return write_response_error(
178 HTTPNotFound, 'requested file with oid `%s` not found in store' % oid)
182 HTTPNotFound, 'requested file with oid `%s` not found in store' % oid)
179
183
180 # TODO(marcink): support range header ?
184 # TODO(marcink): support range header ?
181 # Range: bytes=0-, `bytes=(\d+)\-.*`
185 # Range: bytes=0-, `bytes=(\d+)\-.*`
182
186
183 f = open(store.oid_path, 'rb')
187 f = open(store.oid_path, 'rb')
184 response = Response(
188 response = Response(
185 content_type='application/octet-stream', app_iter=FileIter(f))
189 content_type='application/octet-stream', app_iter=FileIter(f))
186 response.headers.add('X-RC-LFS-Response-Oid', str(oid))
190 response.headers.add('X-RC-LFS-Response-Oid', str(oid))
187 return response
191 return response
188
192
189
193
190 def lfs_objects_verify(request):
194 def lfs_objects_verify(request):
191 request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
195 request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
192 repo = request.matchdict.get('repo')
196 repo = request.matchdict.get('repo')
193
197
194 data = request.json
198 data = request.json
195 oid = data.get('oid')
199 oid = data.get('oid')
196 size = safe_int(data.get('size'))
200 size = safe_int(data.get('size'))
197
201
198 if not (oid and size):
202 if not (oid and size):
199 return write_response_error(
203 return write_response_error(
200 HTTPBadRequest, 'missing oid and size in request data')
204 HTTPBadRequest, 'missing oid and size in request data')
201
205
202 store = LFSOidStore(
206 store = LFSOidStore(
203 oid, repo, store_location=request.registry.git_lfs_store_path)
207 oid, repo, store_location=request.registry.git_lfs_store_path)
204 if not store.has_oid():
208 if not store.has_oid():
205 log.debug('LFS: oid %s does not exists in store', oid)
209 log.debug('LFS: oid %s does not exists in store', oid)
206 return write_response_error(
210 return write_response_error(
207 HTTPNotFound, 'oid `%s` does not exists in store' % oid)
211 HTTPNotFound, 'oid `%s` does not exists in store' % oid)
208
212
209 store_size = store.size_oid()
213 store_size = store.size_oid()
210 if store_size != size:
214 if store_size != size:
211 msg = 'requested file size mismatch store size:%s requested:%s' % (
215 msg = 'requested file size mismatch store size:%s requested:%s' % (
212 store_size, size)
216 store_size, size)
213 return write_response_error(
217 return write_response_error(
214 HTTPUnprocessableEntity, msg)
218 HTTPUnprocessableEntity, msg)
215
219
216 return {'message': {'size': 'ok', 'in_store': 'ok'}}
220 return {'message': {'size': 'ok', 'in_store': 'ok'}}
217
221
218
222
219 def lfs_objects_lock(request):
223 def lfs_objects_lock(request):
220 return write_response_error(
224 return write_response_error(
221 HTTPNotImplemented, 'GIT LFS locking api not supported')
225 HTTPNotImplemented, 'GIT LFS locking api not supported')
222
226
223
227
224 def not_found(request):
228 def not_found(request):
225 return write_response_error(
229 return write_response_error(
226 HTTPNotFound, 'request path not found')
230 HTTPNotFound, 'request path not found')
227
231
228
232
229 def lfs_disabled(request):
233 def lfs_disabled(request):
230 return write_response_error(
234 return write_response_error(
231 HTTPNotImplemented, 'GIT LFS disabled for this repo')
235 HTTPNotImplemented, 'GIT LFS disabled for this repo')
232
236
233
237
234 def git_lfs_app(config):
238 def git_lfs_app(config):
235
239
236 # v1 API deprecation endpoint
240 # v1 API deprecation endpoint
237 config.add_route('lfs_objects',
241 config.add_route('lfs_objects',
238 '/{repo:.*?[^/]}/info/lfs/objects')
242 '/{repo:.*?[^/]}/info/lfs/objects')
239 config.add_view(lfs_objects, route_name='lfs_objects',
243 config.add_view(lfs_objects, route_name='lfs_objects',
240 request_method='POST', renderer='json')
244 request_method='POST', renderer='json')
241
245
242 # locking API
246 # locking API
243 config.add_route('lfs_objects_lock',
247 config.add_route('lfs_objects_lock',
244 '/{repo:.*?[^/]}/info/lfs/locks')
248 '/{repo:.*?[^/]}/info/lfs/locks')
245 config.add_view(lfs_objects_lock, route_name='lfs_objects_lock',
249 config.add_view(lfs_objects_lock, route_name='lfs_objects_lock',
246 request_method=('POST', 'GET'), renderer='json')
250 request_method=('POST', 'GET'), renderer='json')
247
251
248 config.add_route('lfs_objects_lock_verify',
252 config.add_route('lfs_objects_lock_verify',
249 '/{repo:.*?[^/]}/info/lfs/locks/verify')
253 '/{repo:.*?[^/]}/info/lfs/locks/verify')
250 config.add_view(lfs_objects_lock, route_name='lfs_objects_lock_verify',
254 config.add_view(lfs_objects_lock, route_name='lfs_objects_lock_verify',
251 request_method=('POST', 'GET'), renderer='json')
255 request_method=('POST', 'GET'), renderer='json')
252
256
253 # batch API
257 # batch API
254 config.add_route('lfs_objects_batch',
258 config.add_route('lfs_objects_batch',
255 '/{repo:.*?[^/]}/info/lfs/objects/batch')
259 '/{repo:.*?[^/]}/info/lfs/objects/batch')
256 config.add_view(lfs_objects_batch, route_name='lfs_objects_batch',
260 config.add_view(lfs_objects_batch, route_name='lfs_objects_batch',
257 request_method='POST', renderer='json')
261 request_method='POST', renderer='json')
258
262
259 # oid upload/download API
263 # oid upload/download API
260 config.add_route('lfs_objects_oid',
264 config.add_route('lfs_objects_oid',
261 '/{repo:.*?[^/]}/info/lfs/objects/{oid}')
265 '/{repo:.*?[^/]}/info/lfs/objects/{oid}')
262 config.add_view(lfs_objects_oid_upload, route_name='lfs_objects_oid',
266 config.add_view(lfs_objects_oid_upload, route_name='lfs_objects_oid',
263 request_method='PUT', renderer='json')
267 request_method='PUT', renderer='json')
264 config.add_view(lfs_objects_oid_download, route_name='lfs_objects_oid',
268 config.add_view(lfs_objects_oid_download, route_name='lfs_objects_oid',
265 request_method='GET', renderer='json')
269 request_method='GET', renderer='json')
266
270
267 # verification API
271 # verification API
268 config.add_route('lfs_objects_verify',
272 config.add_route('lfs_objects_verify',
269 '/{repo:.*?[^/]}/info/lfs/verify')
273 '/{repo:.*?[^/]}/info/lfs/verify')
270 config.add_view(lfs_objects_verify, route_name='lfs_objects_verify',
274 config.add_view(lfs_objects_verify, route_name='lfs_objects_verify',
271 request_method='POST', renderer='json')
275 request_method='POST', renderer='json')
272
276
273 # not found handler for API
277 # not found handler for API
274 config.add_notfound_view(not_found, renderer='json')
278 config.add_notfound_view(not_found, renderer='json')
275
279
276
280
277 def create_app(git_lfs_enabled, git_lfs_store_path):
281 def create_app(git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme):
278 config = Configurator()
282 config = Configurator()
279 if git_lfs_enabled:
283 if git_lfs_enabled:
280 config.include(git_lfs_app)
284 config.include(git_lfs_app)
281 config.registry.git_lfs_store_path = git_lfs_store_path
285 config.registry.git_lfs_store_path = git_lfs_store_path
286 config.registry.git_lfs_http_scheme = git_lfs_http_scheme
282 else:
287 else:
283 # not found handler for API, reporting disabled LFS support
288 # not found handler for API, reporting disabled LFS support
284 config.add_notfound_view(lfs_disabled, renderer='json')
289 config.add_notfound_view(lfs_disabled, renderer='json')
285
290
286 app = config.make_wsgi_app()
291 app = config.make_wsgi_app()
287 return app
292 return app
@@ -1,239 +1,272
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2019 RhodeCode GmbH
2 # Copyright (C) 2014-2019 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import os
19 import pytest
19 import pytest
20 from webtest.app import TestApp as WebObTestApp
20 from webtest.app import TestApp as WebObTestApp
21 import simplejson as json
21 import simplejson as json
22
22
23 from vcsserver.git_lfs.app import create_app
23 from vcsserver.git_lfs.app import create_app
24
24
25
25
26 @pytest.fixture(scope='function')
26 @pytest.fixture(scope='function')
27 def git_lfs_app(tmpdir):
27 def git_lfs_app(tmpdir):
28 custom_app = WebObTestApp(create_app(
28 custom_app = WebObTestApp(create_app(
29 git_lfs_enabled=True, git_lfs_store_path=str(tmpdir)))
29 git_lfs_enabled=True, git_lfs_store_path=str(tmpdir),
30 git_lfs_http_scheme='http'))
31 custom_app._store = str(tmpdir)
32 return custom_app
33
34
35 @pytest.fixture(scope='function')
36 def git_lfs_https_app(tmpdir):
37 custom_app = WebObTestApp(create_app(
38 git_lfs_enabled=True, git_lfs_store_path=str(tmpdir),
39 git_lfs_http_scheme='https'))
30 custom_app._store = str(tmpdir)
40 custom_app._store = str(tmpdir)
31 return custom_app
41 return custom_app
32
42
33
43
34 @pytest.fixture()
44 @pytest.fixture()
35 def http_auth():
45 def http_auth():
36 return {'HTTP_AUTHORIZATION': "Basic XXXXX"}
46 return {'HTTP_AUTHORIZATION': "Basic XXXXX"}
37
47
38
48
39 class TestLFSApplication(object):
49 class TestLFSApplication(object):
40
50
41 def test_app_wrong_path(self, git_lfs_app):
51 def test_app_wrong_path(self, git_lfs_app):
42 git_lfs_app.get('/repo/info/lfs/xxx', status=404)
52 git_lfs_app.get('/repo/info/lfs/xxx', status=404)
43
53
44 def test_app_deprecated_endpoint(self, git_lfs_app):
54 def test_app_deprecated_endpoint(self, git_lfs_app):
45 response = git_lfs_app.post('/repo/info/lfs/objects', status=501)
55 response = git_lfs_app.post('/repo/info/lfs/objects', status=501)
46 assert response.status_code == 501
56 assert response.status_code == 501
47 assert json.loads(response.text) == {u'message': u'LFS: v1 api not supported'}
57 assert json.loads(response.text) == {u'message': u'LFS: v1 api not supported'}
48
58
49 def test_app_lock_verify_api_not_available(self, git_lfs_app):
59 def test_app_lock_verify_api_not_available(self, git_lfs_app):
50 response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501)
60 response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501)
51 assert response.status_code == 501
61 assert response.status_code == 501
52 assert json.loads(response.text) == {
62 assert json.loads(response.text) == {
53 u'message': u'GIT LFS locking api not supported'}
63 u'message': u'GIT LFS locking api not supported'}
54
64
55 def test_app_lock_api_not_available(self, git_lfs_app):
65 def test_app_lock_api_not_available(self, git_lfs_app):
56 response = git_lfs_app.post('/repo/info/lfs/locks', status=501)
66 response = git_lfs_app.post('/repo/info/lfs/locks', status=501)
57 assert response.status_code == 501
67 assert response.status_code == 501
58 assert json.loads(response.text) == {
68 assert json.loads(response.text) == {
59 u'message': u'GIT LFS locking api not supported'}
69 u'message': u'GIT LFS locking api not supported'}
60
70
61 def test_app_batch_api_missing_auth(self, git_lfs_app,):
71 def test_app_batch_api_missing_auth(self, git_lfs_app):
62 git_lfs_app.post_json(
72 git_lfs_app.post_json(
63 '/repo/info/lfs/objects/batch', params={}, status=403)
73 '/repo/info/lfs/objects/batch', params={}, status=403)
64
74
65 def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth):
75 def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth):
66 response = git_lfs_app.post_json(
76 response = git_lfs_app.post_json(
67 '/repo/info/lfs/objects/batch', params={}, status=400,
77 '/repo/info/lfs/objects/batch', params={}, status=400,
68 extra_environ=http_auth)
78 extra_environ=http_auth)
69 assert json.loads(response.text) == {
79 assert json.loads(response.text) == {
70 u'message': u'unsupported operation mode: `None`'}
80 u'message': u'unsupported operation mode: `None`'}
71
81
72 def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth):
82 def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth):
73 response = git_lfs_app.post_json(
83 response = git_lfs_app.post_json(
74 '/repo/info/lfs/objects/batch', params={'operation': 'download'},
84 '/repo/info/lfs/objects/batch', params={'operation': 'download'},
75 status=400, extra_environ=http_auth)
85 status=400, extra_environ=http_auth)
76 assert json.loads(response.text) == {
86 assert json.loads(response.text) == {
77 u'message': u'missing objects data'}
87 u'message': u'missing objects data'}
78
88
79 def test_app_batch_api_unsupported_data_in_objects(
89 def test_app_batch_api_unsupported_data_in_objects(
80 self, git_lfs_app, http_auth):
90 self, git_lfs_app, http_auth):
81 params = {'operation': 'download',
91 params = {'operation': 'download',
82 'objects': [{}]}
92 'objects': [{}]}
83 response = git_lfs_app.post_json(
93 response = git_lfs_app.post_json(
84 '/repo/info/lfs/objects/batch', params=params, status=400,
94 '/repo/info/lfs/objects/batch', params=params, status=400,
85 extra_environ=http_auth)
95 extra_environ=http_auth)
86 assert json.loads(response.text) == {
96 assert json.loads(response.text) == {
87 u'message': u'unsupported data in objects'}
97 u'message': u'unsupported data in objects'}
88
98
89 def test_app_batch_api_download_missing_object(
99 def test_app_batch_api_download_missing_object(
90 self, git_lfs_app, http_auth):
100 self, git_lfs_app, http_auth):
91 params = {'operation': 'download',
101 params = {'operation': 'download',
92 'objects': [{'oid': '123', 'size': '1024'}]}
102 'objects': [{'oid': '123', 'size': '1024'}]}
93 response = git_lfs_app.post_json(
103 response = git_lfs_app.post_json(
94 '/repo/info/lfs/objects/batch', params=params,
104 '/repo/info/lfs/objects/batch', params=params,
95 extra_environ=http_auth)
105 extra_environ=http_auth)
96
106
97 expected_objects = [
107 expected_objects = [
98 {u'authenticated': True,
108 {u'authenticated': True,
99 u'errors': {u'error': {
109 u'errors': {u'error': {
100 u'code': 404,
110 u'code': 404,
101 u'message': u'object: 123 does not exist in store'}},
111 u'message': u'object: 123 does not exist in store'}},
102 u'oid': u'123',
112 u'oid': u'123',
103 u'size': u'1024'}
113 u'size': u'1024'}
104 ]
114 ]
105 assert json.loads(response.text) == {
115 assert json.loads(response.text) == {
106 'objects': expected_objects, 'transfer': 'basic'}
116 'objects': expected_objects, 'transfer': 'basic'}
107
117
108 def test_app_batch_api_download(self, git_lfs_app, http_auth):
118 def test_app_batch_api_download(self, git_lfs_app, http_auth):
109 oid = '456'
119 oid = '456'
110 oid_path = os.path.join(git_lfs_app._store, oid)
120 oid_path = os.path.join(git_lfs_app._store, oid)
111 if not os.path.isdir(os.path.dirname(oid_path)):
121 if not os.path.isdir(os.path.dirname(oid_path)):
112 os.makedirs(os.path.dirname(oid_path))
122 os.makedirs(os.path.dirname(oid_path))
113 with open(oid_path, 'wb') as f:
123 with open(oid_path, 'wb') as f:
114 f.write('OID_CONTENT')
124 f.write('OID_CONTENT')
115
125
116 params = {'operation': 'download',
126 params = {'operation': 'download',
117 'objects': [{'oid': oid, 'size': '1024'}]}
127 'objects': [{'oid': oid, 'size': '1024'}]}
118 response = git_lfs_app.post_json(
128 response = git_lfs_app.post_json(
119 '/repo/info/lfs/objects/batch', params=params,
129 '/repo/info/lfs/objects/batch', params=params,
120 extra_environ=http_auth)
130 extra_environ=http_auth)
121
131
122 expected_objects = [
132 expected_objects = [
123 {u'authenticated': True,
133 {u'authenticated': True,
124 u'actions': {
134 u'actions': {
125 u'download': {
135 u'download': {
126 u'header': {u'Authorization': u'Basic XXXXX'},
136 u'header': {u'Authorization': u'Basic XXXXX'},
127 u'href': u'http://localhost/repo/info/lfs/objects/456'},
137 u'href': u'http://localhost/repo/info/lfs/objects/456'},
128 },
138 },
129 u'oid': u'456',
139 u'oid': u'456',
130 u'size': u'1024'}
140 u'size': u'1024'}
131 ]
141 ]
132 assert json.loads(response.text) == {
142 assert json.loads(response.text) == {
133 'objects': expected_objects, 'transfer': 'basic'}
143 'objects': expected_objects, 'transfer': 'basic'}
134
144
135 def test_app_batch_api_upload(self, git_lfs_app, http_auth):
145 def test_app_batch_api_upload(self, git_lfs_app, http_auth):
136 params = {'operation': 'upload',
146 params = {'operation': 'upload',
137 'objects': [{'oid': '123', 'size': '1024'}]}
147 'objects': [{'oid': '123', 'size': '1024'}]}
138 response = git_lfs_app.post_json(
148 response = git_lfs_app.post_json(
139 '/repo/info/lfs/objects/batch', params=params,
149 '/repo/info/lfs/objects/batch', params=params,
140 extra_environ=http_auth)
150 extra_environ=http_auth)
141 expected_objects = [
151 expected_objects = [
142 {u'authenticated': True,
152 {u'authenticated': True,
143 u'actions': {
153 u'actions': {
144 u'upload': {
154 u'upload': {
145 u'header': {u'Authorization': u'Basic XXXXX',
155 u'header': {u'Authorization': u'Basic XXXXX',
146 u'Transfer-Encoding': u'chunked'},
156 u'Transfer-Encoding': u'chunked'},
147 u'href': u'http://localhost/repo/info/lfs/objects/123'},
157 u'href': u'http://localhost/repo/info/lfs/objects/123'},
148 u'verify': {
158 u'verify': {
149 u'header': {u'Authorization': u'Basic XXXXX'},
159 u'header': {u'Authorization': u'Basic XXXXX'},
150 u'href': u'http://localhost/repo/info/lfs/verify'}
160 u'href': u'http://localhost/repo/info/lfs/verify'}
151 },
161 },
152 u'oid': u'123',
162 u'oid': u'123',
153 u'size': u'1024'}
163 u'size': u'1024'}
154 ]
164 ]
155 assert json.loads(response.text) == {
165 assert json.loads(response.text) == {
156 'objects': expected_objects, 'transfer': 'basic'}
166 'objects': expected_objects, 'transfer': 'basic'}
157
167
168 def test_app_batch_api_upload_for_https(self, git_lfs_https_app, http_auth):
169 params = {'operation': 'upload',
170 'objects': [{'oid': '123', 'size': '1024'}]}
171 response = git_lfs_https_app.post_json(
172 '/repo/info/lfs/objects/batch', params=params,
173 extra_environ=http_auth)
174 expected_objects = [
175 {u'authenticated': True,
176 u'actions': {
177 u'upload': {
178 u'header': {u'Authorization': u'Basic XXXXX',
179 u'Transfer-Encoding': u'chunked'},
180 u'href': u'https://localhost/repo/info/lfs/objects/123'},
181 u'verify': {
182 u'header': {u'Authorization': u'Basic XXXXX'},
183 u'href': u'https://localhost/repo/info/lfs/verify'}
184 },
185 u'oid': u'123',
186 u'size': u'1024'}
187 ]
188 assert json.loads(response.text) == {
189 'objects': expected_objects, 'transfer': 'basic'}
190
158 def test_app_verify_api_missing_data(self, git_lfs_app):
191 def test_app_verify_api_missing_data(self, git_lfs_app):
159 params = {'oid': 'missing',}
192 params = {'oid': 'missing'}
160 response = git_lfs_app.post_json(
193 response = git_lfs_app.post_json(
161 '/repo/info/lfs/verify', params=params,
194 '/repo/info/lfs/verify', params=params,
162 status=400)
195 status=400)
163
196
164 assert json.loads(response.text) == {
197 assert json.loads(response.text) == {
165 u'message': u'missing oid and size in request data'}
198 u'message': u'missing oid and size in request data'}
166
199
167 def test_app_verify_api_missing_obj(self, git_lfs_app):
200 def test_app_verify_api_missing_obj(self, git_lfs_app):
168 params = {'oid': 'missing', 'size': '1024'}
201 params = {'oid': 'missing', 'size': '1024'}
169 response = git_lfs_app.post_json(
202 response = git_lfs_app.post_json(
170 '/repo/info/lfs/verify', params=params,
203 '/repo/info/lfs/verify', params=params,
171 status=404)
204 status=404)
172
205
173 assert json.loads(response.text) == {
206 assert json.loads(response.text) == {
174 u'message': u'oid `missing` does not exists in store'}
207 u'message': u'oid `missing` does not exists in store'}
175
208
176 def test_app_verify_api_size_mismatch(self, git_lfs_app):
209 def test_app_verify_api_size_mismatch(self, git_lfs_app):
177 oid = 'existing'
210 oid = 'existing'
178 oid_path = os.path.join(git_lfs_app._store, oid)
211 oid_path = os.path.join(git_lfs_app._store, oid)
179 if not os.path.isdir(os.path.dirname(oid_path)):
212 if not os.path.isdir(os.path.dirname(oid_path)):
180 os.makedirs(os.path.dirname(oid_path))
213 os.makedirs(os.path.dirname(oid_path))
181 with open(oid_path, 'wb') as f:
214 with open(oid_path, 'wb') as f:
182 f.write('OID_CONTENT')
215 f.write('OID_CONTENT')
183
216
184 params = {'oid': oid, 'size': '1024'}
217 params = {'oid': oid, 'size': '1024'}
185 response = git_lfs_app.post_json(
218 response = git_lfs_app.post_json(
186 '/repo/info/lfs/verify', params=params, status=422)
219 '/repo/info/lfs/verify', params=params, status=422)
187
220
188 assert json.loads(response.text) == {
221 assert json.loads(response.text) == {
189 u'message': u'requested file size mismatch '
222 u'message': u'requested file size mismatch '
190 u'store size:11 requested:1024'}
223 u'store size:11 requested:1024'}
191
224
192 def test_app_verify_api(self, git_lfs_app):
225 def test_app_verify_api(self, git_lfs_app):
193 oid = 'existing'
226 oid = 'existing'
194 oid_path = os.path.join(git_lfs_app._store, oid)
227 oid_path = os.path.join(git_lfs_app._store, oid)
195 if not os.path.isdir(os.path.dirname(oid_path)):
228 if not os.path.isdir(os.path.dirname(oid_path)):
196 os.makedirs(os.path.dirname(oid_path))
229 os.makedirs(os.path.dirname(oid_path))
197 with open(oid_path, 'wb') as f:
230 with open(oid_path, 'wb') as f:
198 f.write('OID_CONTENT')
231 f.write('OID_CONTENT')
199
232
200 params = {'oid': oid, 'size': 11}
233 params = {'oid': oid, 'size': 11}
201 response = git_lfs_app.post_json(
234 response = git_lfs_app.post_json(
202 '/repo/info/lfs/verify', params=params)
235 '/repo/info/lfs/verify', params=params)
203
236
204 assert json.loads(response.text) == {
237 assert json.loads(response.text) == {
205 u'message': {u'size': u'ok', u'in_store': u'ok'}}
238 u'message': {u'size': u'ok', u'in_store': u'ok'}}
206
239
207 def test_app_download_api_oid_not_existing(self, git_lfs_app):
240 def test_app_download_api_oid_not_existing(self, git_lfs_app):
208 oid = 'missing'
241 oid = 'missing'
209
242
210 response = git_lfs_app.get(
243 response = git_lfs_app.get(
211 '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404)
244 '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404)
212
245
213 assert json.loads(response.text) == {
246 assert json.loads(response.text) == {
214 u'message': u'requested file with oid `missing` not found in store'}
247 u'message': u'requested file with oid `missing` not found in store'}
215
248
216 def test_app_download_api(self, git_lfs_app):
249 def test_app_download_api(self, git_lfs_app):
217 oid = 'existing'
250 oid = 'existing'
218 oid_path = os.path.join(git_lfs_app._store, oid)
251 oid_path = os.path.join(git_lfs_app._store, oid)
219 if not os.path.isdir(os.path.dirname(oid_path)):
252 if not os.path.isdir(os.path.dirname(oid_path)):
220 os.makedirs(os.path.dirname(oid_path))
253 os.makedirs(os.path.dirname(oid_path))
221 with open(oid_path, 'wb') as f:
254 with open(oid_path, 'wb') as f:
222 f.write('OID_CONTENT')
255 f.write('OID_CONTENT')
223
256
224 response = git_lfs_app.get(
257 response = git_lfs_app.get(
225 '/repo/info/lfs/objects/{oid}'.format(oid=oid))
258 '/repo/info/lfs/objects/{oid}'.format(oid=oid))
226 assert response
259 assert response
227
260
228 def test_app_upload(self, git_lfs_app):
261 def test_app_upload(self, git_lfs_app):
229 oid = 'uploaded'
262 oid = 'uploaded'
230
263
231 response = git_lfs_app.put(
264 response = git_lfs_app.put(
232 '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT')
265 '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT')
233
266
234 assert json.loads(response.text) == {u'upload': u'ok'}
267 assert json.loads(response.text) == {u'upload': u'ok'}
235
268
236 # verify that we actually wrote that OID
269 # verify that we actually wrote that OID
237 oid_path = os.path.join(git_lfs_app._store, oid)
270 oid_path = os.path.join(git_lfs_app._store, oid)
238 assert os.path.isfile(oid_path)
271 assert os.path.isfile(oid_path)
239 assert 'CONTENT' == open(oid_path).read()
272 assert 'CONTENT' == open(oid_path).read()
@@ -1,234 +1,235
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2019 RhodeCode GmbH
2 # Copyright (C) 2014-2019 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import os
19 import logging
19 import logging
20 import itertools
20 import itertools
21
21
22 import mercurial
22 import mercurial
23 import mercurial.error
23 import mercurial.error
24 import mercurial.wireprotoserver
24 import mercurial.wireprotoserver
25 import mercurial.hgweb.common
25 import mercurial.hgweb.common
26 import mercurial.hgweb.hgweb_mod
26 import mercurial.hgweb.hgweb_mod
27 import webob.exc
27 import webob.exc
28
28