##// END OF EJS Templates
git-lfs: report chunked encoding support properly to the client....
marcink -
r199:39019410 default
parent child Browse files
Show More
@@ -1,171 +1,172 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import os
19 19 import shutil
20 20 import logging
21 21 from collections import OrderedDict
22 22
23 23 log = logging.getLogger(__name__)
24 24
25 25
26 26 class OidHandler(object):
27 27
28 28 def __init__(self, store, repo_name, auth, oid, obj_size, obj_data, obj_href,
29 29 obj_verify_href=None):
30 30 self.current_store = store
31 31 self.repo_name = repo_name
32 32 self.auth = auth
33 33 self.oid = oid
34 34 self.obj_size = obj_size
35 35 self.obj_data = obj_data
36 36 self.obj_href = obj_href
37 37 self.obj_verify_href = obj_verify_href
38 38
39 39 def get_store(self, mode=None):
40 40 return self.current_store
41 41
42 42 def get_auth(self):
43 43 """returns auth header for re-use in upload/download"""
44 44 return " ".join(self.auth)
45 45
46 46 def download(self):
47 47
48 48 store = self.get_store()
49 49 response = None
50 50 has_errors = None
51 51
52 52 if not store.has_oid():
53 53 # error reply back to client that something is wrong with dl
54 54 err_msg = 'object: {} does not exist in store'.format(store.oid)
55 55 has_errors = OrderedDict(
56 56 error=OrderedDict(
57 57 code=404,
58 58 message=err_msg
59 59 )
60 60 )
61 61
62 62 download_action = OrderedDict(
63 63 href=self.obj_href,
64 64 header=OrderedDict([("Authorization", self.get_auth())])
65 65 )
66 66 if not has_errors:
67 67 response = OrderedDict(download=download_action)
68 68 return response, has_errors
69 69
70 70 def upload(self, skip_existing=True):
71 71 """
72 72 Write upload action for git-lfs server
73 73 """
74 74
75 75 store = self.get_store()
76 76 response = None
77 77 has_errors = None
78 78
79 79 # verify if we have the OID before, if we do, reply with empty
80 80 if store.has_oid():
81 81 log.debug('LFS: store already has oid %s', store.oid)
82 82
83 83 # validate size
84 84 size_match = store.size_oid() == self.obj_size
85 85 if not size_match:
86 86 log.warning('LFS: size mismatch for oid:%s', self.oid)
87 87 elif skip_existing:
88 88 log.debug('LFS: skipping further action as oid is existing')
89 89 return response, has_errors
90 90
91 chunked = ("Transfer-Encoding", "chunked")
91 92 upload_action = OrderedDict(
92 93 href=self.obj_href,
93 header=OrderedDict([("Authorization", self.get_auth())])
94 header=OrderedDict([("Authorization", self.get_auth()), chunked])
94 95 )
95 96 if not has_errors:
96 97 response = OrderedDict(upload=upload_action)
97 98 # if specified in handler, return the verification endpoint
98 99 if self.obj_verify_href:
99 100 verify_action = OrderedDict(
100 101 href=self.obj_verify_href,
101 102 header=OrderedDict([("Authorization", self.get_auth())])
102 103 )
103 104 response['verify'] = verify_action
104 105 return response, has_errors
105 106
106 107 def exec_operation(self, operation, *args, **kwargs):
107 108 handler = getattr(self, operation)
108 109 log.debug('LFS: handling request using %s handler', handler)
109 110 return handler(*args, **kwargs)
110 111
111 112
112 113 class LFSOidStore(object):
113 114
114 115 def __init__(self, oid, repo, store_location=None):
115 116 self.oid = oid
116 117 self.repo = repo
117 118 self.store_path = store_location or self.get_default_store()
118 119 self.tmp_oid_path = os.path.join(self.store_path, oid + '.tmp')
119 120 self.oid_path = os.path.join(self.store_path, oid)
120 121 self.fd = None
121 122
122 123 def get_engine(self, mode):
123 124 """
124 125 engine = .get_engine(mode='wb')
125 126 with engine as f:
126 127 f.write('...')
127 128 """
128 129
129 130 class StoreEngine(object):
130 131 def __init__(self, mode, store_path, oid_path, tmp_oid_path):
131 132 self.mode = mode
132 133 self.store_path = store_path
133 134 self.oid_path = oid_path
134 135 self.tmp_oid_path = tmp_oid_path
135 136
136 137 def __enter__(self):
137 138 if not os.path.isdir(self.store_path):
138 139 os.makedirs(self.store_path)
139 140
140 141 # TODO(marcink): maybe write metadata here with size/oid ?
141 142 fd = open(self.tmp_oid_path, self.mode)
142 143 self.fd = fd
143 144 return fd
144 145
145 146 def __exit__(self, exc_type, exc_value, traceback):
146 147 # close tmp file, and rename to final destination
147 148 self.fd.close()
148 149 shutil.move(self.tmp_oid_path, self.oid_path)
149 150
150 151 return StoreEngine(
151 152 mode, self.store_path, self.oid_path, self.tmp_oid_path)
152 153
153 154 def get_default_store(self):
154 155 """
155 156 Default store, consistent with defaults of Mercurial large files store
156 157 which is /home/username/.cache/largefiles
157 158 """
158 159 user_home = os.path.expanduser("~")
159 160 return os.path.join(user_home, '.cache', 'lfs-store')
160 161
161 162 def has_oid(self):
162 163 return os.path.exists(os.path.join(self.store_path, self.oid))
163 164
164 165 def size_oid(self):
165 166 size = -1
166 167
167 168 if self.has_oid():
168 169 oid = os.path.join(self.store_path, self.oid)
169 170 size = os.stat(oid).st_size
170 171
171 172 return size
@@ -1,238 +1,239 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import os
19 19 import pytest
20 20 from webtest.app import TestApp as WebObTestApp
21 21 import simplejson as json
22 22
23 23 from vcsserver.git_lfs.app import create_app
24 24
25 25
26 26 @pytest.fixture(scope='function')
27 27 def git_lfs_app(tmpdir):
28 28 custom_app = WebObTestApp(create_app(
29 29 git_lfs_enabled=True, git_lfs_store_path=str(tmpdir)))
30 30 custom_app._store = str(tmpdir)
31 31 return custom_app
32 32
33 33
34 34 @pytest.fixture()
35 35 def http_auth():
36 36 return {'HTTP_AUTHORIZATION': "Basic XXXXX"}
37 37
38 38
39 39 class TestLFSApplication(object):
40 40
41 41 def test_app_wrong_path(self, git_lfs_app):
42 42 git_lfs_app.get('/repo/info/lfs/xxx', status=404)
43 43
44 44 def test_app_deprecated_endpoint(self, git_lfs_app):
45 45 response = git_lfs_app.post('/repo/info/lfs/objects', status=501)
46 46 assert response.status_code == 501
47 47 assert json.loads(response.text) == {u'message': u'LFS: v1 api not supported'}
48 48
49 49 def test_app_lock_verify_api_not_available(self, git_lfs_app):
50 50 response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501)
51 51 assert response.status_code == 501
52 52 assert json.loads(response.text) == {
53 53 u'message': u'GIT LFS locking api not supported'}
54 54
55 55 def test_app_lock_api_not_available(self, git_lfs_app):
56 56 response = git_lfs_app.post('/repo/info/lfs/locks', status=501)
57 57 assert response.status_code == 501
58 58 assert json.loads(response.text) == {
59 59 u'message': u'GIT LFS locking api not supported'}
60 60
61 61 def test_app_batch_api_missing_auth(self, git_lfs_app,):
62 62 git_lfs_app.post_json(
63 63 '/repo/info/lfs/objects/batch', params={}, status=403)
64 64
65 65 def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth):
66 66 response = git_lfs_app.post_json(
67 67 '/repo/info/lfs/objects/batch', params={}, status=400,
68 68 extra_environ=http_auth)
69 69 assert json.loads(response.text) == {
70 70 u'message': u'unsupported operation mode: `None`'}
71 71
72 72 def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth):
73 73 response = git_lfs_app.post_json(
74 74 '/repo/info/lfs/objects/batch', params={'operation': 'download'},
75 75 status=400, extra_environ=http_auth)
76 76 assert json.loads(response.text) == {
77 77 u'message': u'missing objects data'}
78 78
79 79 def test_app_batch_api_unsupported_data_in_objects(
80 80 self, git_lfs_app, http_auth):
81 81 params = {'operation': 'download',
82 82 'objects': [{}]}
83 83 response = git_lfs_app.post_json(
84 84 '/repo/info/lfs/objects/batch', params=params, status=400,
85 85 extra_environ=http_auth)
86 86 assert json.loads(response.text) == {
87 87 u'message': u'unsupported data in objects'}
88 88
89 89 def test_app_batch_api_download_missing_object(
90 90 self, git_lfs_app, http_auth):
91 91 params = {'operation': 'download',
92 92 'objects': [{'oid': '123', 'size': '1024'}]}
93 93 response = git_lfs_app.post_json(
94 94 '/repo/info/lfs/objects/batch', params=params,
95 95 extra_environ=http_auth)
96 96
97 97 expected_objects = [
98 98 {u'authenticated': True,
99 99 u'errors': {u'error': {
100 100 u'code': 404,
101 101 u'message': u'object: 123 does not exist in store'}},
102 102 u'oid': u'123',
103 103 u'size': u'1024'}
104 104 ]
105 105 assert json.loads(response.text) == {
106 106 'objects': expected_objects, 'transfer': 'basic'}
107 107
108 108 def test_app_batch_api_download(self, git_lfs_app, http_auth):
109 109 oid = '456'
110 110 oid_path = os.path.join(git_lfs_app._store, oid)
111 111 if not os.path.isdir(os.path.dirname(oid_path)):
112 112 os.makedirs(os.path.dirname(oid_path))
113 113 with open(oid_path, 'wb') as f:
114 114 f.write('OID_CONTENT')
115 115
116 116 params = {'operation': 'download',
117 117 'objects': [{'oid': oid, 'size': '1024'}]}
118 118 response = git_lfs_app.post_json(
119 119 '/repo/info/lfs/objects/batch', params=params,
120 120 extra_environ=http_auth)
121 121
122 122 expected_objects = [
123 123 {u'authenticated': True,
124 124 u'actions': {
125 125 u'download': {
126 126 u'header': {u'Authorization': u'Basic XXXXX'},
127 127 u'href': u'http://localhost/repo/info/lfs/objects/456'},
128 128 },
129 129 u'oid': u'456',
130 130 u'size': u'1024'}
131 131 ]
132 132 assert json.loads(response.text) == {
133 133 'objects': expected_objects, 'transfer': 'basic'}
134 134
135 135 def test_app_batch_api_upload(self, git_lfs_app, http_auth):
136 136 params = {'operation': 'upload',
137 137 'objects': [{'oid': '123', 'size': '1024'}]}
138 138 response = git_lfs_app.post_json(
139 139 '/repo/info/lfs/objects/batch', params=params,
140 140 extra_environ=http_auth)
141 141 expected_objects = [
142 142 {u'authenticated': True,
143 143 u'actions': {
144 144 u'upload': {
145 u'header': {u'Authorization': u'Basic XXXXX'},
145 u'header': {u'Authorization': u'Basic XXXXX',
146 u'Transfer-Encoding': u'chunked'},
146 147 u'href': u'http://localhost/repo/info/lfs/objects/123'},
147 148 u'verify': {
148 149 u'header': {u'Authorization': u'Basic XXXXX'},
149 150 u'href': u'http://localhost/repo/info/lfs/verify'}
150 151 },
151 152 u'oid': u'123',
152 153 u'size': u'1024'}
153 154 ]
154 155 assert json.loads(response.text) == {
155 156 'objects': expected_objects, 'transfer': 'basic'}
156 157
157 158 def test_app_verify_api_missing_data(self, git_lfs_app):
158 159 params = {'oid': 'missing',}
159 160 response = git_lfs_app.post_json(
160 161 '/repo/info/lfs/verify', params=params,
161 162 status=400)
162 163
163 164 assert json.loads(response.text) == {
164 165 u'message': u'missing oid and size in request data'}
165 166
166 167 def test_app_verify_api_missing_obj(self, git_lfs_app):
167 168 params = {'oid': 'missing', 'size': '1024'}
168 169 response = git_lfs_app.post_json(
169 170 '/repo/info/lfs/verify', params=params,
170 171 status=404)
171 172
172 173 assert json.loads(response.text) == {
173 174 u'message': u'oid `missing` does not exists in store'}
174 175
175 176 def test_app_verify_api_size_mismatch(self, git_lfs_app):
176 177 oid = 'existing'
177 178 oid_path = os.path.join(git_lfs_app._store, oid)
178 179 if not os.path.isdir(os.path.dirname(oid_path)):
179 180 os.makedirs(os.path.dirname(oid_path))
180 181 with open(oid_path, 'wb') as f:
181 182 f.write('OID_CONTENT')
182 183
183 184 params = {'oid': oid, 'size': '1024'}
184 185 response = git_lfs_app.post_json(
185 186 '/repo/info/lfs/verify', params=params, status=422)
186 187
187 188 assert json.loads(response.text) == {
188 189 u'message': u'requested file size mismatch '
189 190 u'store size:11 requested:1024'}
190 191
191 192 def test_app_verify_api(self, git_lfs_app):
192 193 oid = 'existing'
193 194 oid_path = os.path.join(git_lfs_app._store, oid)
194 195 if not os.path.isdir(os.path.dirname(oid_path)):
195 196 os.makedirs(os.path.dirname(oid_path))
196 197 with open(oid_path, 'wb') as f:
197 198 f.write('OID_CONTENT')
198 199
199 200 params = {'oid': oid, 'size': 11}
200 201 response = git_lfs_app.post_json(
201 202 '/repo/info/lfs/verify', params=params)
202 203
203 204 assert json.loads(response.text) == {
204 205 u'message': {u'size': u'ok', u'in_store': u'ok'}}
205 206
206 207 def test_app_download_api_oid_not_existing(self, git_lfs_app):
207 208 oid = 'missing'
208 209
209 210 response = git_lfs_app.get(
210 211 '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404)
211 212
212 213 assert json.loads(response.text) == {
213 214 u'message': u'requested file with oid `missing` not found in store'}
214 215
215 216 def test_app_download_api(self, git_lfs_app):
216 217 oid = 'existing'
217 218 oid_path = os.path.join(git_lfs_app._store, oid)
218 219 if not os.path.isdir(os.path.dirname(oid_path)):
219 220 os.makedirs(os.path.dirname(oid_path))
220 221 with open(oid_path, 'wb') as f:
221 222 f.write('OID_CONTENT')
222 223
223 224 response = git_lfs_app.get(
224 225 '/repo/info/lfs/objects/{oid}'.format(oid=oid))
225 226 assert response
226 227
227 228 def test_app_upload(self, git_lfs_app):
228 229 oid = 'uploaded'
229 230
230 231 response = git_lfs_app.put(
231 232 '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT')
232 233
233 234 assert json.loads(response.text) == {u'upload': u'ok'}
234 235
235 236 # verify that we actually wrote that OID
236 237 oid_path = os.path.join(git_lfs_app._store, oid)
237 238 assert os.path.isfile(oid_path)
238 239 assert 'CONTENT' == open(oid_path).read()
General Comments 0
You need to be logged in to leave comments. Login now