##// END OF EJS Templates
git-lfs: don't store oid under repo-path....
marcink -
r181:58195c38 default
parent child Browse files
Show More
@@ -1,276 +1,276 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2017 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import re
18 import re
19 import logging
19 import logging
20 from wsgiref.util import FileWrapper
20 from wsgiref.util import FileWrapper
21
21
22 import simplejson as json
22 import simplejson as json
23 from pyramid.config import Configurator
23 from pyramid.config import Configurator
24 from pyramid.response import Response, FileIter
24 from pyramid.response import Response, FileIter
25 from pyramid.httpexceptions import (
25 from pyramid.httpexceptions import (
26 HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden,
26 HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden,
27 HTTPUnprocessableEntity)
27 HTTPUnprocessableEntity)
28
28
29 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
29 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
30 from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator
30 from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator
31 from vcsserver.utils import safe_int
31 from vcsserver.utils import safe_int
32
32
33 log = logging.getLogger(__name__)
33 log = logging.getLogger(__name__)
34
34
35
35
36 GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs' #+json ?
36 GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs' #+json ?
37 GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))')
37 GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))')
38
38
39
39
40 def write_response_error(http_exception, text=None):
40 def write_response_error(http_exception, text=None):
41 content_type = 'application/json'
41 content_type = 'application/json'
42 _exception = http_exception(content_type=content_type)
42 _exception = http_exception(content_type=content_type)
43 _exception.content_type = content_type
43 _exception.content_type = content_type
44 if text:
44 if text:
45 _exception.body = json.dumps({'message': text})
45 _exception.body = json.dumps({'message': text})
46 log.debug('LFS: writing response of type %s to client with text:%s',
46 log.debug('LFS: writing response of type %s to client with text:%s',
47 http_exception, text)
47 http_exception, text)
48 return _exception
48 return _exception
49
49
50
50
51 class AuthHeaderRequired(object):
51 class AuthHeaderRequired(object):
52 """
52 """
53 Decorator to check if request has proper auth-header
53 Decorator to check if request has proper auth-header
54 """
54 """
55
55
56 def __call__(self, func):
56 def __call__(self, func):
57 return get_cython_compat_decorator(self.__wrapper, func)
57 return get_cython_compat_decorator(self.__wrapper, func)
58
58
59 def __wrapper(self, func, *fargs, **fkwargs):
59 def __wrapper(self, func, *fargs, **fkwargs):
60 request = fargs[1]
60 request = fargs[1]
61 auth = request.authorization
61 auth = request.authorization
62 if not auth:
62 if not auth:
63 return write_response_error(HTTPForbidden)
63 return write_response_error(HTTPForbidden)
64 return func(*fargs[1:], **fkwargs)
64 return func(*fargs[1:], **fkwargs)
65
65
66
66
67 # views
67 # views
68
68
69 def lfs_objects(request):
69 def lfs_objects(request):
70 # indicate not supported, V1 API
70 # indicate not supported, V1 API
71 log.warning('LFS: v1 api not supported, reporting it back to client')
71 log.warning('LFS: v1 api not supported, reporting it back to client')
72 return write_response_error(HTTPNotImplemented, 'LFS: v1 api not supported')
72 return write_response_error(HTTPNotImplemented, 'LFS: v1 api not supported')
73
73
74
74
75 @AuthHeaderRequired()
75 @AuthHeaderRequired()
76 def lfs_objects_batch(request):
76 def lfs_objects_batch(request):
77 """
77 """
78 The client sends the following information to the Batch endpoint to transfer some objects:
78 The client sends the following information to the Batch endpoint to transfer some objects:
79
79
80 operation - Should be download or upload.
80 operation - Should be download or upload.
81 transfers - An optional Array of String identifiers for transfer
81 transfers - An optional Array of String identifiers for transfer
82 adapters that the client has configured. If omitted, the basic
82 adapters that the client has configured. If omitted, the basic
83 transfer adapter MUST be assumed by the server.
83 transfer adapter MUST be assumed by the server.
84 objects - An Array of objects to download.
84 objects - An Array of objects to download.
85 oid - String OID of the LFS object.
85 oid - String OID of the LFS object.
86 size - Integer byte size of the LFS object. Must be at least zero.
86 size - Integer byte size of the LFS object. Must be at least zero.
87 """
87 """
88 auth = request.authorization
88 auth = request.authorization
89
89
90 repo = request.matchdict.get('repo')
90 repo = request.matchdict.get('repo')
91
91
92 data = request.json
92 data = request.json
93 operation = data.get('operation')
93 operation = data.get('operation')
94 if operation not in ('download', 'upload'):
94 if operation not in ('download', 'upload'):
95 log.debug('LFS: unsupported operation:%s', operation)
95 log.debug('LFS: unsupported operation:%s', operation)
96 return write_response_error(
96 return write_response_error(
97 HTTPBadRequest, 'unsupported operation mode: `%s`' % operation)
97 HTTPBadRequest, 'unsupported operation mode: `%s`' % operation)
98
98
99 if 'objects' not in data:
99 if 'objects' not in data:
100 log.debug('LFS: missing objects data')
100 log.debug('LFS: missing objects data')
101 return write_response_error(
101 return write_response_error(
102 HTTPBadRequest, 'missing objects data')
102 HTTPBadRequest, 'missing objects data')
103
103
104 log.debug('LFS: handling operation of type: %s', operation)
104 log.debug('LFS: handling operation of type: %s', operation)
105
105
106 objects = []
106 objects = []
107 for o in data['objects']:
107 for o in data['objects']:
108 try:
108 try:
109 oid = o['oid']
109 oid = o['oid']
110 obj_size = o['size']
110 obj_size = o['size']
111 except KeyError:
111 except KeyError:
112 log.exception('LFS, failed to extract data')
112 log.exception('LFS, failed to extract data')
113 return write_response_error(
113 return write_response_error(
114 HTTPBadRequest, 'unsupported data in objects')
114 HTTPBadRequest, 'unsupported data in objects')
115
115
116 obj_data = {'oid': oid}
116 obj_data = {'oid': oid}
117
117
118 obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid)
118 obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid)
119 obj_verify_href = request.route_url('lfs_objects_verify', repo=repo)
119 obj_verify_href = request.route_url('lfs_objects_verify', repo=repo)
120 store = LFSOidStore(
120 store = LFSOidStore(
121 repo, oid, store_location=request.registry.git_lfs_store_path)
121 oid, repo, store_location=request.registry.git_lfs_store_path)
122 handler = OidHandler(
122 handler = OidHandler(
123 store, repo, auth, oid, obj_size, obj_data,
123 store, repo, auth, oid, obj_size, obj_data,
124 obj_href, obj_verify_href)
124 obj_href, obj_verify_href)
125
125
126 # this verifies also OIDs
126 # this verifies also OIDs
127 actions, errors = handler.exec_operation(operation)
127 actions, errors = handler.exec_operation(operation)
128 if errors:
128 if errors:
129 log.warning('LFS: got following errors: %s', errors)
129 log.warning('LFS: got following errors: %s', errors)
130 obj_data['errors'] = errors
130 obj_data['errors'] = errors
131
131
132 if actions:
132 if actions:
133 obj_data['actions'] = actions
133 obj_data['actions'] = actions
134
134
135 obj_data['size'] = obj_size
135 obj_data['size'] = obj_size
136 obj_data['authenticated'] = True
136 obj_data['authenticated'] = True
137 objects.append(obj_data)
137 objects.append(obj_data)
138
138
139 result = {'objects': objects, 'transfer': 'basic'}
139 result = {'objects': objects, 'transfer': 'basic'}
140 log.debug('LFS Response %s', safe_result(result))
140 log.debug('LFS Response %s', safe_result(result))
141
141
142 return result
142 return result
143
143
144
144
145 def lfs_objects_oid_upload(request):
145 def lfs_objects_oid_upload(request):
146 repo = request.matchdict.get('repo')
146 repo = request.matchdict.get('repo')
147 oid = request.matchdict.get('oid')
147 oid = request.matchdict.get('oid')
148 store = LFSOidStore(
148 store = LFSOidStore(
149 repo, oid, store_location=request.registry.git_lfs_store_path)
149 oid, repo, store_location=request.registry.git_lfs_store_path)
150 engine = store.get_engine(mode='wb')
150 engine = store.get_engine(mode='wb')
151 log.debug('LFS: starting chunked write of LFS oid: %s to storage', oid)
151 log.debug('LFS: starting chunked write of LFS oid: %s to storage', oid)
152 with engine as f:
152 with engine as f:
153 for chunk in FileWrapper(request.body_file_seekable, blksize=64 * 1024):
153 for chunk in FileWrapper(request.body_file_seekable, blksize=64 * 1024):
154 f.write(chunk)
154 f.write(chunk)
155
155
156 return {'upload': 'ok'}
156 return {'upload': 'ok'}
157
157
158
158
159 def lfs_objects_oid_download(request):
159 def lfs_objects_oid_download(request):
160 repo = request.matchdict.get('repo')
160 repo = request.matchdict.get('repo')
161 oid = request.matchdict.get('oid')
161 oid = request.matchdict.get('oid')
162
162
163 store = LFSOidStore(
163 store = LFSOidStore(
164 repo, oid, store_location=request.registry.git_lfs_store_path)
164 oid, repo, store_location=request.registry.git_lfs_store_path)
165 if not store.has_oid():
165 if not store.has_oid():
166 log.debug('LFS: oid %s does not exists in store', oid)
166 log.debug('LFS: oid %s does not exists in store', oid)
167 return write_response_error(
167 return write_response_error(
168 HTTPNotFound, 'requested file with oid `%s` not found in store' % oid)
168 HTTPNotFound, 'requested file with oid `%s` not found in store' % oid)
169
169
170 # TODO(marcink): support range header ?
170 # TODO(marcink): support range header ?
171 # Range: bytes=0-, `bytes=(\d+)\-.*`
171 # Range: bytes=0-, `bytes=(\d+)\-.*`
172
172
173 f = open(store.oid_path, 'rb')
173 f = open(store.oid_path, 'rb')
174 response = Response(
174 response = Response(
175 content_type='application/octet-stream', app_iter=FileIter(f))
175 content_type='application/octet-stream', app_iter=FileIter(f))
176 response.headers.add('X-RC-LFS-Response-Oid', str(oid))
176 response.headers.add('X-RC-LFS-Response-Oid', str(oid))
177 return response
177 return response
178
178
179
179
180 def lfs_objects_verify(request):
180 def lfs_objects_verify(request):
181 repo = request.matchdict.get('repo')
181 repo = request.matchdict.get('repo')
182
182
183 data = request.json
183 data = request.json
184 oid = data.get('oid')
184 oid = data.get('oid')
185 size = safe_int(data.get('size'))
185 size = safe_int(data.get('size'))
186
186
187 if not (oid and size):
187 if not (oid and size):
188 return write_response_error(
188 return write_response_error(
189 HTTPBadRequest, 'missing oid and size in request data')
189 HTTPBadRequest, 'missing oid and size in request data')
190
190
191 store = LFSOidStore(repo, oid,
191 store = LFSOidStore(
192 store_location=request.registry.git_lfs_store_path)
192 oid, repo, store_location=request.registry.git_lfs_store_path)
193 if not store.has_oid():
193 if not store.has_oid():
194 log.debug('LFS: oid %s does not exists in store', oid)
194 log.debug('LFS: oid %s does not exists in store', oid)
195 return write_response_error(
195 return write_response_error(
196 HTTPNotFound, 'oid `%s` does not exists in store' % oid)
196 HTTPNotFound, 'oid `%s` does not exists in store' % oid)
197
197
198 store_size = store.size_oid()
198 store_size = store.size_oid()
199 if store_size != size:
199 if store_size != size:
200 msg = 'requested file size mismatch store size:%s requested:%s' % (
200 msg = 'requested file size mismatch store size:%s requested:%s' % (
201 store_size, size)
201 store_size, size)
202 return write_response_error(
202 return write_response_error(
203 HTTPUnprocessableEntity, msg)
203 HTTPUnprocessableEntity, msg)
204
204
205 return {'message': {'size': 'ok', 'in_store': 'ok'}}
205 return {'message': {'size': 'ok', 'in_store': 'ok'}}
206
206
207
207
208 def lfs_objects_lock(request):
208 def lfs_objects_lock(request):
209 return write_response_error(
209 return write_response_error(
210 HTTPNotImplemented, 'GIT LFS locking api not supported')
210 HTTPNotImplemented, 'GIT LFS locking api not supported')
211
211
212
212
213 def not_found(request):
213 def not_found(request):
214 return write_response_error(
214 return write_response_error(
215 HTTPNotFound, 'request path not found')
215 HTTPNotFound, 'request path not found')
216
216
217
217
218 def lfs_disabled(request):
218 def lfs_disabled(request):
219 return write_response_error(
219 return write_response_error(
220 HTTPNotImplemented, 'GIT LFS disabled for this repo')
220 HTTPNotImplemented, 'GIT LFS disabled for this repo')
221
221
222
222
223 def git_lfs_app(config):
223 def git_lfs_app(config):
224
224
225 # v1 API deprecation endpoint
225 # v1 API deprecation endpoint
226 config.add_route('lfs_objects',
226 config.add_route('lfs_objects',
227 '/{repo:.*?[^/]}/info/lfs/objects')
227 '/{repo:.*?[^/]}/info/lfs/objects')
228 config.add_view(lfs_objects, route_name='lfs_objects',
228 config.add_view(lfs_objects, route_name='lfs_objects',
229 request_method='POST', renderer='json')
229 request_method='POST', renderer='json')
230
230
231 # locking API
231 # locking API
232 config.add_route('lfs_objects_lock',
232 config.add_route('lfs_objects_lock',
233 '/{repo:.*?[^/]}/info/lfs/locks')
233 '/{repo:.*?[^/]}/info/lfs/locks')
234 config.add_view(lfs_objects_lock, route_name='lfs_objects_lock',
234 config.add_view(lfs_objects_lock, route_name='lfs_objects_lock',
235 request_method=('POST', 'GET'), renderer='json')
235 request_method=('POST', 'GET'), renderer='json')
236
236
237 config.add_route('lfs_objects_lock_verify',
237 config.add_route('lfs_objects_lock_verify',
238 '/{repo:.*?[^/]}/info/lfs/locks/verify')
238 '/{repo:.*?[^/]}/info/lfs/locks/verify')
239 config.add_view(lfs_objects_lock, route_name='lfs_objects_lock_verify',
239 config.add_view(lfs_objects_lock, route_name='lfs_objects_lock_verify',
240 request_method=('POST', 'GET'), renderer='json')
240 request_method=('POST', 'GET'), renderer='json')
241
241
242 # batch API
242 # batch API
243 config.add_route('lfs_objects_batch',
243 config.add_route('lfs_objects_batch',
244 '/{repo:.*?[^/]}/info/lfs/objects/batch')
244 '/{repo:.*?[^/]}/info/lfs/objects/batch')
245 config.add_view(lfs_objects_batch, route_name='lfs_objects_batch',
245 config.add_view(lfs_objects_batch, route_name='lfs_objects_batch',
246 request_method='POST', renderer='json')
246 request_method='POST', renderer='json')
247
247
248 # oid upload/download API
248 # oid upload/download API
249 config.add_route('lfs_objects_oid',
249 config.add_route('lfs_objects_oid',
250 '/{repo:.*?[^/]}/info/lfs/objects/{oid}')
250 '/{repo:.*?[^/]}/info/lfs/objects/{oid}')
251 config.add_view(lfs_objects_oid_upload, route_name='lfs_objects_oid',
251 config.add_view(lfs_objects_oid_upload, route_name='lfs_objects_oid',
252 request_method='PUT', renderer='json')
252 request_method='PUT', renderer='json')
253 config.add_view(lfs_objects_oid_download, route_name='lfs_objects_oid',
253 config.add_view(lfs_objects_oid_download, route_name='lfs_objects_oid',
254 request_method='GET', renderer='json')
254 request_method='GET', renderer='json')
255
255
256 # verification API
256 # verification API
257 config.add_route('lfs_objects_verify',
257 config.add_route('lfs_objects_verify',
258 '/{repo:.*?[^/]}/info/lfs/verify')
258 '/{repo:.*?[^/]}/info/lfs/verify')
259 config.add_view(lfs_objects_verify, route_name='lfs_objects_verify',
259 config.add_view(lfs_objects_verify, route_name='lfs_objects_verify',
260 request_method='POST', renderer='json')
260 request_method='POST', renderer='json')
261
261
262 # not found handler for API
262 # not found handler for API
263 config.add_notfound_view(not_found, renderer='json')
263 config.add_notfound_view(not_found, renderer='json')
264
264
265
265
266 def create_app(git_lfs_enabled, git_lfs_store_path):
266 def create_app(git_lfs_enabled, git_lfs_store_path):
267 config = Configurator()
267 config = Configurator()
268 if git_lfs_enabled:
268 if git_lfs_enabled:
269 config.include(git_lfs_app)
269 config.include(git_lfs_app)
270 config.registry.git_lfs_store_path = git_lfs_store_path
270 config.registry.git_lfs_store_path = git_lfs_store_path
271 else:
271 else:
272 # not found handler for API, reporting disabled LFS support
272 # not found handler for API, reporting disabled LFS support
273 config.add_notfound_view(lfs_disabled, renderer='json')
273 config.add_notfound_view(lfs_disabled, renderer='json')
274
274
275 app = config.make_wsgi_app()
275 app = config.make_wsgi_app()
276 return app
276 return app
@@ -1,167 +1,166 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2017 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import os
19 import shutil
19 import shutil
20 import logging
20 import logging
21 from collections import OrderedDict
21 from collections import OrderedDict
22
22
23 log = logging.getLogger(__name__)
23 log = logging.getLogger(__name__)
24
24
25
25
26 class OidHandler(object):
26 class OidHandler(object):
27
27
28 def __init__(self, store, repo_name, auth, oid, obj_size, obj_data, obj_href,
28 def __init__(self, store, repo_name, auth, oid, obj_size, obj_data, obj_href,
29 obj_verify_href=None):
29 obj_verify_href=None):
30 self.current_store = store
30 self.current_store = store
31 self.repo_name = repo_name
31 self.repo_name = repo_name
32 self.auth = auth
32 self.auth = auth
33 self.oid = oid
33 self.oid = oid
34 self.obj_size = obj_size
34 self.obj_size = obj_size
35 self.obj_data = obj_data
35 self.obj_data = obj_data
36 self.obj_href = obj_href
36 self.obj_href = obj_href
37 self.obj_verify_href = obj_verify_href
37 self.obj_verify_href = obj_verify_href
38
38
39 def get_store(self, mode=None):
39 def get_store(self, mode=None):
40 return self.current_store
40 return self.current_store
41
41
42 def get_auth(self):
42 def get_auth(self):
43 """returns auth header for re-use in upload/download"""
43 """returns auth header for re-use in upload/download"""
44 return " ".join(self.auth)
44 return " ".join(self.auth)
45
45
46 def download(self):
46 def download(self):
47
47
48 store = self.get_store()
48 store = self.get_store()
49 response = None
49 response = None
50 has_errors = None
50 has_errors = None
51
51
52 if not store.has_oid():
52 if not store.has_oid():
53 # error reply back to client that something is wrong with dl
53 # error reply back to client that something is wrong with dl
54 err_msg = 'object: {} does not exist in store'.format(store.oid)
54 err_msg = 'object: {} does not exist in store'.format(store.oid)
55 has_errors = OrderedDict(
55 has_errors = OrderedDict(
56 error=OrderedDict(
56 error=OrderedDict(
57 code=404,
57 code=404,
58 message=err_msg
58 message=err_msg
59 )
59 )
60 )
60 )
61
61
62 download_action = OrderedDict(
62 download_action = OrderedDict(
63 href=self.obj_href,
63 href=self.obj_href,
64 header=OrderedDict([("Authorization", self.get_auth())])
64 header=OrderedDict([("Authorization", self.get_auth())])
65 )
65 )
66 if not has_errors:
66 if not has_errors:
67 response = OrderedDict(download=download_action)
67 response = OrderedDict(download=download_action)
68 return response, has_errors
68 return response, has_errors
69
69
70 def upload(self, skip_existing=True):
70 def upload(self, skip_existing=True):
71 """
71 """
72 Write upload action for git-lfs server
72 Write upload action for git-lfs server
73 """
73 """
74
74
75 store = self.get_store()
75 store = self.get_store()
76 response = None
76 response = None
77 has_errors = None
77 has_errors = None
78
78
79 # verify if we have the OID before, if we do, reply with empty
79 # verify if we have the OID before, if we do, reply with empty
80 if store.has_oid():
80 if store.has_oid():
81 log.debug('LFS: store already has oid %s', store.oid)
81 log.debug('LFS: store already has oid %s', store.oid)
82 if skip_existing:
82 if skip_existing:
83 log.debug('LFS: skipping further action as oid is existing')
83 log.debug('LFS: skipping further action as oid is existing')
84 return response, has_errors
84 return response, has_errors
85
85
86 upload_action = OrderedDict(
86 upload_action = OrderedDict(
87 href=self.obj_href,
87 href=self.obj_href,
88 header=OrderedDict([("Authorization", self.get_auth())])
88 header=OrderedDict([("Authorization", self.get_auth())])
89 )
89 )
90 if not has_errors:
90 if not has_errors:
91 response = OrderedDict(upload=upload_action)
91 response = OrderedDict(upload=upload_action)
92 # if specified in handler, return the verification endpoint
92 # if specified in handler, return the verification endpoint
93 if self.obj_verify_href:
93 if self.obj_verify_href:
94 verify_action = OrderedDict(
94 verify_action = OrderedDict(
95 href=self.obj_verify_href,
95 href=self.obj_verify_href,
96 header=OrderedDict([("Authorization", self.get_auth())])
96 header=OrderedDict([("Authorization", self.get_auth())])
97 )
97 )
98 response['verify'] = verify_action
98 response['verify'] = verify_action
99 return response, has_errors
99 return response, has_errors
100
100
101 def exec_operation(self, operation, *args, **kwargs):
101 def exec_operation(self, operation, *args, **kwargs):
102 handler = getattr(self, operation)
102 handler = getattr(self, operation)
103 log.debug('LFS: handling request using %s handler', handler)
103 log.debug('LFS: handling request using %s handler', handler)
104 return handler(*args, **kwargs)
104 return handler(*args, **kwargs)
105
105
106
106
107 class LFSOidStore(object):
107 class LFSOidStore(object):
108
108
109 def __init__(self, repo, oid, store_location=None):
109 def __init__(self, oid, repo, store_location=None):
110 self._store = store_location or self.get_default_store()
111 self.oid = oid
110 self.oid = oid
112 self.repo = repo
111 self.repo = repo
113 self.store_path = os.path.join(self._store, repo)
112 self.store_path = store_location or self.get_default_store()
114 self.tmp_oid_path = os.path.join(self.store_path, oid + '.tmp')
113 self.tmp_oid_path = os.path.join(self.store_path, oid + '.tmp')
115 self.oid_path = os.path.join(self.store_path, oid)
114 self.oid_path = os.path.join(self.store_path, oid)
116 self.fd = None
115 self.fd = None
117
116
118 def get_engine(self, mode):
117 def get_engine(self, mode):
119 """
118 """
120 engine = .get_engine(mode='wb')
119 engine = .get_engine(mode='wb')
121 with engine as f:
120 with engine as f:
122 f.write('...')
121 f.write('...')
123 """
122 """
124
123
125 class StoreEngine(object):
124 class StoreEngine(object):
126 def __init__(self, mode, store_path, oid_path, tmp_oid_path):
125 def __init__(self, mode, store_path, oid_path, tmp_oid_path):
127 self.mode = mode
126 self.mode = mode
128 self.store_path = store_path
127 self.store_path = store_path
129 self.oid_path = oid_path
128 self.oid_path = oid_path
130 self.tmp_oid_path = tmp_oid_path
129 self.tmp_oid_path = tmp_oid_path
131
130
132 def __enter__(self):
131 def __enter__(self):
133 if not os.path.isdir(self.store_path):
132 if not os.path.isdir(self.store_path):
134 os.makedirs(self.store_path)
133 os.makedirs(self.store_path)
135
134
136 # TODO(marcink): maybe write metadata here with size/oid ?
135 # TODO(marcink): maybe write metadata here with size/oid ?
137 fd = open(self.tmp_oid_path, self.mode)
136 fd = open(self.tmp_oid_path, self.mode)
138 self.fd = fd
137 self.fd = fd
139 return fd
138 return fd
140
139
141 def __exit__(self, exc_type, exc_value, traceback):
140 def __exit__(self, exc_type, exc_value, traceback):
142 # close tmp file, and rename to final destination
141 # close tmp file, and rename to final destination
143 self.fd.close()
142 self.fd.close()
144 shutil.move(self.tmp_oid_path, self.oid_path)
143 shutil.move(self.tmp_oid_path, self.oid_path)
145
144
146 return StoreEngine(
145 return StoreEngine(
147 mode, self.store_path, self.oid_path, self.tmp_oid_path)
146 mode, self.store_path, self.oid_path, self.tmp_oid_path)
148
147
149 def get_default_store(self):
148 def get_default_store(self):
150 """
149 """
151 Default store, consistent with defaults of Mercurial large files store
150 Default store, consistent with defaults of Mercurial large files store
152 which is /home/username/.cache/largefiles
151 which is /home/username/.cache/largefiles
153 """
152 """
154 user_home = os.path.expanduser("~")
153 user_home = os.path.expanduser("~")
155 return os.path.join(user_home, '.cache', 'lfs-store')
154 return os.path.join(user_home, '.cache', 'lfs-store')
156
155
157 def has_oid(self):
156 def has_oid(self):
158 return os.path.exists(os.path.join(self.store_path, self.oid))
157 return os.path.exists(os.path.join(self.store_path, self.oid))
159
158
160 def size_oid(self):
159 def size_oid(self):
161 size = -1
160 size = -1
162
161
163 if self.has_oid():
162 if self.has_oid():
164 oid = os.path.join(self.store_path, self.oid)
163 oid = os.path.join(self.store_path, self.oid)
165 size = os.stat(oid).st_size
164 size = os.stat(oid).st_size
166
165
167 return size No newline at end of file
166 return size
@@ -1,233 +1,237 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2017 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import os
19 import pytest
19 import pytest
20 from webtest.app import TestApp as WebObTestApp
20 from webtest.app import TestApp as WebObTestApp
21
21
22 from vcsserver.git_lfs.app import create_app
22 from vcsserver.git_lfs.app import create_app
23
23
24
24
25 @pytest.fixture(scope='function')
25 @pytest.fixture(scope='function')
26 def git_lfs_app(tmpdir):
26 def git_lfs_app(tmpdir):
27 custom_app = WebObTestApp(create_app(
27 custom_app = WebObTestApp(create_app(
28 git_lfs_enabled=True, git_lfs_store_path=str(tmpdir)))
28 git_lfs_enabled=True, git_lfs_store_path=str(tmpdir)))
29 custom_app._store = str(tmpdir)
29 custom_app._store = str(tmpdir)
30 return custom_app
30 return custom_app
31
31
32
32
33 @pytest.fixture()
33 @pytest.fixture()
34 def http_auth():
34 def http_auth():
35 return {'HTTP_AUTHORIZATION': "Basic XXXXX"}
35 return {'HTTP_AUTHORIZATION': "Basic XXXXX"}
36
36
37
37
38 class TestLFSApplication(object):
38 class TestLFSApplication(object):
39
39
40 def test_app_wrong_path(self, git_lfs_app):
40 def test_app_wrong_path(self, git_lfs_app):
41 git_lfs_app.get('/repo/info/lfs/xxx', status=404)
41 git_lfs_app.get('/repo/info/lfs/xxx', status=404)
42
42
43 def test_app_deprecated_endpoint(self, git_lfs_app):
43 def test_app_deprecated_endpoint(self, git_lfs_app):
44 response = git_lfs_app.post('/repo/info/lfs/objects', status=501)
44 response = git_lfs_app.post('/repo/info/lfs/objects', status=501)
45 assert response.status_code == 501
45 assert response.status_code == 501
46 assert response.json == {u'message': u'LFS: v1 api not supported'}
46 assert response.json == {u'message': u'LFS: v1 api not supported'}
47
47
48 def test_app_lock_verify_api_not_available(self, git_lfs_app):
48 def test_app_lock_verify_api_not_available(self, git_lfs_app):
49 response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501)
49 response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501)
50 assert response.status_code == 501
50 assert response.status_code == 501
51 assert response.json == {
51 assert response.json == {
52 u'message': u'GIT LFS locking api not supported'}
52 u'message': u'GIT LFS locking api not supported'}
53
53
54 def test_app_lock_api_not_available(self, git_lfs_app):
54 def test_app_lock_api_not_available(self, git_lfs_app):
55 response = git_lfs_app.post('/repo/info/lfs/locks', status=501)
55 response = git_lfs_app.post('/repo/info/lfs/locks', status=501)
56 assert response.status_code == 501
56 assert response.status_code == 501
57 assert response.json == {
57 assert response.json == {
58 u'message': u'GIT LFS locking api not supported'}
58 u'message': u'GIT LFS locking api not supported'}
59
59
60 def test_app_batch_api_missing_auth(self, git_lfs_app,):
60 def test_app_batch_api_missing_auth(self, git_lfs_app,):
61 git_lfs_app.post_json(
61 git_lfs_app.post_json(
62 '/repo/info/lfs/objects/batch', params={}, status=403)
62 '/repo/info/lfs/objects/batch', params={}, status=403)
63
63
64 def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth):
64 def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth):
65 response = git_lfs_app.post_json(
65 response = git_lfs_app.post_json(
66 '/repo/info/lfs/objects/batch', params={}, status=400,
66 '/repo/info/lfs/objects/batch', params={}, status=400,
67 extra_environ=http_auth)
67 extra_environ=http_auth)
68 assert response.json == {
68 assert response.json == {
69 u'message': u'unsupported operation mode: `None`'}
69 u'message': u'unsupported operation mode: `None`'}
70
70
71 def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth):
71 def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth):
72 response = git_lfs_app.post_json(
72 response = git_lfs_app.post_json(
73 '/repo/info/lfs/objects/batch', params={'operation': 'download'},
73 '/repo/info/lfs/objects/batch', params={'operation': 'download'},
74 status=400, extra_environ=http_auth)
74 status=400, extra_environ=http_auth)
75 assert response.json == {
75 assert response.json == {
76 u'message': u'missing objects data'}
76 u'message': u'missing objects data'}
77
77
78 def test_app_batch_api_unsupported_data_in_objects(
78 def test_app_batch_api_unsupported_data_in_objects(
79 self, git_lfs_app, http_auth):
79 self, git_lfs_app, http_auth):
80 params = {'operation': 'download',
80 params = {'operation': 'download',
81 'objects': [{}]}
81 'objects': [{}]}
82 response = git_lfs_app.post_json(
82 response = git_lfs_app.post_json(
83 '/repo/info/lfs/objects/batch', params=params, status=400,
83 '/repo/info/lfs/objects/batch', params=params, status=400,
84 extra_environ=http_auth)
84 extra_environ=http_auth)
85 assert response.json == {
85 assert response.json == {
86 u'message': u'unsupported data in objects'}
86 u'message': u'unsupported data in objects'}
87
87
88 def test_app_batch_api_download_missing_object(
88 def test_app_batch_api_download_missing_object(
89 self, git_lfs_app, http_auth):
89 self, git_lfs_app, http_auth):
90 params = {'operation': 'download',
90 params = {'operation': 'download',
91 'objects': [{'oid': '123', 'size': '1024'}]}
91 'objects': [{'oid': '123', 'size': '1024'}]}
92 response = git_lfs_app.post_json(
92 response = git_lfs_app.post_json(
93 '/repo/info/lfs/objects/batch', params=params,
93 '/repo/info/lfs/objects/batch', params=params,
94 extra_environ=http_auth)
94 extra_environ=http_auth)
95
95
96 expected_objects = [
96 expected_objects = [
97 {u'authenticated': True,
97 {u'authenticated': True,
98 u'errors': {u'error': {
98 u'errors': {u'error': {
99 u'code': 404,
99 u'code': 404,
100 u'message': u'object: 123 does not exist in store'}},
100 u'message': u'object: 123 does not exist in store'}},
101 u'oid': u'123',
101 u'oid': u'123',
102 u'size': u'1024'}
102 u'size': u'1024'}
103 ]
103 ]
104 assert response.json == {
104 assert response.json == {
105 'objects': expected_objects, 'transfer': 'basic'}
105 'objects': expected_objects, 'transfer': 'basic'}
106
106
107 def test_app_batch_api_download(self, git_lfs_app, http_auth):
107 def test_app_batch_api_download(self, git_lfs_app, http_auth):
108 oid = '456'
108 oid = '456'
109 oid_path = os.path.join(git_lfs_app._store, 'repo', oid)
109 oid_path = os.path.join(git_lfs_app._store, oid)
110 if not os.path.isdir(os.path.dirname(oid_path)):
110 os.makedirs(os.path.dirname(oid_path))
111 os.makedirs(os.path.dirname(oid_path))
111 with open(oid_path, 'wb') as f:
112 with open(oid_path, 'wb') as f:
112 f.write('OID_CONTENT')
113 f.write('OID_CONTENT')
113
114
114 params = {'operation': 'download',
115 params = {'operation': 'download',
115 'objects': [{'oid': oid, 'size': '1024'}]}
116 'objects': [{'oid': oid, 'size': '1024'}]}
116 response = git_lfs_app.post_json(
117 response = git_lfs_app.post_json(
117 '/repo/info/lfs/objects/batch', params=params,
118 '/repo/info/lfs/objects/batch', params=params,
118 extra_environ=http_auth)
119 extra_environ=http_auth)
119
120
120 expected_objects = [
121 expected_objects = [
121 {u'authenticated': True,
122 {u'authenticated': True,
122 u'actions': {
123 u'actions': {
123 u'download': {
124 u'download': {
124 u'header': {u'Authorization': u'Basic XXXXX'},
125 u'header': {u'Authorization': u'Basic XXXXX'},
125 u'href': u'http://localhost/repo/info/lfs/objects/456'},
126 u'href': u'http://localhost/repo/info/lfs/objects/456'},
126 },
127 },
127 u'oid': u'456',
128 u'oid': u'456',
128 u'size': u'1024'}
129 u'size': u'1024'}
129 ]
130 ]
130 assert response.json == {
131 assert response.json == {
131 'objects': expected_objects, 'transfer': 'basic'}
132 'objects': expected_objects, 'transfer': 'basic'}
132
133
133 def test_app_batch_api_upload(self, git_lfs_app, http_auth):
134 def test_app_batch_api_upload(self, git_lfs_app, http_auth):
134 params = {'operation': 'upload',
135 params = {'operation': 'upload',
135 'objects': [{'oid': '123', 'size': '1024'}]}
136 'objects': [{'oid': '123', 'size': '1024'}]}
136 response = git_lfs_app.post_json(
137 response = git_lfs_app.post_json(
137 '/repo/info/lfs/objects/batch', params=params,
138 '/repo/info/lfs/objects/batch', params=params,
138 extra_environ=http_auth)
139 extra_environ=http_auth)
139 expected_objects = [
140 expected_objects = [
140 {u'authenticated': True,
141 {u'authenticated': True,
141 u'actions': {
142 u'actions': {
142 u'upload': {
143 u'upload': {
143 u'header': {u'Authorization': u'Basic XXXXX'},
144 u'header': {u'Authorization': u'Basic XXXXX'},
144 u'href': u'http://localhost/repo/info/lfs/objects/123'},
145 u'href': u'http://localhost/repo/info/lfs/objects/123'},
145 u'verify': {
146 u'verify': {
146 u'header': {u'Authorization': u'Basic XXXXX'},
147 u'header': {u'Authorization': u'Basic XXXXX'},
147 u'href': u'http://localhost/repo/info/lfs/verify'}
148 u'href': u'http://localhost/repo/info/lfs/verify'}
148 },
149 },
149 u'oid': u'123',
150 u'oid': u'123',
150 u'size': u'1024'}
151 u'size': u'1024'}
151 ]
152 ]
152 assert response.json == {
153 assert response.json == {
153 'objects': expected_objects, 'transfer': 'basic'}
154 'objects': expected_objects, 'transfer': 'basic'}
154
155
155 def test_app_verify_api_missing_data(self, git_lfs_app):
156 def test_app_verify_api_missing_data(self, git_lfs_app):
156 params = {'oid': 'missing',}
157 params = {'oid': 'missing',}
157 response = git_lfs_app.post_json(
158 response = git_lfs_app.post_json(
158 '/repo/info/lfs/verify', params=params,
159 '/repo/info/lfs/verify', params=params,
159 status=400)
160 status=400)
160
161
161 assert response.json == {
162 assert response.json == {
162 u'message': u'missing oid and size in request data'}
163 u'message': u'missing oid and size in request data'}
163
164
164 def test_app_verify_api_missing_obj(self, git_lfs_app):
165 def test_app_verify_api_missing_obj(self, git_lfs_app):
165 params = {'oid': 'missing', 'size': '1024'}
166 params = {'oid': 'missing', 'size': '1024'}
166 response = git_lfs_app.post_json(
167 response = git_lfs_app.post_json(
167 '/repo/info/lfs/verify', params=params,
168 '/repo/info/lfs/verify', params=params,
168 status=404)
169 status=404)
169
170
170 assert response.json == {
171 assert response.json == {
171 u'message': u'oid `missing` does not exists in store'}
172 u'message': u'oid `missing` does not exists in store'}
172
173
173 def test_app_verify_api_size_mismatch(self, git_lfs_app):
174 def test_app_verify_api_size_mismatch(self, git_lfs_app):
174 oid = 'existing'
175 oid = 'existing'
175 oid_path = os.path.join(git_lfs_app._store, 'repo', oid)
176 oid_path = os.path.join(git_lfs_app._store, oid)
177 if not os.path.isdir(os.path.dirname(oid_path)):
176 os.makedirs(os.path.dirname(oid_path))
178 os.makedirs(os.path.dirname(oid_path))
177 with open(oid_path, 'wb') as f:
179 with open(oid_path, 'wb') as f:
178 f.write('OID_CONTENT')
180 f.write('OID_CONTENT')
179
181
180 params = {'oid': oid, 'size': '1024'}
182 params = {'oid': oid, 'size': '1024'}
181 response = git_lfs_app.post_json(
183 response = git_lfs_app.post_json(
182 '/repo/info/lfs/verify', params=params, status=422)
184 '/repo/info/lfs/verify', params=params, status=422)
183
185
184 assert response.json == {
186 assert response.json == {
185 u'message': u'requested file size mismatch '
187 u'message': u'requested file size mismatch '
186 u'store size:11 requested:1024'}
188 u'store size:11 requested:1024'}
187
189
188 def test_app_verify_api(self, git_lfs_app):
190 def test_app_verify_api(self, git_lfs_app):
189 oid = 'existing'
191 oid = 'existing'
190 oid_path = os.path.join(git_lfs_app._store, 'repo', oid)
192 oid_path = os.path.join(git_lfs_app._store, oid)
193 if not os.path.isdir(os.path.dirname(oid_path)):
191 os.makedirs(os.path.dirname(oid_path))
194 os.makedirs(os.path.dirname(oid_path))
192 with open(oid_path, 'wb') as f:
195 with open(oid_path, 'wb') as f:
193 f.write('OID_CONTENT')
196 f.write('OID_CONTENT')
194
197
195 params = {'oid': oid, 'size': 11}
198 params = {'oid': oid, 'size': 11}
196 response = git_lfs_app.post_json(
199 response = git_lfs_app.post_json(
197 '/repo/info/lfs/verify', params=params)
200 '/repo/info/lfs/verify', params=params)
198
201
199 assert response.json == {
202 assert response.json == {
200 u'message': {u'size': u'ok', u'in_store': u'ok'}}
203 u'message': {u'size': u'ok', u'in_store': u'ok'}}
201
204
202 def test_app_download_api_oid_not_existing(self, git_lfs_app):
205 def test_app_download_api_oid_not_existing(self, git_lfs_app):
203 oid = 'missing'
206 oid = 'missing'
204
207
205 response = git_lfs_app.get(
208 response = git_lfs_app.get(
206 '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404)
209 '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404)
207
210
208 assert response.json == {
211 assert response.json == {
209 u'message': u'requested file with oid `missing` not found in store'}
212 u'message': u'requested file with oid `missing` not found in store'}
210
213
211 def test_app_download_api(self, git_lfs_app):
214 def test_app_download_api(self, git_lfs_app):
212 oid = 'existing'
215 oid = 'existing'
213 oid_path = os.path.join(git_lfs_app._store, 'repo', oid)
216 oid_path = os.path.join(git_lfs_app._store, oid)
217 if not os.path.isdir(os.path.dirname(oid_path)):
214 os.makedirs(os.path.dirname(oid_path))
218 os.makedirs(os.path.dirname(oid_path))
215 with open(oid_path, 'wb') as f:
219 with open(oid_path, 'wb') as f:
216 f.write('OID_CONTENT')
220 f.write('OID_CONTENT')
217
221
218 response = git_lfs_app.get(
222 response = git_lfs_app.get(
219 '/repo/info/lfs/objects/{oid}'.format(oid=oid))
223 '/repo/info/lfs/objects/{oid}'.format(oid=oid))
220 assert response
224 assert response
221
225
222 def test_app_upload(self, git_lfs_app):
226 def test_app_upload(self, git_lfs_app):
223 oid = 'uploaded'
227 oid = 'uploaded'
224
228
225 response = git_lfs_app.put(
229 response = git_lfs_app.put(
226 '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT')
230 '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT')
227
231
228 assert response.json == {u'upload': u'ok'}
232 assert response.json == {u'upload': u'ok'}
229
233
230 # verify that we actually wrote that OID
234 # verify that we actually wrote that OID
231 oid_path = os.path.join(git_lfs_app._store, 'repo', oid)
235 oid_path = os.path.join(git_lfs_app._store, oid)
232 assert os.path.isfile(oid_path)
236 assert os.path.isfile(oid_path)
233 assert 'CONTENT' == open(oid_path).read()
237 assert 'CONTENT' == open(oid_path).read()
@@ -1,121 +1,123 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2017 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import os
19 import pytest
19 import pytest
20 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
20 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
21
21
22
22
23 @pytest.fixture()
23 @pytest.fixture()
24 def lfs_store(tmpdir):
24 def lfs_store(tmpdir):
25 repo = 'test'
25 repo = 'test'
26 oid = '123456789'
26 oid = '123456789'
27 store = LFSOidStore(repo=repo, oid=oid, store_location=str(tmpdir))
27 store = LFSOidStore(oid=oid, repo=repo, store_location=str(tmpdir))
28 return store
28 return store
29
29
30
30
31 @pytest.fixture()
31 @pytest.fixture()
32 def oid_handler(lfs_store):
32 def oid_handler(lfs_store):
33 store = lfs_store
33 store = lfs_store
34 repo = store.repo
34 repo = store.repo
35 oid = store.oid
35 oid = store.oid
36
36
37 oid_handler = OidHandler(
37 oid_handler = OidHandler(
38 store=store, repo_name=repo, auth=('basic', 'xxxx'),
38 store=store, repo_name=repo, auth=('basic', 'xxxx'),
39 oid=oid,
39 oid=oid,
40 obj_size='1024', obj_data={}, obj_href='http://localhost/handle_oid',
40 obj_size='1024', obj_data={}, obj_href='http://localhost/handle_oid',
41 obj_verify_href='http://localhost/verify')
41 obj_verify_href='http://localhost/verify')
42 return oid_handler
42 return oid_handler
43
43
44
44
45 class TestOidHandler(object):
45 class TestOidHandler(object):
46
46
47 @pytest.mark.parametrize('exec_action', [
47 @pytest.mark.parametrize('exec_action', [
48 'download',
48 'download',
49 'upload',
49 'upload',
50 ])
50 ])
51 def test_exec_action(self, exec_action, oid_handler):
51 def test_exec_action(self, exec_action, oid_handler):
52 handler = oid_handler.exec_operation(exec_action)
52 handler = oid_handler.exec_operation(exec_action)
53 assert handler
53 assert handler
54
54
55 def test_exec_action_undefined(self, oid_handler):
55 def test_exec_action_undefined(self, oid_handler):
56 with pytest.raises(AttributeError):
56 with pytest.raises(AttributeError):
57 oid_handler.exec_operation('wrong')
57 oid_handler.exec_operation('wrong')
58
58
59 def test_download_oid_not_existing(self, oid_handler):
59 def test_download_oid_not_existing(self, oid_handler):
60 response, has_errors = oid_handler.exec_operation('download')
60 response, has_errors = oid_handler.exec_operation('download')
61
61
62 assert response is None
62 assert response is None
63 assert has_errors['error'] == {
63 assert has_errors['error'] == {
64 'code': 404,
64 'code': 404,
65 'message': 'object: 123456789 does not exist in store'}
65 'message': 'object: 123456789 does not exist in store'}
66
66
67 def test_download_oid(self, oid_handler):
67 def test_download_oid(self, oid_handler):
68 store = oid_handler.get_store()
68 store = oid_handler.get_store()
69 if not os.path.isdir(os.path.dirname(store.oid_path)):
70 os.makedirs(os.path.dirname(store.oid_path))
69
71
70 os.makedirs(os.path.dirname(store.oid_path))
71 with open(store.oid_path, 'wb') as f:
72 with open(store.oid_path, 'wb') as f:
72 f.write('CONTENT')
73 f.write('CONTENT')
73
74
74 response, has_errors = oid_handler.exec_operation('download')
75 response, has_errors = oid_handler.exec_operation('download')
75
76
76 assert has_errors is None
77 assert has_errors is None
77 assert response['download'] == {
78 assert response['download'] == {
78 'header': {'Authorization': 'basic xxxx'},
79 'header': {'Authorization': 'basic xxxx'},
79 'href': 'http://localhost/handle_oid'
80 'href': 'http://localhost/handle_oid'
80 }
81 }
81
82
82 def test_upload_oid_that_exists(self, oid_handler):
83 def test_upload_oid_that_exists(self, oid_handler):
83 store = oid_handler.get_store()
84 store = oid_handler.get_store()
85 if not os.path.isdir(os.path.dirname(store.oid_path)):
86 os.makedirs(os.path.dirname(store.oid_path))
84
87
85 os.makedirs(os.path.dirname(store.oid_path))
86 with open(store.oid_path, 'wb') as f:
88 with open(store.oid_path, 'wb') as f:
87 f.write('CONTENT')
89 f.write('CONTENT')
88
90
89 response, has_errors = oid_handler.exec_operation('upload')
91 response, has_errors = oid_handler.exec_operation('upload')
90 assert has_errors is None
92 assert has_errors is None
91 assert response is None
93 assert response is None
92
94
93 def test_upload_oid(self, oid_handler):
95 def test_upload_oid(self, oid_handler):
94 response, has_errors = oid_handler.exec_operation('upload')
96 response, has_errors = oid_handler.exec_operation('upload')
95 assert has_errors is None
97 assert has_errors is None
96 assert response['upload'] == {
98 assert response['upload'] == {
97 'header': {'Authorization': 'basic xxxx'},
99 'header': {'Authorization': 'basic xxxx'},
98 'href': 'http://localhost/handle_oid'
100 'href': 'http://localhost/handle_oid'
99 }
101 }
100
102
101
103
102 class TestLFSStore(object):
104 class TestLFSStore(object):
103 def test_write_oid(self, lfs_store):
105 def test_write_oid(self, lfs_store):
104 oid_location = lfs_store.oid_path
106 oid_location = lfs_store.oid_path
105
107
106 assert not os.path.isfile(oid_location)
108 assert not os.path.isfile(oid_location)
107
109
108 engine = lfs_store.get_engine(mode='wb')
110 engine = lfs_store.get_engine(mode='wb')
109 with engine as f:
111 with engine as f:
110 f.write('CONTENT')
112 f.write('CONTENT')
111
113
112 assert os.path.isfile(oid_location)
114 assert os.path.isfile(oid_location)
113
115
114 def test_detect_has_oid(self, lfs_store):
116 def test_detect_has_oid(self, lfs_store):
115
117
116 assert lfs_store.has_oid() is False
118 assert lfs_store.has_oid() is False
117 engine = lfs_store.get_engine(mode='wb')
119 engine = lfs_store.get_engine(mode='wb')
118 with engine as f:
120 with engine as f:
119 f.write('CONTENT')
121 f.write('CONTENT')
120
122
121 assert lfs_store.has_oid() is True No newline at end of file
123 assert lfs_store.has_oid() is True
General Comments 0
You need to be logged in to leave comments. Login now