##// END OF EJS Templates
git-lfs: added vcsserver handling of git-lfs objects....
marcink -
r180:bb8ad7ab default
parent child Browse files
Show More
@@ -0,0 +1,19 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
18
19 from app import create_app
@@ -0,0 +1,276 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
18 import re
19 import logging
20 from wsgiref.util import FileWrapper
21
22 import simplejson as json
23 from pyramid.config import Configurator
24 from pyramid.response import Response, FileIter
25 from pyramid.httpexceptions import (
26 HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden,
27 HTTPUnprocessableEntity)
28
29 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
30 from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator
31 from vcsserver.utils import safe_int
32
33 log = logging.getLogger(__name__)
34
35
36 GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs' #+json ?
37 GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))')
38
39
40 def write_response_error(http_exception, text=None):
41 content_type = 'application/json'
42 _exception = http_exception(content_type=content_type)
43 _exception.content_type = content_type
44 if text:
45 _exception.body = json.dumps({'message': text})
46 log.debug('LFS: writing response of type %s to client with text:%s',
47 http_exception, text)
48 return _exception
49
50
51 class AuthHeaderRequired(object):
52 """
53 Decorator to check if request has proper auth-header
54 """
55
56 def __call__(self, func):
57 return get_cython_compat_decorator(self.__wrapper, func)
58
59 def __wrapper(self, func, *fargs, **fkwargs):
60 request = fargs[1]
61 auth = request.authorization
62 if not auth:
63 return write_response_error(HTTPForbidden)
64 return func(*fargs[1:], **fkwargs)
65
66
67 # views
68
69 def lfs_objects(request):
70 # indicate not supported, V1 API
71 log.warning('LFS: v1 api not supported, reporting it back to client')
72 return write_response_error(HTTPNotImplemented, 'LFS: v1 api not supported')
73
74
75 @AuthHeaderRequired()
76 def lfs_objects_batch(request):
77 """
78 The client sends the following information to the Batch endpoint to transfer some objects:
79
80 operation - Should be download or upload.
81 transfers - An optional Array of String identifiers for transfer
82 adapters that the client has configured. If omitted, the basic
83 transfer adapter MUST be assumed by the server.
84 objects - An Array of objects to download.
85 oid - String OID of the LFS object.
86 size - Integer byte size of the LFS object. Must be at least zero.
87 """
88 auth = request.authorization
89
90 repo = request.matchdict.get('repo')
91
92 data = request.json
93 operation = data.get('operation')
94 if operation not in ('download', 'upload'):
95 log.debug('LFS: unsupported operation:%s', operation)
96 return write_response_error(
97 HTTPBadRequest, 'unsupported operation mode: `%s`' % operation)
98
99 if 'objects' not in data:
100 log.debug('LFS: missing objects data')
101 return write_response_error(
102 HTTPBadRequest, 'missing objects data')
103
104 log.debug('LFS: handling operation of type: %s', operation)
105
106 objects = []
107 for o in data['objects']:
108 try:
109 oid = o['oid']
110 obj_size = o['size']
111 except KeyError:
112 log.exception('LFS, failed to extract data')
113 return write_response_error(
114 HTTPBadRequest, 'unsupported data in objects')
115
116 obj_data = {'oid': oid}
117
118 obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid)
119 obj_verify_href = request.route_url('lfs_objects_verify', repo=repo)
120 store = LFSOidStore(
121 repo, oid, store_location=request.registry.git_lfs_store_path)
122 handler = OidHandler(
123 store, repo, auth, oid, obj_size, obj_data,
124 obj_href, obj_verify_href)
125
126 # this verifies also OIDs
127 actions, errors = handler.exec_operation(operation)
128 if errors:
129 log.warning('LFS: got following errors: %s', errors)
130 obj_data['errors'] = errors
131
132 if actions:
133 obj_data['actions'] = actions
134
135 obj_data['size'] = obj_size
136 obj_data['authenticated'] = True
137 objects.append(obj_data)
138
139 result = {'objects': objects, 'transfer': 'basic'}
140 log.debug('LFS Response %s', safe_result(result))
141
142 return result
143
144
145 def lfs_objects_oid_upload(request):
146 repo = request.matchdict.get('repo')
147 oid = request.matchdict.get('oid')
148 store = LFSOidStore(
149 repo, oid, store_location=request.registry.git_lfs_store_path)
150 engine = store.get_engine(mode='wb')
151 log.debug('LFS: starting chunked write of LFS oid: %s to storage', oid)
152 with engine as f:
153 for chunk in FileWrapper(request.body_file_seekable, blksize=64 * 1024):
154 f.write(chunk)
155
156 return {'upload': 'ok'}
157
158
159 def lfs_objects_oid_download(request):
160 repo = request.matchdict.get('repo')
161 oid = request.matchdict.get('oid')
162
163 store = LFSOidStore(
164 repo, oid, store_location=request.registry.git_lfs_store_path)
165 if not store.has_oid():
166 log.debug('LFS: oid %s does not exists in store', oid)
167 return write_response_error(
168 HTTPNotFound, 'requested file with oid `%s` not found in store' % oid)
169
170 # TODO(marcink): support range header ?
171 # Range: bytes=0-, `bytes=(\d+)\-.*`
172
173 f = open(store.oid_path, 'rb')
174 response = Response(
175 content_type='application/octet-stream', app_iter=FileIter(f))
176 response.headers.add('X-RC-LFS-Response-Oid', str(oid))
177 return response
178
179
180 def lfs_objects_verify(request):
181 repo = request.matchdict.get('repo')
182
183 data = request.json
184 oid = data.get('oid')
185 size = safe_int(data.get('size'))
186
187 if not (oid and size):
188 return write_response_error(
189 HTTPBadRequest, 'missing oid and size in request data')
190
191 store = LFSOidStore(repo, oid,
192 store_location=request.registry.git_lfs_store_path)
193 if not store.has_oid():
194 log.debug('LFS: oid %s does not exists in store', oid)
195 return write_response_error(
196 HTTPNotFound, 'oid `%s` does not exists in store' % oid)
197
198 store_size = store.size_oid()
199 if store_size != size:
200 msg = 'requested file size mismatch store size:%s requested:%s' % (
201 store_size, size)
202 return write_response_error(
203 HTTPUnprocessableEntity, msg)
204
205 return {'message': {'size': 'ok', 'in_store': 'ok'}}
206
207
208 def lfs_objects_lock(request):
209 return write_response_error(
210 HTTPNotImplemented, 'GIT LFS locking api not supported')
211
212
213 def not_found(request):
214 return write_response_error(
215 HTTPNotFound, 'request path not found')
216
217
218 def lfs_disabled(request):
219 return write_response_error(
220 HTTPNotImplemented, 'GIT LFS disabled for this repo')
221
222
223 def git_lfs_app(config):
224
225 # v1 API deprecation endpoint
226 config.add_route('lfs_objects',
227 '/{repo:.*?[^/]}/info/lfs/objects')
228 config.add_view(lfs_objects, route_name='lfs_objects',
229 request_method='POST', renderer='json')
230
231 # locking API
232 config.add_route('lfs_objects_lock',
233 '/{repo:.*?[^/]}/info/lfs/locks')
234 config.add_view(lfs_objects_lock, route_name='lfs_objects_lock',
235 request_method=('POST', 'GET'), renderer='json')
236
237 config.add_route('lfs_objects_lock_verify',
238 '/{repo:.*?[^/]}/info/lfs/locks/verify')
239 config.add_view(lfs_objects_lock, route_name='lfs_objects_lock_verify',
240 request_method=('POST', 'GET'), renderer='json')
241
242 # batch API
243 config.add_route('lfs_objects_batch',
244 '/{repo:.*?[^/]}/info/lfs/objects/batch')
245 config.add_view(lfs_objects_batch, route_name='lfs_objects_batch',
246 request_method='POST', renderer='json')
247
248 # oid upload/download API
249 config.add_route('lfs_objects_oid',
250 '/{repo:.*?[^/]}/info/lfs/objects/{oid}')
251 config.add_view(lfs_objects_oid_upload, route_name='lfs_objects_oid',
252 request_method='PUT', renderer='json')
253 config.add_view(lfs_objects_oid_download, route_name='lfs_objects_oid',
254 request_method='GET', renderer='json')
255
256 # verification API
257 config.add_route('lfs_objects_verify',
258 '/{repo:.*?[^/]}/info/lfs/verify')
259 config.add_view(lfs_objects_verify, route_name='lfs_objects_verify',
260 request_method='POST', renderer='json')
261
262 # not found handler for API
263 config.add_notfound_view(not_found, renderer='json')
264
265
266 def create_app(git_lfs_enabled, git_lfs_store_path):
267 config = Configurator()
268 if git_lfs_enabled:
269 config.include(git_lfs_app)
270 config.registry.git_lfs_store_path = git_lfs_store_path
271 else:
272 # not found handler for API, reporting disabled LFS support
273 config.add_notfound_view(lfs_disabled, renderer='json')
274
275 app = config.make_wsgi_app()
276 return app
@@ -0,0 +1,167 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
18 import os
19 import shutil
20 import logging
21 from collections import OrderedDict
22
23 log = logging.getLogger(__name__)
24
25
26 class OidHandler(object):
27
28 def __init__(self, store, repo_name, auth, oid, obj_size, obj_data, obj_href,
29 obj_verify_href=None):
30 self.current_store = store
31 self.repo_name = repo_name
32 self.auth = auth
33 self.oid = oid
34 self.obj_size = obj_size
35 self.obj_data = obj_data
36 self.obj_href = obj_href
37 self.obj_verify_href = obj_verify_href
38
39 def get_store(self, mode=None):
40 return self.current_store
41
42 def get_auth(self):
43 """returns auth header for re-use in upload/download"""
44 return " ".join(self.auth)
45
46 def download(self):
47
48 store = self.get_store()
49 response = None
50 has_errors = None
51
52 if not store.has_oid():
53 # error reply back to client that something is wrong with dl
54 err_msg = 'object: {} does not exist in store'.format(store.oid)
55 has_errors = OrderedDict(
56 error=OrderedDict(
57 code=404,
58 message=err_msg
59 )
60 )
61
62 download_action = OrderedDict(
63 href=self.obj_href,
64 header=OrderedDict([("Authorization", self.get_auth())])
65 )
66 if not has_errors:
67 response = OrderedDict(download=download_action)
68 return response, has_errors
69
70 def upload(self, skip_existing=True):
71 """
72 Write upload action for git-lfs server
73 """
74
75 store = self.get_store()
76 response = None
77 has_errors = None
78
79 # verify if we have the OID before, if we do, reply with empty
80 if store.has_oid():
81 log.debug('LFS: store already has oid %s', store.oid)
82 if skip_existing:
83 log.debug('LFS: skipping further action as oid is existing')
84 return response, has_errors
85
86 upload_action = OrderedDict(
87 href=self.obj_href,
88 header=OrderedDict([("Authorization", self.get_auth())])
89 )
90 if not has_errors:
91 response = OrderedDict(upload=upload_action)
92 # if specified in handler, return the verification endpoint
93 if self.obj_verify_href:
94 verify_action = OrderedDict(
95 href=self.obj_verify_href,
96 header=OrderedDict([("Authorization", self.get_auth())])
97 )
98 response['verify'] = verify_action
99 return response, has_errors
100
101 def exec_operation(self, operation, *args, **kwargs):
102 handler = getattr(self, operation)
103 log.debug('LFS: handling request using %s handler', handler)
104 return handler(*args, **kwargs)
105
106
107 class LFSOidStore(object):
108
109 def __init__(self, repo, oid, store_location=None):
110 self._store = store_location or self.get_default_store()
111 self.oid = oid
112 self.repo = repo
113 self.store_path = os.path.join(self._store, repo)
114 self.tmp_oid_path = os.path.join(self.store_path, oid + '.tmp')
115 self.oid_path = os.path.join(self.store_path, oid)
116 self.fd = None
117
118 def get_engine(self, mode):
119 """
120 engine = .get_engine(mode='wb')
121 with engine as f:
122 f.write('...')
123 """
124
125 class StoreEngine(object):
126 def __init__(self, mode, store_path, oid_path, tmp_oid_path):
127 self.mode = mode
128 self.store_path = store_path
129 self.oid_path = oid_path
130 self.tmp_oid_path = tmp_oid_path
131
132 def __enter__(self):
133 if not os.path.isdir(self.store_path):
134 os.makedirs(self.store_path)
135
136 # TODO(marcink): maybe write metadata here with size/oid ?
137 fd = open(self.tmp_oid_path, self.mode)
138 self.fd = fd
139 return fd
140
141 def __exit__(self, exc_type, exc_value, traceback):
142 # close tmp file, and rename to final destination
143 self.fd.close()
144 shutil.move(self.tmp_oid_path, self.oid_path)
145
146 return StoreEngine(
147 mode, self.store_path, self.oid_path, self.tmp_oid_path)
148
149 def get_default_store(self):
150 """
151 Default store, consistent with defaults of Mercurial large files store
152 which is /home/username/.cache/largefiles
153 """
154 user_home = os.path.expanduser("~")
155 return os.path.join(user_home, '.cache', 'lfs-store')
156
157 def has_oid(self):
158 return os.path.exists(os.path.join(self.store_path, self.oid))
159
160 def size_oid(self):
161 size = -1
162
163 if self.has_oid():
164 oid = os.path.join(self.store_path, self.oid)
165 size = os.stat(oid).st_size
166
167 return size No newline at end of file
@@ -0,0 +1,16 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@@ -0,0 +1,233 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
18 import os
19 import pytest
20 from webtest.app import TestApp as WebObTestApp
21
22 from vcsserver.git_lfs.app import create_app
23
24
25 @pytest.fixture(scope='function')
26 def git_lfs_app(tmpdir):
27 custom_app = WebObTestApp(create_app(
28 git_lfs_enabled=True, git_lfs_store_path=str(tmpdir)))
29 custom_app._store = str(tmpdir)
30 return custom_app
31
32
33 @pytest.fixture()
34 def http_auth():
35 return {'HTTP_AUTHORIZATION': "Basic XXXXX"}
36
37
38 class TestLFSApplication(object):
39
40 def test_app_wrong_path(self, git_lfs_app):
41 git_lfs_app.get('/repo/info/lfs/xxx', status=404)
42
43 def test_app_deprecated_endpoint(self, git_lfs_app):
44 response = git_lfs_app.post('/repo/info/lfs/objects', status=501)
45 assert response.status_code == 501
46 assert response.json == {u'message': u'LFS: v1 api not supported'}
47
48 def test_app_lock_verify_api_not_available(self, git_lfs_app):
49 response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501)
50 assert response.status_code == 501
51 assert response.json == {
52 u'message': u'GIT LFS locking api not supported'}
53
54 def test_app_lock_api_not_available(self, git_lfs_app):
55 response = git_lfs_app.post('/repo/info/lfs/locks', status=501)
56 assert response.status_code == 501
57 assert response.json == {
58 u'message': u'GIT LFS locking api not supported'}
59
60 def test_app_batch_api_missing_auth(self, git_lfs_app,):
61 git_lfs_app.post_json(
62 '/repo/info/lfs/objects/batch', params={}, status=403)
63
64 def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth):
65 response = git_lfs_app.post_json(
66 '/repo/info/lfs/objects/batch', params={}, status=400,
67 extra_environ=http_auth)
68 assert response.json == {
69 u'message': u'unsupported operation mode: `None`'}
70
71 def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth):
72 response = git_lfs_app.post_json(
73 '/repo/info/lfs/objects/batch', params={'operation': 'download'},
74 status=400, extra_environ=http_auth)
75 assert response.json == {
76 u'message': u'missing objects data'}
77
78 def test_app_batch_api_unsupported_data_in_objects(
79 self, git_lfs_app, http_auth):
80 params = {'operation': 'download',
81 'objects': [{}]}
82 response = git_lfs_app.post_json(
83 '/repo/info/lfs/objects/batch', params=params, status=400,
84 extra_environ=http_auth)
85 assert response.json == {
86 u'message': u'unsupported data in objects'}
87
88 def test_app_batch_api_download_missing_object(
89 self, git_lfs_app, http_auth):
90 params = {'operation': 'download',
91 'objects': [{'oid': '123', 'size': '1024'}]}
92 response = git_lfs_app.post_json(
93 '/repo/info/lfs/objects/batch', params=params,
94 extra_environ=http_auth)
95
96 expected_objects = [
97 {u'authenticated': True,
98 u'errors': {u'error': {
99 u'code': 404,
100 u'message': u'object: 123 does not exist in store'}},
101 u'oid': u'123',
102 u'size': u'1024'}
103 ]
104 assert response.json == {
105 'objects': expected_objects, 'transfer': 'basic'}
106
107 def test_app_batch_api_download(self, git_lfs_app, http_auth):
108 oid = '456'
109 oid_path = os.path.join(git_lfs_app._store, 'repo', oid)
110 os.makedirs(os.path.dirname(oid_path))
111 with open(oid_path, 'wb') as f:
112 f.write('OID_CONTENT')
113
114 params = {'operation': 'download',
115 'objects': [{'oid': oid, 'size': '1024'}]}
116 response = git_lfs_app.post_json(
117 '/repo/info/lfs/objects/batch', params=params,
118 extra_environ=http_auth)
119
120 expected_objects = [
121 {u'authenticated': True,
122 u'actions': {
123 u'download': {
124 u'header': {u'Authorization': u'Basic XXXXX'},
125 u'href': u'http://localhost/repo/info/lfs/objects/456'},
126 },
127 u'oid': u'456',
128 u'size': u'1024'}
129 ]
130 assert response.json == {
131 'objects': expected_objects, 'transfer': 'basic'}
132
133 def test_app_batch_api_upload(self, git_lfs_app, http_auth):
134 params = {'operation': 'upload',
135 'objects': [{'oid': '123', 'size': '1024'}]}
136 response = git_lfs_app.post_json(
137 '/repo/info/lfs/objects/batch', params=params,
138 extra_environ=http_auth)
139 expected_objects = [
140 {u'authenticated': True,
141 u'actions': {
142 u'upload': {
143 u'header': {u'Authorization': u'Basic XXXXX'},
144 u'href': u'http://localhost/repo/info/lfs/objects/123'},
145 u'verify': {
146 u'header': {u'Authorization': u'Basic XXXXX'},
147 u'href': u'http://localhost/repo/info/lfs/verify'}
148 },
149 u'oid': u'123',
150 u'size': u'1024'}
151 ]
152 assert response.json == {
153 'objects': expected_objects, 'transfer': 'basic'}
154
155 def test_app_verify_api_missing_data(self, git_lfs_app):
156 params = {'oid': 'missing',}
157 response = git_lfs_app.post_json(
158 '/repo/info/lfs/verify', params=params,
159 status=400)
160
161 assert response.json == {
162 u'message': u'missing oid and size in request data'}
163
164 def test_app_verify_api_missing_obj(self, git_lfs_app):
165 params = {'oid': 'missing', 'size': '1024'}
166 response = git_lfs_app.post_json(
167 '/repo/info/lfs/verify', params=params,
168 status=404)
169
170 assert response.json == {
171 u'message': u'oid `missing` does not exists in store'}
172
173 def test_app_verify_api_size_mismatch(self, git_lfs_app):
174 oid = 'existing'
175 oid_path = os.path.join(git_lfs_app._store, 'repo', oid)
176 os.makedirs(os.path.dirname(oid_path))
177 with open(oid_path, 'wb') as f:
178 f.write('OID_CONTENT')
179
180 params = {'oid': oid, 'size': '1024'}
181 response = git_lfs_app.post_json(
182 '/repo/info/lfs/verify', params=params, status=422)
183
184 assert response.json == {
185 u'message': u'requested file size mismatch '
186 u'store size:11 requested:1024'}
187
188 def test_app_verify_api(self, git_lfs_app):
189 oid = 'existing'
190 oid_path = os.path.join(git_lfs_app._store, 'repo', oid)
191 os.makedirs(os.path.dirname(oid_path))
192 with open(oid_path, 'wb') as f:
193 f.write('OID_CONTENT')
194
195 params = {'oid': oid, 'size': 11}
196 response = git_lfs_app.post_json(
197 '/repo/info/lfs/verify', params=params)
198
199 assert response.json == {
200 u'message': {u'size': u'ok', u'in_store': u'ok'}}
201
202 def test_app_download_api_oid_not_existing(self, git_lfs_app):
203 oid = 'missing'
204
205 response = git_lfs_app.get(
206 '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404)
207
208 assert response.json == {
209 u'message': u'requested file with oid `missing` not found in store'}
210
211 def test_app_download_api(self, git_lfs_app):
212 oid = 'existing'
213 oid_path = os.path.join(git_lfs_app._store, 'repo', oid)
214 os.makedirs(os.path.dirname(oid_path))
215 with open(oid_path, 'wb') as f:
216 f.write('OID_CONTENT')
217
218 response = git_lfs_app.get(
219 '/repo/info/lfs/objects/{oid}'.format(oid=oid))
220 assert response
221
222 def test_app_upload(self, git_lfs_app):
223 oid = 'uploaded'
224
225 response = git_lfs_app.put(
226 '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT')
227
228 assert response.json == {u'upload': u'ok'}
229
230 # verify that we actually wrote that OID
231 oid_path = os.path.join(git_lfs_app._store, 'repo', oid)
232 assert os.path.isfile(oid_path)
233 assert 'CONTENT' == open(oid_path).read()
@@ -0,0 +1,121 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
18 import os
19 import pytest
20 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
21
22
23 @pytest.fixture()
24 def lfs_store(tmpdir):
25 repo = 'test'
26 oid = '123456789'
27 store = LFSOidStore(repo=repo, oid=oid, store_location=str(tmpdir))
28 return store
29
30
31 @pytest.fixture()
32 def oid_handler(lfs_store):
33 store = lfs_store
34 repo = store.repo
35 oid = store.oid
36
37 oid_handler = OidHandler(
38 store=store, repo_name=repo, auth=('basic', 'xxxx'),
39 oid=oid,
40 obj_size='1024', obj_data={}, obj_href='http://localhost/handle_oid',
41 obj_verify_href='http://localhost/verify')
42 return oid_handler
43
44
45 class TestOidHandler(object):
46
47 @pytest.mark.parametrize('exec_action', [
48 'download',
49 'upload',
50 ])
51 def test_exec_action(self, exec_action, oid_handler):
52 handler = oid_handler.exec_operation(exec_action)
53 assert handler
54
55 def test_exec_action_undefined(self, oid_handler):
56 with pytest.raises(AttributeError):
57 oid_handler.exec_operation('wrong')
58
59 def test_download_oid_not_existing(self, oid_handler):
60 response, has_errors = oid_handler.exec_operation('download')
61
62 assert response is None
63 assert has_errors['error'] == {
64 'code': 404,
65 'message': 'object: 123456789 does not exist in store'}
66
67 def test_download_oid(self, oid_handler):
68 store = oid_handler.get_store()
69
70 os.makedirs(os.path.dirname(store.oid_path))
71 with open(store.oid_path, 'wb') as f:
72 f.write('CONTENT')
73
74 response, has_errors = oid_handler.exec_operation('download')
75
76 assert has_errors is None
77 assert response['download'] == {
78 'header': {'Authorization': 'basic xxxx'},
79 'href': 'http://localhost/handle_oid'
80 }
81
82 def test_upload_oid_that_exists(self, oid_handler):
83 store = oid_handler.get_store()
84
85 os.makedirs(os.path.dirname(store.oid_path))
86 with open(store.oid_path, 'wb') as f:
87 f.write('CONTENT')
88
89 response, has_errors = oid_handler.exec_operation('upload')
90 assert has_errors is None
91 assert response is None
92
93 def test_upload_oid(self, oid_handler):
94 response, has_errors = oid_handler.exec_operation('upload')
95 assert has_errors is None
96 assert response['upload'] == {
97 'header': {'Authorization': 'basic xxxx'},
98 'href': 'http://localhost/handle_oid'
99 }
100
101
102 class TestLFSStore(object):
103 def test_write_oid(self, lfs_store):
104 oid_location = lfs_store.oid_path
105
106 assert not os.path.isfile(oid_location)
107
108 engine = lfs_store.get_engine(mode='wb')
109 with engine as f:
110 f.write('CONTENT')
111
112 assert os.path.isfile(oid_location)
113
114 def test_detect_has_oid(self, lfs_store):
115
116 assert lfs_store.has_oid() is False
117 engine = lfs_store.get_engine(mode='wb')
118 with engine as f:
119 f.write('CONTENT')
120
121 assert lfs_store.has_oid() is True No newline at end of file
@@ -0,0 +1,50 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 import copy
18 from functools import wraps
19
20
21 def get_cython_compat_decorator(wrapper, func):
22 """
23 Creates a cython compatible decorator. The previously used
24 decorator.decorator() function seems to be incompatible with cython.
25
26 :param wrapper: __wrapper method of the decorator class
27 :param func: decorated function
28 """
29 @wraps(func)
30 def local_wrapper(*args, **kwds):
31 return wrapper(func, *args, **kwds)
32 local_wrapper.__wrapped__ = func
33 return local_wrapper
34
35
36 def safe_result(result):
37 """clean result for better representation in logs"""
38 clean_copy = copy.deepcopy(result)
39
40 try:
41 if 'objects' in clean_copy:
42 for oid_data in clean_copy['objects']:
43 if 'actions' in oid_data:
44 for action_name, data in oid_data['actions'].items():
45 if 'header' in data:
46 data['header'] = {'Authorization': '*****'}
47 except Exception:
48 return result
49
50 return clean_copy
@@ -30,6 +30,7 b' from pyramid.config import Configurator'
30 from pyramid.wsgi import wsgiapp
30 from pyramid.wsgi import wsgiapp
31
31
32 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
32 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
33 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
33 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
34 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
34 from vcsserver.echo_stub.echo_app import EchoApp
35 from vcsserver.echo_stub.echo_app import EchoApp
35 from vcsserver.exceptions import HTTPRepoLocked
36 from vcsserver.exceptions import HTTPRepoLocked
@@ -40,11 +41,13 b' try:'
40 except ImportError:
41 except ImportError:
41 GitFactory = None
42 GitFactory = None
42 GitRemote = None
43 GitRemote = None
44
43 try:
45 try:
44 from vcsserver.hg import MercurialFactory, HgRemote
46 from vcsserver.hg import MercurialFactory, HgRemote
45 except ImportError:
47 except ImportError:
46 MercurialFactory = None
48 MercurialFactory = None
47 HgRemote = None
49 HgRemote = None
50
48 try:
51 try:
49 from vcsserver.svn import SubversionFactory, SvnRemote
52 from vcsserver.svn import SubversionFactory, SvnRemote
50 except ImportError:
53 except ImportError:
@@ -362,9 +365,31 b' class HTTPApplication(object):'
362 config = msgpack.unpackb(packed_config)
365 config = msgpack.unpackb(packed_config)
363
366
364 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
367 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
368 content_type = environ.get('CONTENT_TYPE', '')
369
370 path = environ['PATH_INFO']
371 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
372 log.debug(
373 'LFS: Detecting if request `%s` is LFS server path based '
374 'on content type:`%s`, is_lfs:%s',
375 path, content_type, is_lfs_request)
376
377 if not is_lfs_request:
378 # fallback detection by path
379 if GIT_LFS_PROTO_PAT.match(path):
380 is_lfs_request = True
381 log.debug(
382 'LFS: fallback detection by path of: `%s`, is_lfs:%s',
383 path, is_lfs_request)
384
385 if is_lfs_request:
386 app = scm_app.create_git_lfs_wsgi_app(
387 repo_path, repo_name, config)
388 else:
365 app = scm_app.create_git_wsgi_app(
389 app = scm_app.create_git_wsgi_app(
366 repo_path, repo_name, config)
390 repo_path, repo_name, config)
367 return app(environ, start_response)
391 return app(environ, start_response)
392
368 return _git_stream
393 return _git_stream
369
394
370 def is_vcs_view(self, context, request):
395 def is_vcs_view(self, context, request):
@@ -15,8 +15,8 b''
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import logging
19 import logging
19 import os
20
20
21 import mercurial
21 import mercurial
22 import mercurial.error
22 import mercurial.error
@@ -25,7 +25,7 b' import mercurial.hgweb.hgweb_mod'
25 import mercurial.hgweb.protocol
25 import mercurial.hgweb.protocol
26 import webob.exc
26 import webob.exc
27
27
28 from vcsserver import pygrack, exceptions, settings
28 from vcsserver import pygrack, exceptions, settings, git_lfs
29
29
30
30
31 log = logging.getLogger(__name__)
31 log = logging.getLogger(__name__)
@@ -132,6 +132,9 b' def create_hg_wsgi_app(repo_path, repo_n'
132
132
133
133
134 class GitHandler(object):
134 class GitHandler(object):
135 """
136 Handler for Git operations like push/pull etc
137 """
135 def __init__(self, repo_location, repo_name, git_path, update_server_info,
138 def __init__(self, repo_location, repo_name, git_path, update_server_info,
136 extras):
139 extras):
137 if not os.path.isdir(repo_location):
140 if not os.path.isdir(repo_location):
@@ -172,3 +175,35 b' def create_git_wsgi_app(repo_path, repo_'
172 repo_path, repo_name, git_path, update_server_info, config)
175 repo_path, repo_name, git_path, update_server_info, config)
173
176
174 return app
177 return app
178
179
180 class GitLFSHandler(object):
181 """
182 Handler for Git LFS operations
183 """
184
185 def __init__(self, repo_location, repo_name, git_path, update_server_info,
186 extras):
187 if not os.path.isdir(repo_location):
188 raise OSError(repo_location)
189 self.content_path = repo_location
190 self.repo_name = repo_name
191 self.repo_location = repo_location
192 self.extras = extras
193 self.git_path = git_path
194 self.update_server_info = update_server_info
195
196 def get_app(self, git_lfs_enabled, git_lfs_store_path):
197 app = git_lfs.create_app(git_lfs_enabled, git_lfs_store_path)
198 return app
199
200
201 def create_git_lfs_wsgi_app(repo_path, repo_name, config):
202 git_path = settings.GIT_EXECUTABLE
203 update_server_info = config.pop('git_update_server_info')
204 git_lfs_enabled = config.pop('git_lfs_enabled')
205 git_lfs_store_path = config.pop('git_lfs_store_path')
206 app = GitLFSHandler(
207 repo_path, repo_name, git_path, update_server_info, config)
208
209 return app.get_app(git_lfs_enabled, git_lfs_store_path)
@@ -16,8 +16,23 b''
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18
18
19 def safe_int(val, default=None):
20 """
21 Returns int() of val if val is not convertable to int use default
22 instead
19
23
20 # TODO: johbo: That's a copy from rhodecode
24 :param val:
25 :param default:
26 """
27
28 try:
29 val = int(val)
30 except (ValueError, TypeError):
31 val = default
32
33 return val
34
35
21 def safe_str(unicode_, to_encoding=['utf8']):
36 def safe_str(unicode_, to_encoding=['utf8']):
22 """
37 """
23 safe str function. Does few trick to turn unicode_ into string
38 safe str function. Does few trick to turn unicode_ into string
General Comments 0
You need to be logged in to leave comments. Login now