Show More
@@ -1,287 +1,292 b'' | |||||
1 | # RhodeCode VCSServer provides access to different vcs backends via network. |
|
1 | # RhodeCode VCSServer provides access to different vcs backends via network. | |
2 | # Copyright (C) 2014-2019 RhodeCode GmbH |
|
2 | # Copyright (C) 2014-2019 RhodeCode GmbH | |
3 | # |
|
3 | # | |
4 | # This program is free software; you can redistribute it and/or modify |
|
4 | # This program is free software; you can redistribute it and/or modify | |
5 | # it under the terms of the GNU General Public License as published by |
|
5 | # it under the terms of the GNU General Public License as published by | |
6 | # the Free Software Foundation; either version 3 of the License, or |
|
6 | # the Free Software Foundation; either version 3 of the License, or | |
7 | # (at your option) any later version. |
|
7 | # (at your option) any later version. | |
8 | # |
|
8 | # | |
9 | # This program is distributed in the hope that it will be useful, |
|
9 | # This program is distributed in the hope that it will be useful, | |
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | # GNU General Public License for more details. |
|
12 | # GNU General Public License for more details. | |
13 | # |
|
13 | # | |
14 | # You should have received a copy of the GNU General Public License |
|
14 | # You should have received a copy of the GNU General Public License | |
15 | # along with this program; if not, write to the Free Software Foundation, |
|
15 | # along with this program; if not, write to the Free Software Foundation, | |
16 | # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
|
16 | # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | |
17 |
|
17 | |||
18 | import re |
|
18 | import re | |
19 | import logging |
|
19 | import logging | |
20 | from wsgiref.util import FileWrapper |
|
20 | from wsgiref.util import FileWrapper | |
21 |
|
21 | |||
22 | import simplejson as json |
|
22 | import simplejson as json | |
23 | from pyramid.config import Configurator |
|
23 | from pyramid.config import Configurator | |
24 | from pyramid.response import Response, FileIter |
|
24 | from pyramid.response import Response, FileIter | |
25 | from pyramid.httpexceptions import ( |
|
25 | from pyramid.httpexceptions import ( | |
26 | HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden, |
|
26 | HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden, | |
27 | HTTPUnprocessableEntity) |
|
27 | HTTPUnprocessableEntity) | |
28 |
|
28 | |||
29 | from vcsserver.git_lfs.lib import OidHandler, LFSOidStore |
|
29 | from vcsserver.git_lfs.lib import OidHandler, LFSOidStore | |
30 | from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator |
|
30 | from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator | |
31 | from vcsserver.utils import safe_int |
|
31 | from vcsserver.utils import safe_int | |
32 |
|
32 | |||
33 | log = logging.getLogger(__name__) |
|
33 | log = logging.getLogger(__name__) | |
34 |
|
34 | |||
35 |
|
35 | |||
36 | GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs' #+json ? |
|
36 | GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs' #+json ? | |
37 | GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))') |
|
37 | GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))') | |
38 |
|
38 | |||
39 |
|
39 | |||
40 | def write_response_error(http_exception, text=None): |
|
40 | def write_response_error(http_exception, text=None): | |
41 | content_type = GIT_LFS_CONTENT_TYPE + '+json' |
|
41 | content_type = GIT_LFS_CONTENT_TYPE + '+json' | |
42 | _exception = http_exception(content_type=content_type) |
|
42 | _exception = http_exception(content_type=content_type) | |
43 | _exception.content_type = content_type |
|
43 | _exception.content_type = content_type | |
44 | if text: |
|
44 | if text: | |
45 | _exception.body = json.dumps({'message': text}) |
|
45 | _exception.body = json.dumps({'message': text}) | |
46 | log.debug('LFS: writing response of type %s to client with text:%s', |
|
46 | log.debug('LFS: writing response of type %s to client with text:%s', | |
47 | http_exception, text) |
|
47 | http_exception, text) | |
48 | return _exception |
|
48 | return _exception | |
49 |
|
49 | |||
50 |
|
50 | |||
51 | class AuthHeaderRequired(object): |
|
51 | class AuthHeaderRequired(object): | |
52 | """ |
|
52 | """ | |
53 | Decorator to check if request has proper auth-header |
|
53 | Decorator to check if request has proper auth-header | |
54 | """ |
|
54 | """ | |
55 |
|
55 | |||
56 | def __call__(self, func): |
|
56 | def __call__(self, func): | |
57 | return get_cython_compat_decorator(self.__wrapper, func) |
|
57 | return get_cython_compat_decorator(self.__wrapper, func) | |
58 |
|
58 | |||
59 | def __wrapper(self, func, *fargs, **fkwargs): |
|
59 | def __wrapper(self, func, *fargs, **fkwargs): | |
60 | request = fargs[1] |
|
60 | request = fargs[1] | |
61 | auth = request.authorization |
|
61 | auth = request.authorization | |
62 | if not auth: |
|
62 | if not auth: | |
63 | return write_response_error(HTTPForbidden) |
|
63 | return write_response_error(HTTPForbidden) | |
64 | return func(*fargs[1:], **fkwargs) |
|
64 | return func(*fargs[1:], **fkwargs) | |
65 |
|
65 | |||
66 |
|
66 | |||
67 | # views |
|
67 | # views | |
68 |
|
68 | |||
69 | def lfs_objects(request): |
|
69 | def lfs_objects(request): | |
70 | # indicate not supported, V1 API |
|
70 | # indicate not supported, V1 API | |
71 | log.warning('LFS: v1 api not supported, reporting it back to client') |
|
71 | log.warning('LFS: v1 api not supported, reporting it back to client') | |
72 | return write_response_error(HTTPNotImplemented, 'LFS: v1 api not supported') |
|
72 | return write_response_error(HTTPNotImplemented, 'LFS: v1 api not supported') | |
73 |
|
73 | |||
74 |
|
74 | |||
75 | @AuthHeaderRequired() |
|
75 | @AuthHeaderRequired() | |
76 | def lfs_objects_batch(request): |
|
76 | def lfs_objects_batch(request): | |
77 | """ |
|
77 | """ | |
78 | The client sends the following information to the Batch endpoint to transfer some objects: |
|
78 | The client sends the following information to the Batch endpoint to transfer some objects: | |
79 |
|
79 | |||
80 | operation - Should be download or upload. |
|
80 | operation - Should be download or upload. | |
81 | transfers - An optional Array of String identifiers for transfer |
|
81 | transfers - An optional Array of String identifiers for transfer | |
82 | adapters that the client has configured. If omitted, the basic |
|
82 | adapters that the client has configured. If omitted, the basic | |
83 | transfer adapter MUST be assumed by the server. |
|
83 | transfer adapter MUST be assumed by the server. | |
84 | objects - An Array of objects to download. |
|
84 | objects - An Array of objects to download. | |
85 | oid - String OID of the LFS object. |
|
85 | oid - String OID of the LFS object. | |
86 | size - Integer byte size of the LFS object. Must be at least zero. |
|
86 | size - Integer byte size of the LFS object. Must be at least zero. | |
87 | """ |
|
87 | """ | |
88 | request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json' |
|
88 | request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json' | |
89 | auth = request.authorization |
|
89 | auth = request.authorization | |
90 | repo = request.matchdict.get('repo') |
|
90 | repo = request.matchdict.get('repo') | |
91 | data = request.json |
|
91 | data = request.json | |
92 | operation = data.get('operation') |
|
92 | operation = data.get('operation') | |
|
93 | http_scheme = request.registry.git_lfs_http_scheme | |||
|
94 | ||||
93 | if operation not in ('download', 'upload'): |
|
95 | if operation not in ('download', 'upload'): | |
94 | log.debug('LFS: unsupported operation:%s', operation) |
|
96 | log.debug('LFS: unsupported operation:%s', operation) | |
95 | return write_response_error( |
|
97 | return write_response_error( | |
96 | HTTPBadRequest, 'unsupported operation mode: `%s`' % operation) |
|
98 | HTTPBadRequest, 'unsupported operation mode: `%s`' % operation) | |
97 |
|
99 | |||
98 | if 'objects' not in data: |
|
100 | if 'objects' not in data: | |
99 | log.debug('LFS: missing objects data') |
|
101 | log.debug('LFS: missing objects data') | |
100 | return write_response_error( |
|
102 | return write_response_error( | |
101 | HTTPBadRequest, 'missing objects data') |
|
103 | HTTPBadRequest, 'missing objects data') | |
102 |
|
104 | |||
103 | log.debug('LFS: handling operation of type: %s', operation) |
|
105 | log.debug('LFS: handling operation of type: %s', operation) | |
104 |
|
106 | |||
105 | objects = [] |
|
107 | objects = [] | |
106 | for o in data['objects']: |
|
108 | for o in data['objects']: | |
107 | try: |
|
109 | try: | |
108 | oid = o['oid'] |
|
110 | oid = o['oid'] | |
109 | obj_size = o['size'] |
|
111 | obj_size = o['size'] | |
110 | except KeyError: |
|
112 | except KeyError: | |
111 | log.exception('LFS, failed to extract data') |
|
113 | log.exception('LFS, failed to extract data') | |
112 | return write_response_error( |
|
114 | return write_response_error( | |
113 | HTTPBadRequest, 'unsupported data in objects') |
|
115 | HTTPBadRequest, 'unsupported data in objects') | |
114 |
|
116 | |||
115 | obj_data = {'oid': oid} |
|
117 | obj_data = {'oid': oid} | |
116 |
|
118 | |||
117 |
obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid |
|
119 | obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid, | |
118 | obj_verify_href = request.route_url('lfs_objects_verify', repo=repo) |
|
120 | _scheme=http_scheme) | |
|
121 | obj_verify_href = request.route_url('lfs_objects_verify', repo=repo, | |||
|
122 | _scheme=http_scheme) | |||
119 | store = LFSOidStore( |
|
123 | store = LFSOidStore( | |
120 | oid, repo, store_location=request.registry.git_lfs_store_path) |
|
124 | oid, repo, store_location=request.registry.git_lfs_store_path) | |
121 | handler = OidHandler( |
|
125 | handler = OidHandler( | |
122 | store, repo, auth, oid, obj_size, obj_data, |
|
126 | store, repo, auth, oid, obj_size, obj_data, | |
123 | obj_href, obj_verify_href) |
|
127 | obj_href, obj_verify_href) | |
124 |
|
128 | |||
125 | # this verifies also OIDs |
|
129 | # this verifies also OIDs | |
126 | actions, errors = handler.exec_operation(operation) |
|
130 | actions, errors = handler.exec_operation(operation) | |
127 | if errors: |
|
131 | if errors: | |
128 | log.warning('LFS: got following errors: %s', errors) |
|
132 | log.warning('LFS: got following errors: %s', errors) | |
129 | obj_data['errors'] = errors |
|
133 | obj_data['errors'] = errors | |
130 |
|
134 | |||
131 | if actions: |
|
135 | if actions: | |
132 | obj_data['actions'] = actions |
|
136 | obj_data['actions'] = actions | |
133 |
|
137 | |||
134 | obj_data['size'] = obj_size |
|
138 | obj_data['size'] = obj_size | |
135 | obj_data['authenticated'] = True |
|
139 | obj_data['authenticated'] = True | |
136 | objects.append(obj_data) |
|
140 | objects.append(obj_data) | |
137 |
|
141 | |||
138 | result = {'objects': objects, 'transfer': 'basic'} |
|
142 | result = {'objects': objects, 'transfer': 'basic'} | |
139 | log.debug('LFS Response %s', safe_result(result)) |
|
143 | log.debug('LFS Response %s', safe_result(result)) | |
140 |
|
144 | |||
141 | return result |
|
145 | return result | |
142 |
|
146 | |||
143 |
|
147 | |||
144 | def lfs_objects_oid_upload(request): |
|
148 | def lfs_objects_oid_upload(request): | |
145 | request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json' |
|
149 | request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json' | |
146 | repo = request.matchdict.get('repo') |
|
150 | repo = request.matchdict.get('repo') | |
147 | oid = request.matchdict.get('oid') |
|
151 | oid = request.matchdict.get('oid') | |
148 | store = LFSOidStore( |
|
152 | store = LFSOidStore( | |
149 | oid, repo, store_location=request.registry.git_lfs_store_path) |
|
153 | oid, repo, store_location=request.registry.git_lfs_store_path) | |
150 | engine = store.get_engine(mode='wb') |
|
154 | engine = store.get_engine(mode='wb') | |
151 | log.debug('LFS: starting chunked write of LFS oid: %s to storage', oid) |
|
155 | log.debug('LFS: starting chunked write of LFS oid: %s to storage', oid) | |
152 |
|
156 | |||
153 | body = request.environ['wsgi.input'] |
|
157 | body = request.environ['wsgi.input'] | |
154 |
|
158 | |||
155 | with engine as f: |
|
159 | with engine as f: | |
156 | blksize = 64 * 1024 # 64kb |
|
160 | blksize = 64 * 1024 # 64kb | |
157 | while True: |
|
161 | while True: | |
158 | # read in chunks as stream comes in from Gunicorn |
|
162 | # read in chunks as stream comes in from Gunicorn | |
159 | # this is a specific Gunicorn support function. |
|
163 | # this is a specific Gunicorn support function. | |
160 | # might work differently on waitress |
|
164 | # might work differently on waitress | |
161 | chunk = body.read(blksize) |
|
165 | chunk = body.read(blksize) | |
162 | if not chunk: |
|
166 | if not chunk: | |
163 | break |
|
167 | break | |
164 | f.write(chunk) |
|
168 | f.write(chunk) | |
165 |
|
169 | |||
166 | return {'upload': 'ok'} |
|
170 | return {'upload': 'ok'} | |
167 |
|
171 | |||
168 |
|
172 | |||
169 | def lfs_objects_oid_download(request): |
|
173 | def lfs_objects_oid_download(request): | |
170 | repo = request.matchdict.get('repo') |
|
174 | repo = request.matchdict.get('repo') | |
171 | oid = request.matchdict.get('oid') |
|
175 | oid = request.matchdict.get('oid') | |
172 |
|
176 | |||
173 | store = LFSOidStore( |
|
177 | store = LFSOidStore( | |
174 | oid, repo, store_location=request.registry.git_lfs_store_path) |
|
178 | oid, repo, store_location=request.registry.git_lfs_store_path) | |
175 | if not store.has_oid(): |
|
179 | if not store.has_oid(): | |
176 | log.debug('LFS: oid %s does not exists in store', oid) |
|
180 | log.debug('LFS: oid %s does not exists in store', oid) | |
177 | return write_response_error( |
|
181 | return write_response_error( | |
178 | HTTPNotFound, 'requested file with oid `%s` not found in store' % oid) |
|
182 | HTTPNotFound, 'requested file with oid `%s` not found in store' % oid) | |
179 |
|
183 | |||
180 | # TODO(marcink): support range header ? |
|
184 | # TODO(marcink): support range header ? | |
181 | # Range: bytes=0-, `bytes=(\d+)\-.*` |
|
185 | # Range: bytes=0-, `bytes=(\d+)\-.*` | |
182 |
|
186 | |||
183 | f = open(store.oid_path, 'rb') |
|
187 | f = open(store.oid_path, 'rb') | |
184 | response = Response( |
|
188 | response = Response( | |
185 | content_type='application/octet-stream', app_iter=FileIter(f)) |
|
189 | content_type='application/octet-stream', app_iter=FileIter(f)) | |
186 | response.headers.add('X-RC-LFS-Response-Oid', str(oid)) |
|
190 | response.headers.add('X-RC-LFS-Response-Oid', str(oid)) | |
187 | return response |
|
191 | return response | |
188 |
|
192 | |||
189 |
|
193 | |||
190 | def lfs_objects_verify(request): |
|
194 | def lfs_objects_verify(request): | |
191 | request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json' |
|
195 | request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json' | |
192 | repo = request.matchdict.get('repo') |
|
196 | repo = request.matchdict.get('repo') | |
193 |
|
197 | |||
194 | data = request.json |
|
198 | data = request.json | |
195 | oid = data.get('oid') |
|
199 | oid = data.get('oid') | |
196 | size = safe_int(data.get('size')) |
|
200 | size = safe_int(data.get('size')) | |
197 |
|
201 | |||
198 | if not (oid and size): |
|
202 | if not (oid and size): | |
199 | return write_response_error( |
|
203 | return write_response_error( | |
200 | HTTPBadRequest, 'missing oid and size in request data') |
|
204 | HTTPBadRequest, 'missing oid and size in request data') | |
201 |
|
205 | |||
202 | store = LFSOidStore( |
|
206 | store = LFSOidStore( | |
203 | oid, repo, store_location=request.registry.git_lfs_store_path) |
|
207 | oid, repo, store_location=request.registry.git_lfs_store_path) | |
204 | if not store.has_oid(): |
|
208 | if not store.has_oid(): | |
205 | log.debug('LFS: oid %s does not exists in store', oid) |
|
209 | log.debug('LFS: oid %s does not exists in store', oid) | |
206 | return write_response_error( |
|
210 | return write_response_error( | |
207 | HTTPNotFound, 'oid `%s` does not exists in store' % oid) |
|
211 | HTTPNotFound, 'oid `%s` does not exists in store' % oid) | |
208 |
|
212 | |||
209 | store_size = store.size_oid() |
|
213 | store_size = store.size_oid() | |
210 | if store_size != size: |
|
214 | if store_size != size: | |
211 | msg = 'requested file size mismatch store size:%s requested:%s' % ( |
|
215 | msg = 'requested file size mismatch store size:%s requested:%s' % ( | |
212 | store_size, size) |
|
216 | store_size, size) | |
213 | return write_response_error( |
|
217 | return write_response_error( | |
214 | HTTPUnprocessableEntity, msg) |
|
218 | HTTPUnprocessableEntity, msg) | |
215 |
|
219 | |||
216 | return {'message': {'size': 'ok', 'in_store': 'ok'}} |
|
220 | return {'message': {'size': 'ok', 'in_store': 'ok'}} | |
217 |
|
221 | |||
218 |
|
222 | |||
219 | def lfs_objects_lock(request): |
|
223 | def lfs_objects_lock(request): | |
220 | return write_response_error( |
|
224 | return write_response_error( | |
221 | HTTPNotImplemented, 'GIT LFS locking api not supported') |
|
225 | HTTPNotImplemented, 'GIT LFS locking api not supported') | |
222 |
|
226 | |||
223 |
|
227 | |||
224 | def not_found(request): |
|
228 | def not_found(request): | |
225 | return write_response_error( |
|
229 | return write_response_error( | |
226 | HTTPNotFound, 'request path not found') |
|
230 | HTTPNotFound, 'request path not found') | |
227 |
|
231 | |||
228 |
|
232 | |||
229 | def lfs_disabled(request): |
|
233 | def lfs_disabled(request): | |
230 | return write_response_error( |
|
234 | return write_response_error( | |
231 | HTTPNotImplemented, 'GIT LFS disabled for this repo') |
|
235 | HTTPNotImplemented, 'GIT LFS disabled for this repo') | |
232 |
|
236 | |||
233 |
|
237 | |||
234 | def git_lfs_app(config): |
|
238 | def git_lfs_app(config): | |
235 |
|
239 | |||
236 | # v1 API deprecation endpoint |
|
240 | # v1 API deprecation endpoint | |
237 | config.add_route('lfs_objects', |
|
241 | config.add_route('lfs_objects', | |
238 | '/{repo:.*?[^/]}/info/lfs/objects') |
|
242 | '/{repo:.*?[^/]}/info/lfs/objects') | |
239 | config.add_view(lfs_objects, route_name='lfs_objects', |
|
243 | config.add_view(lfs_objects, route_name='lfs_objects', | |
240 | request_method='POST', renderer='json') |
|
244 | request_method='POST', renderer='json') | |
241 |
|
245 | |||
242 | # locking API |
|
246 | # locking API | |
243 | config.add_route('lfs_objects_lock', |
|
247 | config.add_route('lfs_objects_lock', | |
244 | '/{repo:.*?[^/]}/info/lfs/locks') |
|
248 | '/{repo:.*?[^/]}/info/lfs/locks') | |
245 | config.add_view(lfs_objects_lock, route_name='lfs_objects_lock', |
|
249 | config.add_view(lfs_objects_lock, route_name='lfs_objects_lock', | |
246 | request_method=('POST', 'GET'), renderer='json') |
|
250 | request_method=('POST', 'GET'), renderer='json') | |
247 |
|
251 | |||
248 | config.add_route('lfs_objects_lock_verify', |
|
252 | config.add_route('lfs_objects_lock_verify', | |
249 | '/{repo:.*?[^/]}/info/lfs/locks/verify') |
|
253 | '/{repo:.*?[^/]}/info/lfs/locks/verify') | |
250 | config.add_view(lfs_objects_lock, route_name='lfs_objects_lock_verify', |
|
254 | config.add_view(lfs_objects_lock, route_name='lfs_objects_lock_verify', | |
251 | request_method=('POST', 'GET'), renderer='json') |
|
255 | request_method=('POST', 'GET'), renderer='json') | |
252 |
|
256 | |||
253 | # batch API |
|
257 | # batch API | |
254 | config.add_route('lfs_objects_batch', |
|
258 | config.add_route('lfs_objects_batch', | |
255 | '/{repo:.*?[^/]}/info/lfs/objects/batch') |
|
259 | '/{repo:.*?[^/]}/info/lfs/objects/batch') | |
256 | config.add_view(lfs_objects_batch, route_name='lfs_objects_batch', |
|
260 | config.add_view(lfs_objects_batch, route_name='lfs_objects_batch', | |
257 | request_method='POST', renderer='json') |
|
261 | request_method='POST', renderer='json') | |
258 |
|
262 | |||
259 | # oid upload/download API |
|
263 | # oid upload/download API | |
260 | config.add_route('lfs_objects_oid', |
|
264 | config.add_route('lfs_objects_oid', | |
261 | '/{repo:.*?[^/]}/info/lfs/objects/{oid}') |
|
265 | '/{repo:.*?[^/]}/info/lfs/objects/{oid}') | |
262 | config.add_view(lfs_objects_oid_upload, route_name='lfs_objects_oid', |
|
266 | config.add_view(lfs_objects_oid_upload, route_name='lfs_objects_oid', | |
263 | request_method='PUT', renderer='json') |
|
267 | request_method='PUT', renderer='json') | |
264 | config.add_view(lfs_objects_oid_download, route_name='lfs_objects_oid', |
|
268 | config.add_view(lfs_objects_oid_download, route_name='lfs_objects_oid', | |
265 | request_method='GET', renderer='json') |
|
269 | request_method='GET', renderer='json') | |
266 |
|
270 | |||
267 | # verification API |
|
271 | # verification API | |
268 | config.add_route('lfs_objects_verify', |
|
272 | config.add_route('lfs_objects_verify', | |
269 | '/{repo:.*?[^/]}/info/lfs/verify') |
|
273 | '/{repo:.*?[^/]}/info/lfs/verify') | |
270 | config.add_view(lfs_objects_verify, route_name='lfs_objects_verify', |
|
274 | config.add_view(lfs_objects_verify, route_name='lfs_objects_verify', | |
271 | request_method='POST', renderer='json') |
|
275 | request_method='POST', renderer='json') | |
272 |
|
276 | |||
273 | # not found handler for API |
|
277 | # not found handler for API | |
274 | config.add_notfound_view(not_found, renderer='json') |
|
278 | config.add_notfound_view(not_found, renderer='json') | |
275 |
|
279 | |||
276 |
|
280 | |||
277 | def create_app(git_lfs_enabled, git_lfs_store_path): |
|
281 | def create_app(git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme): | |
278 | config = Configurator() |
|
282 | config = Configurator() | |
279 | if git_lfs_enabled: |
|
283 | if git_lfs_enabled: | |
280 | config.include(git_lfs_app) |
|
284 | config.include(git_lfs_app) | |
281 | config.registry.git_lfs_store_path = git_lfs_store_path |
|
285 | config.registry.git_lfs_store_path = git_lfs_store_path | |
|
286 | config.registry.git_lfs_http_scheme = git_lfs_http_scheme | |||
282 | else: |
|
287 | else: | |
283 | # not found handler for API, reporting disabled LFS support |
|
288 | # not found handler for API, reporting disabled LFS support | |
284 | config.add_notfound_view(lfs_disabled, renderer='json') |
|
289 | config.add_notfound_view(lfs_disabled, renderer='json') | |
285 |
|
290 | |||
286 | app = config.make_wsgi_app() |
|
291 | app = config.make_wsgi_app() | |
287 | return app |
|
292 | return app |
@@ -1,239 +1,272 b'' | |||||
1 | # RhodeCode VCSServer provides access to different vcs backends via network. |
|
1 | # RhodeCode VCSServer provides access to different vcs backends via network. | |
2 | # Copyright (C) 2014-2019 RhodeCode GmbH |
|
2 | # Copyright (C) 2014-2019 RhodeCode GmbH | |
3 | # |
|
3 | # | |
4 | # This program is free software; you can redistribute it and/or modify |
|
4 | # This program is free software; you can redistribute it and/or modify | |
5 | # it under the terms of the GNU General Public License as published by |
|
5 | # it under the terms of the GNU General Public License as published by | |
6 | # the Free Software Foundation; either version 3 of the License, or |
|
6 | # the Free Software Foundation; either version 3 of the License, or | |
7 | # (at your option) any later version. |
|
7 | # (at your option) any later version. | |
8 | # |
|
8 | # | |
9 | # This program is distributed in the hope that it will be useful, |
|
9 | # This program is distributed in the hope that it will be useful, | |
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | # GNU General Public License for more details. |
|
12 | # GNU General Public License for more details. | |
13 | # |
|
13 | # | |
14 | # You should have received a copy of the GNU General Public License |
|
14 | # You should have received a copy of the GNU General Public License | |
15 | # along with this program; if not, write to the Free Software Foundation, |
|
15 | # along with this program; if not, write to the Free Software Foundation, | |
16 | # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
|
16 | # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | |
17 |
|
17 | |||
18 | import os |
|
18 | import os | |
19 | import pytest |
|
19 | import pytest | |
20 | from webtest.app import TestApp as WebObTestApp |
|
20 | from webtest.app import TestApp as WebObTestApp | |
21 | import simplejson as json |
|
21 | import simplejson as json | |
22 |
|
22 | |||
23 | from vcsserver.git_lfs.app import create_app |
|
23 | from vcsserver.git_lfs.app import create_app | |
24 |
|
24 | |||
25 |
|
25 | |||
26 | @pytest.fixture(scope='function') |
|
26 | @pytest.fixture(scope='function') | |
27 | def git_lfs_app(tmpdir): |
|
27 | def git_lfs_app(tmpdir): | |
28 | custom_app = WebObTestApp(create_app( |
|
28 | custom_app = WebObTestApp(create_app( | |
29 |
git_lfs_enabled=True, git_lfs_store_path=str(tmpdir) |
|
29 | git_lfs_enabled=True, git_lfs_store_path=str(tmpdir), | |
|
30 | git_lfs_http_scheme='http')) | |||
|
31 | custom_app._store = str(tmpdir) | |||
|
32 | return custom_app | |||
|
33 | ||||
|
34 | ||||
|
35 | @pytest.fixture(scope='function') | |||
|
36 | def git_lfs_https_app(tmpdir): | |||
|
37 | custom_app = WebObTestApp(create_app( | |||
|
38 | git_lfs_enabled=True, git_lfs_store_path=str(tmpdir), | |||
|
39 | git_lfs_http_scheme='https')) | |||
30 | custom_app._store = str(tmpdir) |
|
40 | custom_app._store = str(tmpdir) | |
31 | return custom_app |
|
41 | return custom_app | |
32 |
|
42 | |||
33 |
|
43 | |||
34 | @pytest.fixture() |
|
44 | @pytest.fixture() | |
35 | def http_auth(): |
|
45 | def http_auth(): | |
36 | return {'HTTP_AUTHORIZATION': "Basic XXXXX"} |
|
46 | return {'HTTP_AUTHORIZATION': "Basic XXXXX"} | |
37 |
|
47 | |||
38 |
|
48 | |||
39 | class TestLFSApplication(object): |
|
49 | class TestLFSApplication(object): | |
40 |
|
50 | |||
41 | def test_app_wrong_path(self, git_lfs_app): |
|
51 | def test_app_wrong_path(self, git_lfs_app): | |
42 | git_lfs_app.get('/repo/info/lfs/xxx', status=404) |
|
52 | git_lfs_app.get('/repo/info/lfs/xxx', status=404) | |
43 |
|
53 | |||
44 | def test_app_deprecated_endpoint(self, git_lfs_app): |
|
54 | def test_app_deprecated_endpoint(self, git_lfs_app): | |
45 | response = git_lfs_app.post('/repo/info/lfs/objects', status=501) |
|
55 | response = git_lfs_app.post('/repo/info/lfs/objects', status=501) | |
46 | assert response.status_code == 501 |
|
56 | assert response.status_code == 501 | |
47 | assert json.loads(response.text) == {u'message': u'LFS: v1 api not supported'} |
|
57 | assert json.loads(response.text) == {u'message': u'LFS: v1 api not supported'} | |
48 |
|
58 | |||
49 | def test_app_lock_verify_api_not_available(self, git_lfs_app): |
|
59 | def test_app_lock_verify_api_not_available(self, git_lfs_app): | |
50 | response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501) |
|
60 | response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501) | |
51 | assert response.status_code == 501 |
|
61 | assert response.status_code == 501 | |
52 | assert json.loads(response.text) == { |
|
62 | assert json.loads(response.text) == { | |
53 | u'message': u'GIT LFS locking api not supported'} |
|
63 | u'message': u'GIT LFS locking api not supported'} | |
54 |
|
64 | |||
55 | def test_app_lock_api_not_available(self, git_lfs_app): |
|
65 | def test_app_lock_api_not_available(self, git_lfs_app): | |
56 | response = git_lfs_app.post('/repo/info/lfs/locks', status=501) |
|
66 | response = git_lfs_app.post('/repo/info/lfs/locks', status=501) | |
57 | assert response.status_code == 501 |
|
67 | assert response.status_code == 501 | |
58 | assert json.loads(response.text) == { |
|
68 | assert json.loads(response.text) == { | |
59 | u'message': u'GIT LFS locking api not supported'} |
|
69 | u'message': u'GIT LFS locking api not supported'} | |
60 |
|
70 | |||
61 |
def test_app_batch_api_missing_auth(self, git_lfs_app |
|
71 | def test_app_batch_api_missing_auth(self, git_lfs_app): | |
62 | git_lfs_app.post_json( |
|
72 | git_lfs_app.post_json( | |
63 | '/repo/info/lfs/objects/batch', params={}, status=403) |
|
73 | '/repo/info/lfs/objects/batch', params={}, status=403) | |
64 |
|
74 | |||
65 | def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth): |
|
75 | def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth): | |
66 | response = git_lfs_app.post_json( |
|
76 | response = git_lfs_app.post_json( | |
67 | '/repo/info/lfs/objects/batch', params={}, status=400, |
|
77 | '/repo/info/lfs/objects/batch', params={}, status=400, | |
68 | extra_environ=http_auth) |
|
78 | extra_environ=http_auth) | |
69 | assert json.loads(response.text) == { |
|
79 | assert json.loads(response.text) == { | |
70 | u'message': u'unsupported operation mode: `None`'} |
|
80 | u'message': u'unsupported operation mode: `None`'} | |
71 |
|
81 | |||
72 | def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth): |
|
82 | def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth): | |
73 | response = git_lfs_app.post_json( |
|
83 | response = git_lfs_app.post_json( | |
74 | '/repo/info/lfs/objects/batch', params={'operation': 'download'}, |
|
84 | '/repo/info/lfs/objects/batch', params={'operation': 'download'}, | |
75 | status=400, extra_environ=http_auth) |
|
85 | status=400, extra_environ=http_auth) | |
76 | assert json.loads(response.text) == { |
|
86 | assert json.loads(response.text) == { | |
77 | u'message': u'missing objects data'} |
|
87 | u'message': u'missing objects data'} | |
78 |
|
88 | |||
79 | def test_app_batch_api_unsupported_data_in_objects( |
|
89 | def test_app_batch_api_unsupported_data_in_objects( | |
80 | self, git_lfs_app, http_auth): |
|
90 | self, git_lfs_app, http_auth): | |
81 | params = {'operation': 'download', |
|
91 | params = {'operation': 'download', | |
82 | 'objects': [{}]} |
|
92 | 'objects': [{}]} | |
83 | response = git_lfs_app.post_json( |
|
93 | response = git_lfs_app.post_json( | |
84 | '/repo/info/lfs/objects/batch', params=params, status=400, |
|
94 | '/repo/info/lfs/objects/batch', params=params, status=400, | |
85 | extra_environ=http_auth) |
|
95 | extra_environ=http_auth) | |
86 | assert json.loads(response.text) == { |
|
96 | assert json.loads(response.text) == { | |
87 | u'message': u'unsupported data in objects'} |
|
97 | u'message': u'unsupported data in objects'} | |
88 |
|
98 | |||
89 | def test_app_batch_api_download_missing_object( |
|
99 | def test_app_batch_api_download_missing_object( | |
90 | self, git_lfs_app, http_auth): |
|
100 | self, git_lfs_app, http_auth): | |
91 | params = {'operation': 'download', |
|
101 | params = {'operation': 'download', | |
92 | 'objects': [{'oid': '123', 'size': '1024'}]} |
|
102 | 'objects': [{'oid': '123', 'size': '1024'}]} | |
93 | response = git_lfs_app.post_json( |
|
103 | response = git_lfs_app.post_json( | |
94 | '/repo/info/lfs/objects/batch', params=params, |
|
104 | '/repo/info/lfs/objects/batch', params=params, | |
95 | extra_environ=http_auth) |
|
105 | extra_environ=http_auth) | |
96 |
|
106 | |||
97 | expected_objects = [ |
|
107 | expected_objects = [ | |
98 | {u'authenticated': True, |
|
108 | {u'authenticated': True, | |
99 | u'errors': {u'error': { |
|
109 | u'errors': {u'error': { | |
100 | u'code': 404, |
|
110 | u'code': 404, | |
101 | u'message': u'object: 123 does not exist in store'}}, |
|
111 | u'message': u'object: 123 does not exist in store'}}, | |
102 | u'oid': u'123', |
|
112 | u'oid': u'123', | |
103 | u'size': u'1024'} |
|
113 | u'size': u'1024'} | |
104 | ] |
|
114 | ] | |
105 | assert json.loads(response.text) == { |
|
115 | assert json.loads(response.text) == { | |
106 | 'objects': expected_objects, 'transfer': 'basic'} |
|
116 | 'objects': expected_objects, 'transfer': 'basic'} | |
107 |
|
117 | |||
108 | def test_app_batch_api_download(self, git_lfs_app, http_auth): |
|
118 | def test_app_batch_api_download(self, git_lfs_app, http_auth): | |
109 | oid = '456' |
|
119 | oid = '456' | |
110 | oid_path = os.path.join(git_lfs_app._store, oid) |
|
120 | oid_path = os.path.join(git_lfs_app._store, oid) | |
111 | if not os.path.isdir(os.path.dirname(oid_path)): |
|
121 | if not os.path.isdir(os.path.dirname(oid_path)): | |
112 | os.makedirs(os.path.dirname(oid_path)) |
|
122 | os.makedirs(os.path.dirname(oid_path)) | |
113 | with open(oid_path, 'wb') as f: |
|
123 | with open(oid_path, 'wb') as f: | |
114 | f.write('OID_CONTENT') |
|
124 | f.write('OID_CONTENT') | |
115 |
|
125 | |||
116 | params = {'operation': 'download', |
|
126 | params = {'operation': 'download', | |
117 | 'objects': [{'oid': oid, 'size': '1024'}]} |
|
127 | 'objects': [{'oid': oid, 'size': '1024'}]} | |
118 | response = git_lfs_app.post_json( |
|
128 | response = git_lfs_app.post_json( | |
119 | '/repo/info/lfs/objects/batch', params=params, |
|
129 | '/repo/info/lfs/objects/batch', params=params, | |
120 | extra_environ=http_auth) |
|
130 | extra_environ=http_auth) | |
121 |
|
131 | |||
122 | expected_objects = [ |
|
132 | expected_objects = [ | |
123 | {u'authenticated': True, |
|
133 | {u'authenticated': True, | |
124 | u'actions': { |
|
134 | u'actions': { | |
125 | u'download': { |
|
135 | u'download': { | |
126 | u'header': {u'Authorization': u'Basic XXXXX'}, |
|
136 | u'header': {u'Authorization': u'Basic XXXXX'}, | |
127 | u'href': u'http://localhost/repo/info/lfs/objects/456'}, |
|
137 | u'href': u'http://localhost/repo/info/lfs/objects/456'}, | |
128 | }, |
|
138 | }, | |
129 | u'oid': u'456', |
|
139 | u'oid': u'456', | |
130 | u'size': u'1024'} |
|
140 | u'size': u'1024'} | |
131 | ] |
|
141 | ] | |
132 | assert json.loads(response.text) == { |
|
142 | assert json.loads(response.text) == { | |
133 | 'objects': expected_objects, 'transfer': 'basic'} |
|
143 | 'objects': expected_objects, 'transfer': 'basic'} | |
134 |
|
144 | |||
135 | def test_app_batch_api_upload(self, git_lfs_app, http_auth): |
|
145 | def test_app_batch_api_upload(self, git_lfs_app, http_auth): | |
136 | params = {'operation': 'upload', |
|
146 | params = {'operation': 'upload', | |
137 | 'objects': [{'oid': '123', 'size': '1024'}]} |
|
147 | 'objects': [{'oid': '123', 'size': '1024'}]} | |
138 | response = git_lfs_app.post_json( |
|
148 | response = git_lfs_app.post_json( | |
139 | '/repo/info/lfs/objects/batch', params=params, |
|
149 | '/repo/info/lfs/objects/batch', params=params, | |
140 | extra_environ=http_auth) |
|
150 | extra_environ=http_auth) | |
141 | expected_objects = [ |
|
151 | expected_objects = [ | |
142 | {u'authenticated': True, |
|
152 | {u'authenticated': True, | |
143 | u'actions': { |
|
153 | u'actions': { | |
144 | u'upload': { |
|
154 | u'upload': { | |
145 | u'header': {u'Authorization': u'Basic XXXXX', |
|
155 | u'header': {u'Authorization': u'Basic XXXXX', | |
146 | u'Transfer-Encoding': u'chunked'}, |
|
156 | u'Transfer-Encoding': u'chunked'}, | |
147 | u'href': u'http://localhost/repo/info/lfs/objects/123'}, |
|
157 | u'href': u'http://localhost/repo/info/lfs/objects/123'}, | |
148 | u'verify': { |
|
158 | u'verify': { | |
149 | u'header': {u'Authorization': u'Basic XXXXX'}, |
|
159 | u'header': {u'Authorization': u'Basic XXXXX'}, | |
150 | u'href': u'http://localhost/repo/info/lfs/verify'} |
|
160 | u'href': u'http://localhost/repo/info/lfs/verify'} | |
151 | }, |
|
161 | }, | |
152 | u'oid': u'123', |
|
162 | u'oid': u'123', | |
153 | u'size': u'1024'} |
|
163 | u'size': u'1024'} | |
154 | ] |
|
164 | ] | |
155 | assert json.loads(response.text) == { |
|
165 | assert json.loads(response.text) == { | |
156 | 'objects': expected_objects, 'transfer': 'basic'} |
|
166 | 'objects': expected_objects, 'transfer': 'basic'} | |
157 |
|
167 | |||
|
168 | def test_app_batch_api_upload_for_https(self, git_lfs_https_app, http_auth): | |||
|
169 | params = {'operation': 'upload', | |||
|
170 | 'objects': [{'oid': '123', 'size': '1024'}]} | |||
|
171 | response = git_lfs_https_app.post_json( | |||
|
172 | '/repo/info/lfs/objects/batch', params=params, | |||
|
173 | extra_environ=http_auth) | |||
|
174 | expected_objects = [ | |||
|
175 | {u'authenticated': True, | |||
|
176 | u'actions': { | |||
|
177 | u'upload': { | |||
|
178 | u'header': {u'Authorization': u'Basic XXXXX', | |||
|
179 | u'Transfer-Encoding': u'chunked'}, | |||
|
180 | u'href': u'https://localhost/repo/info/lfs/objects/123'}, | |||
|
181 | u'verify': { | |||
|
182 | u'header': {u'Authorization': u'Basic XXXXX'}, | |||
|
183 | u'href': u'https://localhost/repo/info/lfs/verify'} | |||
|
184 | }, | |||
|
185 | u'oid': u'123', | |||
|
186 | u'size': u'1024'} | |||
|
187 | ] | |||
|
188 | assert json.loads(response.text) == { | |||
|
189 | 'objects': expected_objects, 'transfer': 'basic'} | |||
|
190 | ||||
158 | def test_app_verify_api_missing_data(self, git_lfs_app): |
|
191 | def test_app_verify_api_missing_data(self, git_lfs_app): | |
159 |
params = {'oid': 'missing' |
|
192 | params = {'oid': 'missing'} | |
160 | response = git_lfs_app.post_json( |
|
193 | response = git_lfs_app.post_json( | |
161 | '/repo/info/lfs/verify', params=params, |
|
194 | '/repo/info/lfs/verify', params=params, | |
162 | status=400) |
|
195 | status=400) | |
163 |
|
196 | |||
164 | assert json.loads(response.text) == { |
|
197 | assert json.loads(response.text) == { | |
165 | u'message': u'missing oid and size in request data'} |
|
198 | u'message': u'missing oid and size in request data'} | |
166 |
|
199 | |||
167 | def test_app_verify_api_missing_obj(self, git_lfs_app): |
|
200 | def test_app_verify_api_missing_obj(self, git_lfs_app): | |
168 | params = {'oid': 'missing', 'size': '1024'} |
|
201 | params = {'oid': 'missing', 'size': '1024'} | |
169 | response = git_lfs_app.post_json( |
|
202 | response = git_lfs_app.post_json( | |
170 | '/repo/info/lfs/verify', params=params, |
|
203 | '/repo/info/lfs/verify', params=params, | |
171 | status=404) |
|
204 | status=404) | |
172 |
|
205 | |||
173 | assert json.loads(response.text) == { |
|
206 | assert json.loads(response.text) == { | |
174 | u'message': u'oid `missing` does not exists in store'} |
|
207 | u'message': u'oid `missing` does not exists in store'} | |
175 |
|
208 | |||
176 | def test_app_verify_api_size_mismatch(self, git_lfs_app): |
|
209 | def test_app_verify_api_size_mismatch(self, git_lfs_app): | |
177 | oid = 'existing' |
|
210 | oid = 'existing' | |
178 | oid_path = os.path.join(git_lfs_app._store, oid) |
|
211 | oid_path = os.path.join(git_lfs_app._store, oid) | |
179 | if not os.path.isdir(os.path.dirname(oid_path)): |
|
212 | if not os.path.isdir(os.path.dirname(oid_path)): | |
180 | os.makedirs(os.path.dirname(oid_path)) |
|
213 | os.makedirs(os.path.dirname(oid_path)) | |
181 | with open(oid_path, 'wb') as f: |
|
214 | with open(oid_path, 'wb') as f: | |
182 | f.write('OID_CONTENT') |
|
215 | f.write('OID_CONTENT') | |
183 |
|
216 | |||
184 | params = {'oid': oid, 'size': '1024'} |
|
217 | params = {'oid': oid, 'size': '1024'} | |
185 | response = git_lfs_app.post_json( |
|
218 | response = git_lfs_app.post_json( | |
186 | '/repo/info/lfs/verify', params=params, status=422) |
|
219 | '/repo/info/lfs/verify', params=params, status=422) | |
187 |
|
220 | |||
188 | assert json.loads(response.text) == { |
|
221 | assert json.loads(response.text) == { | |
189 | u'message': u'requested file size mismatch ' |
|
222 | u'message': u'requested file size mismatch ' | |
190 | u'store size:11 requested:1024'} |
|
223 | u'store size:11 requested:1024'} | |
191 |
|
224 | |||
192 | def test_app_verify_api(self, git_lfs_app): |
|
225 | def test_app_verify_api(self, git_lfs_app): | |
193 | oid = 'existing' |
|
226 | oid = 'existing' | |
194 | oid_path = os.path.join(git_lfs_app._store, oid) |
|
227 | oid_path = os.path.join(git_lfs_app._store, oid) | |
195 | if not os.path.isdir(os.path.dirname(oid_path)): |
|
228 | if not os.path.isdir(os.path.dirname(oid_path)): | |
196 | os.makedirs(os.path.dirname(oid_path)) |
|
229 | os.makedirs(os.path.dirname(oid_path)) | |
197 | with open(oid_path, 'wb') as f: |
|
230 | with open(oid_path, 'wb') as f: | |
198 | f.write('OID_CONTENT') |
|
231 | f.write('OID_CONTENT') | |
199 |
|
232 | |||
200 | params = {'oid': oid, 'size': 11} |
|
233 | params = {'oid': oid, 'size': 11} | |
201 | response = git_lfs_app.post_json( |
|
234 | response = git_lfs_app.post_json( | |
202 | '/repo/info/lfs/verify', params=params) |
|
235 | '/repo/info/lfs/verify', params=params) | |
203 |
|
236 | |||
204 | assert json.loads(response.text) == { |
|
237 | assert json.loads(response.text) == { | |
205 | u'message': {u'size': u'ok', u'in_store': u'ok'}} |
|
238 | u'message': {u'size': u'ok', u'in_store': u'ok'}} | |
206 |
|
239 | |||
207 | def test_app_download_api_oid_not_existing(self, git_lfs_app): |
|
240 | def test_app_download_api_oid_not_existing(self, git_lfs_app): | |
208 | oid = 'missing' |
|
241 | oid = 'missing' | |
209 |
|
242 | |||
210 | response = git_lfs_app.get( |
|
243 | response = git_lfs_app.get( | |
211 | '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404) |
|
244 | '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404) | |
212 |
|
245 | |||
213 | assert json.loads(response.text) == { |
|
246 | assert json.loads(response.text) == { | |
214 | u'message': u'requested file with oid `missing` not found in store'} |
|
247 | u'message': u'requested file with oid `missing` not found in store'} | |
215 |
|
248 | |||
216 | def test_app_download_api(self, git_lfs_app): |
|
249 | def test_app_download_api(self, git_lfs_app): | |
217 | oid = 'existing' |
|
250 | oid = 'existing' | |
218 | oid_path = os.path.join(git_lfs_app._store, oid) |
|
251 | oid_path = os.path.join(git_lfs_app._store, oid) | |
219 | if not os.path.isdir(os.path.dirname(oid_path)): |
|
252 | if not os.path.isdir(os.path.dirname(oid_path)): | |
220 | os.makedirs(os.path.dirname(oid_path)) |
|
253 | os.makedirs(os.path.dirname(oid_path)) | |
221 | with open(oid_path, 'wb') as f: |
|
254 | with open(oid_path, 'wb') as f: | |
222 | f.write('OID_CONTENT') |
|
255 | f.write('OID_CONTENT') | |
223 |
|
256 | |||
224 | response = git_lfs_app.get( |
|
257 | response = git_lfs_app.get( | |
225 | '/repo/info/lfs/objects/{oid}'.format(oid=oid)) |
|
258 | '/repo/info/lfs/objects/{oid}'.format(oid=oid)) | |
226 | assert response |
|
259 | assert response | |
227 |
|
260 | |||
228 | def test_app_upload(self, git_lfs_app): |
|
261 | def test_app_upload(self, git_lfs_app): | |
229 | oid = 'uploaded' |
|
262 | oid = 'uploaded' | |
230 |
|
263 | |||
231 | response = git_lfs_app.put( |
|
264 | response = git_lfs_app.put( | |
232 | '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT') |
|
265 | '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT') | |
233 |
|
266 | |||
234 | assert json.loads(response.text) == {u'upload': u'ok'} |
|
267 | assert json.loads(response.text) == {u'upload': u'ok'} | |
235 |
|
268 | |||
236 | # verify that we actually wrote that OID |
|
269 | # verify that we actually wrote that OID | |
237 | oid_path = os.path.join(git_lfs_app._store, oid) |
|
270 | oid_path = os.path.join(git_lfs_app._store, oid) | |
238 | assert os.path.isfile(oid_path) |
|
271 | assert os.path.isfile(oid_path) | |
239 | assert 'CONTENT' == open(oid_path).read() |
|
272 | assert 'CONTENT' == open(oid_path).read() |
@@ -1,234 +1,235 b'' | |||||
1 | # RhodeCode VCSServer provides access to different vcs backends via network. |
|
1 | # RhodeCode VCSServer provides access to different vcs backends via network. | |
2 | # Copyright (C) 2014-2019 RhodeCode GmbH |
|
2 | # Copyright (C) 2014-2019 RhodeCode GmbH | |
3 | # |
|
3 | # | |
4 | # This program is free software; you can redistribute it and/or modify |
|
4 | # This program is free software; you can redistribute it and/or modify | |
5 | # it under the terms of the GNU General Public License as published by |
|
5 | # it under the terms of the GNU General Public License as published by | |
6 | # the Free Software Foundation; either version 3 of the License, or |
|
6 | # the Free Software Foundation; either version 3 of the License, or | |
7 | # (at your option) any later version. |
|
7 | # (at your option) any later version. | |
8 | # |
|
8 | # | |
9 | # This program is distributed in the hope that it will be useful, |
|
9 | # This program is distributed in the hope that it will be useful, | |
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | # GNU General Public License for more details. |
|
12 | # GNU General Public License for more details. | |
13 | # |
|
13 | # | |
14 | # You should have received a copy of the GNU General Public License |
|
14 | # You should have received a copy of the GNU General Public License | |
15 | # along with this program; if not, write to the Free Software Foundation, |
|
15 | # along with this program; if not, write to the Free Software Foundation, | |
16 | # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
|
16 | # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | |
17 |
|
17 | |||
18 | import os |
|
18 | import os | |
19 | import logging |
|
19 | import logging | |
20 | import itertools |
|
20 | import itertools | |
21 |
|
21 | |||
22 | import mercurial |
|
22 | import mercurial | |
23 | import mercurial.error |
|
23 | import mercurial.error | |
24 | import mercurial.wireprotoserver |
|
24 | import mercurial.wireprotoserver | |
25 | import mercurial.hgweb.common |
|
25 | import mercurial.hgweb.common | |
26 | import mercurial.hgweb.hgweb_mod |
|
26 | import mercurial.hgweb.hgweb_mod | |
27 | import webob.exc |
|
27 | import webob.exc | |
28 |
|
28 | |||
29 | from vcsserver import pygrack, exceptions, settings, git_lfs |
|
29 | from vcsserver import pygrack, exceptions, settings, git_lfs | |
30 |
|
30 | |||
31 |
|
31 | |||
32 | log = logging.getLogger(__name__) |
|
32 | log = logging.getLogger(__name__) | |
33 |
|
33 | |||
34 |
|
34 | |||
35 | # propagated from mercurial documentation |
|
35 | # propagated from mercurial documentation | |
36 | HG_UI_SECTIONS = [ |
|
36 | HG_UI_SECTIONS = [ | |
37 | 'alias', 'auth', 'decode/encode', 'defaults', 'diff', 'email', 'extensions', |
|
37 | 'alias', 'auth', 'decode/encode', 'defaults', 'diff', 'email', 'extensions', | |
38 | 'format', 'merge-patterns', 'merge-tools', 'hooks', 'http_proxy', 'smtp', |
|
38 | 'format', 'merge-patterns', 'merge-tools', 'hooks', 'http_proxy', 'smtp', | |
39 | 'patch', 'paths', 'profiling', 'server', 'trusted', 'ui', 'web', |
|
39 | 'patch', 'paths', 'profiling', 'server', 'trusted', 'ui', 'web', | |
40 | ] |
|
40 | ] | |
41 |
|
41 | |||
42 |
|
42 | |||
43 | class HgWeb(mercurial.hgweb.hgweb_mod.hgweb): |
|
43 | class HgWeb(mercurial.hgweb.hgweb_mod.hgweb): | |
44 | """Extension of hgweb that simplifies some functions.""" |
|
44 | """Extension of hgweb that simplifies some functions.""" | |
45 |
|
45 | |||
46 | def _get_view(self, repo): |
|
46 | def _get_view(self, repo): | |
47 | """Views are not supported.""" |
|
47 | """Views are not supported.""" | |
48 | return repo |
|
48 | return repo | |
49 |
|
49 | |||
50 | def loadsubweb(self): |
|
50 | def loadsubweb(self): | |
51 | """The result is only used in the templater method which is not used.""" |
|
51 | """The result is only used in the templater method which is not used.""" | |
52 | return None |
|
52 | return None | |
53 |
|
53 | |||
54 | def run(self): |
|
54 | def run(self): | |
55 | """Unused function so raise an exception if accidentally called.""" |
|
55 | """Unused function so raise an exception if accidentally called.""" | |
56 | raise NotImplementedError |
|
56 | raise NotImplementedError | |
57 |
|
57 | |||
58 | def templater(self, req): |
|
58 | def templater(self, req): | |
59 | """Function used in an unreachable code path. |
|
59 | """Function used in an unreachable code path. | |
60 |
|
60 | |||
61 | This code is unreachable because we guarantee that the HTTP request, |
|
61 | This code is unreachable because we guarantee that the HTTP request, | |
62 | corresponds to a Mercurial command. See the is_hg method. So, we are |
|
62 | corresponds to a Mercurial command. See the is_hg method. So, we are | |
63 | never going to get a user-visible url. |
|
63 | never going to get a user-visible url. | |
64 | """ |
|
64 | """ | |
65 | raise NotImplementedError |
|
65 | raise NotImplementedError | |
66 |
|
66 | |||
67 | def archivelist(self, nodeid): |
|
67 | def archivelist(self, nodeid): | |
68 | """Unused function so raise an exception if accidentally called.""" |
|
68 | """Unused function so raise an exception if accidentally called.""" | |
69 | raise NotImplementedError |
|
69 | raise NotImplementedError | |
70 |
|
70 | |||
71 | def __call__(self, environ, start_response): |
|
71 | def __call__(self, environ, start_response): | |
72 | """Run the WSGI application. |
|
72 | """Run the WSGI application. | |
73 |
|
73 | |||
74 | This may be called by multiple threads. |
|
74 | This may be called by multiple threads. | |
75 | """ |
|
75 | """ | |
76 | from mercurial.hgweb import request as requestmod |
|
76 | from mercurial.hgweb import request as requestmod | |
77 | req = requestmod.parserequestfromenv(environ) |
|
77 | req = requestmod.parserequestfromenv(environ) | |
78 | res = requestmod.wsgiresponse(req, start_response) |
|
78 | res = requestmod.wsgiresponse(req, start_response) | |
79 | gen = self.run_wsgi(req, res) |
|
79 | gen = self.run_wsgi(req, res) | |
80 |
|
80 | |||
81 | first_chunk = None |
|
81 | first_chunk = None | |
82 |
|
82 | |||
83 | try: |
|
83 | try: | |
84 | data = gen.next() |
|
84 | data = gen.next() | |
85 |
|
85 | |||
86 | def first_chunk(): |
|
86 | def first_chunk(): | |
87 | yield data |
|
87 | yield data | |
88 | except StopIteration: |
|
88 | except StopIteration: | |
89 | pass |
|
89 | pass | |
90 |
|
90 | |||
91 | if first_chunk: |
|
91 | if first_chunk: | |
92 | return itertools.chain(first_chunk(), gen) |
|
92 | return itertools.chain(first_chunk(), gen) | |
93 | return gen |
|
93 | return gen | |
94 |
|
94 | |||
95 | def _runwsgi(self, req, res, repo): |
|
95 | def _runwsgi(self, req, res, repo): | |
96 |
|
96 | |||
97 | cmd = req.qsparams.get('cmd', '') |
|
97 | cmd = req.qsparams.get('cmd', '') | |
98 | if not mercurial.wireprotoserver.iscmd(cmd): |
|
98 | if not mercurial.wireprotoserver.iscmd(cmd): | |
99 | # NOTE(marcink): for unsupported commands, we return bad request |
|
99 | # NOTE(marcink): for unsupported commands, we return bad request | |
100 | # internally from HG |
|
100 | # internally from HG | |
101 | from mercurial.hgweb.common import statusmessage |
|
101 | from mercurial.hgweb.common import statusmessage | |
102 | res.status = statusmessage(mercurial.hgweb.common.HTTP_BAD_REQUEST) |
|
102 | res.status = statusmessage(mercurial.hgweb.common.HTTP_BAD_REQUEST) | |
103 | res.setbodybytes('') |
|
103 | res.setbodybytes('') | |
104 | return res.sendresponse() |
|
104 | return res.sendresponse() | |
105 |
|
105 | |||
106 | return super(HgWeb, self)._runwsgi(req, res, repo) |
|
106 | return super(HgWeb, self)._runwsgi(req, res, repo) | |
107 |
|
107 | |||
108 |
|
108 | |||
109 | def make_hg_ui_from_config(repo_config): |
|
109 | def make_hg_ui_from_config(repo_config): | |
110 | baseui = mercurial.ui.ui() |
|
110 | baseui = mercurial.ui.ui() | |
111 |
|
111 | |||
112 | # clean the baseui object |
|
112 | # clean the baseui object | |
113 | baseui._ocfg = mercurial.config.config() |
|
113 | baseui._ocfg = mercurial.config.config() | |
114 | baseui._ucfg = mercurial.config.config() |
|
114 | baseui._ucfg = mercurial.config.config() | |
115 | baseui._tcfg = mercurial.config.config() |
|
115 | baseui._tcfg = mercurial.config.config() | |
116 |
|
116 | |||
117 | for section, option, value in repo_config: |
|
117 | for section, option, value in repo_config: | |
118 | baseui.setconfig(section, option, value) |
|
118 | baseui.setconfig(section, option, value) | |
119 |
|
119 | |||
120 | # make our hgweb quiet so it doesn't print output |
|
120 | # make our hgweb quiet so it doesn't print output | |
121 | baseui.setconfig('ui', 'quiet', 'true') |
|
121 | baseui.setconfig('ui', 'quiet', 'true') | |
122 |
|
122 | |||
123 | return baseui |
|
123 | return baseui | |
124 |
|
124 | |||
125 |
|
125 | |||
126 | def update_hg_ui_from_hgrc(baseui, repo_path): |
|
126 | def update_hg_ui_from_hgrc(baseui, repo_path): | |
127 | path = os.path.join(repo_path, '.hg', 'hgrc') |
|
127 | path = os.path.join(repo_path, '.hg', 'hgrc') | |
128 |
|
128 | |||
129 | if not os.path.isfile(path): |
|
129 | if not os.path.isfile(path): | |
130 | log.debug('hgrc file is not present at %s, skipping...', path) |
|
130 | log.debug('hgrc file is not present at %s, skipping...', path) | |
131 | return |
|
131 | return | |
132 | log.debug('reading hgrc from %s', path) |
|
132 | log.debug('reading hgrc from %s', path) | |
133 | cfg = mercurial.config.config() |
|
133 | cfg = mercurial.config.config() | |
134 | cfg.read(path) |
|
134 | cfg.read(path) | |
135 | for section in HG_UI_SECTIONS: |
|
135 | for section in HG_UI_SECTIONS: | |
136 | for k, v in cfg.items(section): |
|
136 | for k, v in cfg.items(section): | |
137 | log.debug('settings ui from file: [%s] %s=%s', section, k, v) |
|
137 | log.debug('settings ui from file: [%s] %s=%s', section, k, v) | |
138 | baseui.setconfig(section, k, v) |
|
138 | baseui.setconfig(section, k, v) | |
139 |
|
139 | |||
140 |
|
140 | |||
141 | def create_hg_wsgi_app(repo_path, repo_name, config): |
|
141 | def create_hg_wsgi_app(repo_path, repo_name, config): | |
142 | """ |
|
142 | """ | |
143 | Prepares a WSGI application to handle Mercurial requests. |
|
143 | Prepares a WSGI application to handle Mercurial requests. | |
144 |
|
144 | |||
145 | :param config: is a list of 3-item tuples representing a ConfigObject |
|
145 | :param config: is a list of 3-item tuples representing a ConfigObject | |
146 | (it is the serialized version of the config object). |
|
146 | (it is the serialized version of the config object). | |
147 | """ |
|
147 | """ | |
148 | log.debug("Creating Mercurial WSGI application") |
|
148 | log.debug("Creating Mercurial WSGI application") | |
149 |
|
149 | |||
150 | baseui = make_hg_ui_from_config(config) |
|
150 | baseui = make_hg_ui_from_config(config) | |
151 | update_hg_ui_from_hgrc(baseui, repo_path) |
|
151 | update_hg_ui_from_hgrc(baseui, repo_path) | |
152 |
|
152 | |||
153 | try: |
|
153 | try: | |
154 | return HgWeb(repo_path, name=repo_name, baseui=baseui) |
|
154 | return HgWeb(repo_path, name=repo_name, baseui=baseui) | |
155 | except mercurial.error.RequirementError as e: |
|
155 | except mercurial.error.RequirementError as e: | |
156 | raise exceptions.RequirementException(e)(e) |
|
156 | raise exceptions.RequirementException(e)(e) | |
157 |
|
157 | |||
158 |
|
158 | |||
159 | class GitHandler(object): |
|
159 | class GitHandler(object): | |
160 | """ |
|
160 | """ | |
161 | Handler for Git operations like push/pull etc |
|
161 | Handler for Git operations like push/pull etc | |
162 | """ |
|
162 | """ | |
163 | def __init__(self, repo_location, repo_name, git_path, update_server_info, |
|
163 | def __init__(self, repo_location, repo_name, git_path, update_server_info, | |
164 | extras): |
|
164 | extras): | |
165 | if not os.path.isdir(repo_location): |
|
165 | if not os.path.isdir(repo_location): | |
166 | raise OSError(repo_location) |
|
166 | raise OSError(repo_location) | |
167 | self.content_path = repo_location |
|
167 | self.content_path = repo_location | |
168 | self.repo_name = repo_name |
|
168 | self.repo_name = repo_name | |
169 | self.repo_location = repo_location |
|
169 | self.repo_location = repo_location | |
170 | self.extras = extras |
|
170 | self.extras = extras | |
171 | self.git_path = git_path |
|
171 | self.git_path = git_path | |
172 | self.update_server_info = update_server_info |
|
172 | self.update_server_info = update_server_info | |
173 |
|
173 | |||
174 | def __call__(self, environ, start_response): |
|
174 | def __call__(self, environ, start_response): | |
175 | app = webob.exc.HTTPNotFound() |
|
175 | app = webob.exc.HTTPNotFound() | |
176 | candidate_paths = ( |
|
176 | candidate_paths = ( | |
177 | self.content_path, os.path.join(self.content_path, '.git')) |
|
177 | self.content_path, os.path.join(self.content_path, '.git')) | |
178 |
|
178 | |||
179 | for content_path in candidate_paths: |
|
179 | for content_path in candidate_paths: | |
180 | try: |
|
180 | try: | |
181 | app = pygrack.GitRepository( |
|
181 | app = pygrack.GitRepository( | |
182 | self.repo_name, content_path, self.git_path, |
|
182 | self.repo_name, content_path, self.git_path, | |
183 | self.update_server_info, self.extras) |
|
183 | self.update_server_info, self.extras) | |
184 | break |
|
184 | break | |
185 | except OSError: |
|
185 | except OSError: | |
186 | continue |
|
186 | continue | |
187 |
|
187 | |||
188 | return app(environ, start_response) |
|
188 | return app(environ, start_response) | |
189 |
|
189 | |||
190 |
|
190 | |||
191 | def create_git_wsgi_app(repo_path, repo_name, config): |
|
191 | def create_git_wsgi_app(repo_path, repo_name, config): | |
192 | """ |
|
192 | """ | |
193 | Creates a WSGI application to handle Git requests. |
|
193 | Creates a WSGI application to handle Git requests. | |
194 |
|
194 | |||
195 | :param config: is a dictionary holding the extras. |
|
195 | :param config: is a dictionary holding the extras. | |
196 | """ |
|
196 | """ | |
197 | git_path = settings.GIT_EXECUTABLE |
|
197 | git_path = settings.GIT_EXECUTABLE | |
198 | update_server_info = config.pop('git_update_server_info') |
|
198 | update_server_info = config.pop('git_update_server_info') | |
199 | app = GitHandler( |
|
199 | app = GitHandler( | |
200 | repo_path, repo_name, git_path, update_server_info, config) |
|
200 | repo_path, repo_name, git_path, update_server_info, config) | |
201 |
|
201 | |||
202 | return app |
|
202 | return app | |
203 |
|
203 | |||
204 |
|
204 | |||
205 | class GitLFSHandler(object): |
|
205 | class GitLFSHandler(object): | |
206 | """ |
|
206 | """ | |
207 | Handler for Git LFS operations |
|
207 | Handler for Git LFS operations | |
208 | """ |
|
208 | """ | |
209 |
|
209 | |||
210 | def __init__(self, repo_location, repo_name, git_path, update_server_info, |
|
210 | def __init__(self, repo_location, repo_name, git_path, update_server_info, | |
211 | extras): |
|
211 | extras): | |
212 | if not os.path.isdir(repo_location): |
|
212 | if not os.path.isdir(repo_location): | |
213 | raise OSError(repo_location) |
|
213 | raise OSError(repo_location) | |
214 | self.content_path = repo_location |
|
214 | self.content_path = repo_location | |
215 | self.repo_name = repo_name |
|
215 | self.repo_name = repo_name | |
216 | self.repo_location = repo_location |
|
216 | self.repo_location = repo_location | |
217 | self.extras = extras |
|
217 | self.extras = extras | |
218 | self.git_path = git_path |
|
218 | self.git_path = git_path | |
219 | self.update_server_info = update_server_info |
|
219 | self.update_server_info = update_server_info | |
220 |
|
220 | |||
221 | def get_app(self, git_lfs_enabled, git_lfs_store_path): |
|
221 | def get_app(self, git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme): | |
222 | app = git_lfs.create_app(git_lfs_enabled, git_lfs_store_path) |
|
222 | app = git_lfs.create_app(git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme) | |
223 | return app |
|
223 | return app | |
224 |
|
224 | |||
225 |
|
225 | |||
226 | def create_git_lfs_wsgi_app(repo_path, repo_name, config): |
|
226 | def create_git_lfs_wsgi_app(repo_path, repo_name, config): | |
227 | git_path = settings.GIT_EXECUTABLE |
|
227 | git_path = settings.GIT_EXECUTABLE | |
228 | update_server_info = config.pop('git_update_server_info') |
|
228 | update_server_info = config.pop('git_update_server_info') | |
229 | git_lfs_enabled = config.pop('git_lfs_enabled') |
|
229 | git_lfs_enabled = config.pop('git_lfs_enabled') | |
230 | git_lfs_store_path = config.pop('git_lfs_store_path') |
|
230 | git_lfs_store_path = config.pop('git_lfs_store_path') | |
|
231 | git_lfs_http_scheme = config.pop('git_lfs_http_scheme', 'http') | |||
231 | app = GitLFSHandler( |
|
232 | app = GitLFSHandler( | |
232 | repo_path, repo_name, git_path, update_server_info, config) |
|
233 | repo_path, repo_name, git_path, update_server_info, config) | |
233 |
|
234 | |||
234 | return app.get_app(git_lfs_enabled, git_lfs_store_path) |
|
235 | return app.get_app(git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme) |
General Comments 0
You need to be logged in to leave comments.
Login now