##// END OF EJS Templates
git-lfs: fixed tests for upload, and added new for file verification.
marcink -
r201:333281c9 default
parent child Browse files
Show More
@@ -1,172 +1,175 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2017 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import os
19 import shutil
19 import shutil
20 import logging
20 import logging
21 from collections import OrderedDict
21 from collections import OrderedDict
22
22
23 log = logging.getLogger(__name__)
23 log = logging.getLogger(__name__)
24
24
25
25
26 class OidHandler(object):
26 class OidHandler(object):
27
27
28 def __init__(self, store, repo_name, auth, oid, obj_size, obj_data, obj_href,
28 def __init__(self, store, repo_name, auth, oid, obj_size, obj_data, obj_href,
29 obj_verify_href=None):
29 obj_verify_href=None):
30 self.current_store = store
30 self.current_store = store
31 self.repo_name = repo_name
31 self.repo_name = repo_name
32 self.auth = auth
32 self.auth = auth
33 self.oid = oid
33 self.oid = oid
34 self.obj_size = obj_size
34 self.obj_size = obj_size
35 self.obj_data = obj_data
35 self.obj_data = obj_data
36 self.obj_href = obj_href
36 self.obj_href = obj_href
37 self.obj_verify_href = obj_verify_href
37 self.obj_verify_href = obj_verify_href
38
38
39 def get_store(self, mode=None):
39 def get_store(self, mode=None):
40 return self.current_store
40 return self.current_store
41
41
42 def get_auth(self):
42 def get_auth(self):
43 """returns auth header for re-use in upload/download"""
43 """returns auth header for re-use in upload/download"""
44 return " ".join(self.auth)
44 return " ".join(self.auth)
45
45
46 def download(self):
46 def download(self):
47
47
48 store = self.get_store()
48 store = self.get_store()
49 response = None
49 response = None
50 has_errors = None
50 has_errors = None
51
51
52 if not store.has_oid():
52 if not store.has_oid():
53 # error reply back to client that something is wrong with dl
53 # error reply back to client that something is wrong with dl
54 err_msg = 'object: {} does not exist in store'.format(store.oid)
54 err_msg = 'object: {} does not exist in store'.format(store.oid)
55 has_errors = OrderedDict(
55 has_errors = OrderedDict(
56 error=OrderedDict(
56 error=OrderedDict(
57 code=404,
57 code=404,
58 message=err_msg
58 message=err_msg
59 )
59 )
60 )
60 )
61
61
62 download_action = OrderedDict(
62 download_action = OrderedDict(
63 href=self.obj_href,
63 href=self.obj_href,
64 header=OrderedDict([("Authorization", self.get_auth())])
64 header=OrderedDict([("Authorization", self.get_auth())])
65 )
65 )
66 if not has_errors:
66 if not has_errors:
67 response = OrderedDict(download=download_action)
67 response = OrderedDict(download=download_action)
68 return response, has_errors
68 return response, has_errors
69
69
70 def upload(self, skip_existing=True):
70 def upload(self, skip_existing=True):
71 """
71 """
72 Write upload action for git-lfs server
72 Write upload action for git-lfs server
73 """
73 """
74
74
75 store = self.get_store()
75 store = self.get_store()
76 response = None
76 response = None
77 has_errors = None
77 has_errors = None
78
78
79 # verify if we have the OID before, if we do, reply with empty
79 # verify if we have the OID before, if we do, reply with empty
80 if store.has_oid():
80 if store.has_oid():
81 log.debug('LFS: store already has oid %s', store.oid)
81 log.debug('LFS: store already has oid %s', store.oid)
82
82
83 # validate size
83 # validate size
84 size_match = store.size_oid() == self.obj_size
84 store_size = store.size_oid()
85 size_match = store_size == self.obj_size
85 if not size_match:
86 if not size_match:
86 log.warning('LFS: size mismatch for oid:%s', self.oid)
87 log.warning(
88 'LFS: size mismatch for oid:%s, in store:%s expected: %s',
89 self.oid, store_size, self.obj_size)
87 elif skip_existing:
90 elif skip_existing:
88 log.debug('LFS: skipping further action as oid is existing')
91 log.debug('LFS: skipping further action as oid is existing')
89 return response, has_errors
92 return response, has_errors
90
93
91 chunked = ("Transfer-Encoding", "chunked")
94 chunked = ("Transfer-Encoding", "chunked")
92 upload_action = OrderedDict(
95 upload_action = OrderedDict(
93 href=self.obj_href,
96 href=self.obj_href,
94 header=OrderedDict([("Authorization", self.get_auth()), chunked])
97 header=OrderedDict([("Authorization", self.get_auth()), chunked])
95 )
98 )
96 if not has_errors:
99 if not has_errors:
97 response = OrderedDict(upload=upload_action)
100 response = OrderedDict(upload=upload_action)
98 # if specified in handler, return the verification endpoint
101 # if specified in handler, return the verification endpoint
99 if self.obj_verify_href:
102 if self.obj_verify_href:
100 verify_action = OrderedDict(
103 verify_action = OrderedDict(
101 href=self.obj_verify_href,
104 href=self.obj_verify_href,
102 header=OrderedDict([("Authorization", self.get_auth())])
105 header=OrderedDict([("Authorization", self.get_auth())])
103 )
106 )
104 response['verify'] = verify_action
107 response['verify'] = verify_action
105 return response, has_errors
108 return response, has_errors
106
109
107 def exec_operation(self, operation, *args, **kwargs):
110 def exec_operation(self, operation, *args, **kwargs):
108 handler = getattr(self, operation)
111 handler = getattr(self, operation)
109 log.debug('LFS: handling request using %s handler', handler)
112 log.debug('LFS: handling request using %s handler', handler)
110 return handler(*args, **kwargs)
113 return handler(*args, **kwargs)
111
114
112
115
113 class LFSOidStore(object):
116 class LFSOidStore(object):
114
117
115 def __init__(self, oid, repo, store_location=None):
118 def __init__(self, oid, repo, store_location=None):
116 self.oid = oid
119 self.oid = oid
117 self.repo = repo
120 self.repo = repo
118 self.store_path = store_location or self.get_default_store()
121 self.store_path = store_location or self.get_default_store()
119 self.tmp_oid_path = os.path.join(self.store_path, oid + '.tmp')
122 self.tmp_oid_path = os.path.join(self.store_path, oid + '.tmp')
120 self.oid_path = os.path.join(self.store_path, oid)
123 self.oid_path = os.path.join(self.store_path, oid)
121 self.fd = None
124 self.fd = None
122
125
123 def get_engine(self, mode):
126 def get_engine(self, mode):
124 """
127 """
125 engine = .get_engine(mode='wb')
128 engine = .get_engine(mode='wb')
126 with engine as f:
129 with engine as f:
127 f.write('...')
130 f.write('...')
128 """
131 """
129
132
130 class StoreEngine(object):
133 class StoreEngine(object):
131 def __init__(self, mode, store_path, oid_path, tmp_oid_path):
134 def __init__(self, mode, store_path, oid_path, tmp_oid_path):
132 self.mode = mode
135 self.mode = mode
133 self.store_path = store_path
136 self.store_path = store_path
134 self.oid_path = oid_path
137 self.oid_path = oid_path
135 self.tmp_oid_path = tmp_oid_path
138 self.tmp_oid_path = tmp_oid_path
136
139
137 def __enter__(self):
140 def __enter__(self):
138 if not os.path.isdir(self.store_path):
141 if not os.path.isdir(self.store_path):
139 os.makedirs(self.store_path)
142 os.makedirs(self.store_path)
140
143
141 # TODO(marcink): maybe write metadata here with size/oid ?
144 # TODO(marcink): maybe write metadata here with size/oid ?
142 fd = open(self.tmp_oid_path, self.mode)
145 fd = open(self.tmp_oid_path, self.mode)
143 self.fd = fd
146 self.fd = fd
144 return fd
147 return fd
145
148
146 def __exit__(self, exc_type, exc_value, traceback):
149 def __exit__(self, exc_type, exc_value, traceback):
147 # close tmp file, and rename to final destination
150 # close tmp file, and rename to final destination
148 self.fd.close()
151 self.fd.close()
149 shutil.move(self.tmp_oid_path, self.oid_path)
152 shutil.move(self.tmp_oid_path, self.oid_path)
150
153
151 return StoreEngine(
154 return StoreEngine(
152 mode, self.store_path, self.oid_path, self.tmp_oid_path)
155 mode, self.store_path, self.oid_path, self.tmp_oid_path)
153
156
154 def get_default_store(self):
157 def get_default_store(self):
155 """
158 """
156 Default store, consistent with defaults of Mercurial large files store
159 Default store, consistent with defaults of Mercurial large files store
157 which is /home/username/.cache/largefiles
160 which is /home/username/.cache/largefiles
158 """
161 """
159 user_home = os.path.expanduser("~")
162 user_home = os.path.expanduser("~")
160 return os.path.join(user_home, '.cache', 'lfs-store')
163 return os.path.join(user_home, '.cache', 'lfs-store')
161
164
162 def has_oid(self):
165 def has_oid(self):
163 return os.path.exists(os.path.join(self.store_path, self.oid))
166 return os.path.exists(os.path.join(self.store_path, self.oid))
164
167
165 def size_oid(self):
168 def size_oid(self):
166 size = -1
169 size = -1
167
170
168 if self.has_oid():
171 if self.has_oid():
169 oid = os.path.join(self.store_path, self.oid)
172 oid = os.path.join(self.store_path, self.oid)
170 size = os.stat(oid).st_size
173 size = os.stat(oid).st_size
171
174
172 return size
175 return size
@@ -1,123 +1,141 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2017 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import os
19 import pytest
19 import pytest
20 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
20 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
21
21
22
22
23 @pytest.fixture()
23 @pytest.fixture()
24 def lfs_store(tmpdir):
24 def lfs_store(tmpdir):
25 repo = 'test'
25 repo = 'test'
26 oid = '123456789'
26 oid = '123456789'
27 store = LFSOidStore(oid=oid, repo=repo, store_location=str(tmpdir))
27 store = LFSOidStore(oid=oid, repo=repo, store_location=str(tmpdir))
28 return store
28 return store
29
29
30
30
31 @pytest.fixture()
31 @pytest.fixture()
32 def oid_handler(lfs_store):
32 def oid_handler(lfs_store):
33 store = lfs_store
33 store = lfs_store
34 repo = store.repo
34 repo = store.repo
35 oid = store.oid
35 oid = store.oid
36
36
37 oid_handler = OidHandler(
37 oid_handler = OidHandler(
38 store=store, repo_name=repo, auth=('basic', 'xxxx'),
38 store=store, repo_name=repo, auth=('basic', 'xxxx'),
39 oid=oid,
39 oid=oid,
40 obj_size='1024', obj_data={}, obj_href='http://localhost/handle_oid',
40 obj_size='1024', obj_data={}, obj_href='http://localhost/handle_oid',
41 obj_verify_href='http://localhost/verify')
41 obj_verify_href='http://localhost/verify')
42 return oid_handler
42 return oid_handler
43
43
44
44
45 class TestOidHandler(object):
45 class TestOidHandler(object):
46
46
47 @pytest.mark.parametrize('exec_action', [
47 @pytest.mark.parametrize('exec_action', [
48 'download',
48 'download',
49 'upload',
49 'upload',
50 ])
50 ])
51 def test_exec_action(self, exec_action, oid_handler):
51 def test_exec_action(self, exec_action, oid_handler):
52 handler = oid_handler.exec_operation(exec_action)
52 handler = oid_handler.exec_operation(exec_action)
53 assert handler
53 assert handler
54
54
55 def test_exec_action_undefined(self, oid_handler):
55 def test_exec_action_undefined(self, oid_handler):
56 with pytest.raises(AttributeError):
56 with pytest.raises(AttributeError):
57 oid_handler.exec_operation('wrong')
57 oid_handler.exec_operation('wrong')
58
58
59 def test_download_oid_not_existing(self, oid_handler):
59 def test_download_oid_not_existing(self, oid_handler):
60 response, has_errors = oid_handler.exec_operation('download')
60 response, has_errors = oid_handler.exec_operation('download')
61
61
62 assert response is None
62 assert response is None
63 assert has_errors['error'] == {
63 assert has_errors['error'] == {
64 'code': 404,
64 'code': 404,
65 'message': 'object: 123456789 does not exist in store'}
65 'message': 'object: 123456789 does not exist in store'}
66
66
67 def test_download_oid(self, oid_handler):
67 def test_download_oid(self, oid_handler):
68 store = oid_handler.get_store()
68 store = oid_handler.get_store()
69 if not os.path.isdir(os.path.dirname(store.oid_path)):
69 if not os.path.isdir(os.path.dirname(store.oid_path)):
70 os.makedirs(os.path.dirname(store.oid_path))
70 os.makedirs(os.path.dirname(store.oid_path))
71
71
72 with open(store.oid_path, 'wb') as f:
72 with open(store.oid_path, 'wb') as f:
73 f.write('CONTENT')
73 f.write('CONTENT')
74
74
75 response, has_errors = oid_handler.exec_operation('download')
75 response, has_errors = oid_handler.exec_operation('download')
76
76
77 assert has_errors is None
77 assert has_errors is None
78 assert response['download'] == {
78 assert response['download'] == {
79 'header': {'Authorization': 'basic xxxx'},
79 'header': {'Authorization': 'basic xxxx'},
80 'href': 'http://localhost/handle_oid'
80 'href': 'http://localhost/handle_oid'
81 }
81 }
82
82
83 def test_upload_oid_that_exists(self, oid_handler):
83 def test_upload_oid_that_exists(self, oid_handler):
84 store = oid_handler.get_store()
84 store = oid_handler.get_store()
85 if not os.path.isdir(os.path.dirname(store.oid_path)):
85 if not os.path.isdir(os.path.dirname(store.oid_path)):
86 os.makedirs(os.path.dirname(store.oid_path))
86 os.makedirs(os.path.dirname(store.oid_path))
87
87
88 with open(store.oid_path, 'wb') as f:
88 with open(store.oid_path, 'wb') as f:
89 f.write('CONTENT')
89 f.write('CONTENT')
90
90 oid_handler.obj_size = 7
91 response, has_errors = oid_handler.exec_operation('upload')
91 response, has_errors = oid_handler.exec_operation('upload')
92 assert has_errors is None
92 assert has_errors is None
93 assert response is None
93 assert response is None
94
94
95 def test_upload_oid_that_exists_but_has_wrong_size(self, oid_handler):
96 store = oid_handler.get_store()
97 if not os.path.isdir(os.path.dirname(store.oid_path)):
98 os.makedirs(os.path.dirname(store.oid_path))
99
100 with open(store.oid_path, 'wb') as f:
101 f.write('CONTENT')
102
103 oid_handler.obj_size = 10240
104 response, has_errors = oid_handler.exec_operation('upload')
105 assert has_errors is None
106 assert response['upload'] == {
107 'header': {'Authorization': 'basic xxxx',
108 'Transfer-Encoding': 'chunked'},
109 'href': 'http://localhost/handle_oid',
110 }
111
95 def test_upload_oid(self, oid_handler):
112 def test_upload_oid(self, oid_handler):
96 response, has_errors = oid_handler.exec_operation('upload')
113 response, has_errors = oid_handler.exec_operation('upload')
97 assert has_errors is None
114 assert has_errors is None
98 assert response['upload'] == {
115 assert response['upload'] == {
99 'header': {'Authorization': 'basic xxxx'},
116 'header': {'Authorization': 'basic xxxx',
117 'Transfer-Encoding': 'chunked'},
100 'href': 'http://localhost/handle_oid'
118 'href': 'http://localhost/handle_oid'
101 }
119 }
102
120
103
121
104 class TestLFSStore(object):
122 class TestLFSStore(object):
105 def test_write_oid(self, lfs_store):
123 def test_write_oid(self, lfs_store):
106 oid_location = lfs_store.oid_path
124 oid_location = lfs_store.oid_path
107
125
108 assert not os.path.isfile(oid_location)
126 assert not os.path.isfile(oid_location)
109
127
110 engine = lfs_store.get_engine(mode='wb')
128 engine = lfs_store.get_engine(mode='wb')
111 with engine as f:
129 with engine as f:
112 f.write('CONTENT')
130 f.write('CONTENT')
113
131
114 assert os.path.isfile(oid_location)
132 assert os.path.isfile(oid_location)
115
133
116 def test_detect_has_oid(self, lfs_store):
134 def test_detect_has_oid(self, lfs_store):
117
135
118 assert lfs_store.has_oid() is False
136 assert lfs_store.has_oid() is False
119 engine = lfs_store.get_engine(mode='wb')
137 engine = lfs_store.get_engine(mode='wb')
120 with engine as f:
138 with engine as f:
121 f.write('CONTENT')
139 f.write('CONTENT')
122
140
123 assert lfs_store.has_oid() is True No newline at end of file
141 assert lfs_store.has_oid() is True
General Comments 0
You need to be logged in to leave comments. Login now