##// END OF EJS Templates
git-lfs: always validate uploaded files size....
marcink -
r197:e75ea5c6 default
parent child Browse files
Show More
@@ -1,166 +1,171 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2017 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import os
19 import shutil
19 import shutil
20 import logging
20 import logging
21 from collections import OrderedDict
21 from collections import OrderedDict
22
22
23 log = logging.getLogger(__name__)
23 log = logging.getLogger(__name__)
24
24
25
25
26 class OidHandler(object):
26 class OidHandler(object):
27
27
28 def __init__(self, store, repo_name, auth, oid, obj_size, obj_data, obj_href,
28 def __init__(self, store, repo_name, auth, oid, obj_size, obj_data, obj_href,
29 obj_verify_href=None):
29 obj_verify_href=None):
30 self.current_store = store
30 self.current_store = store
31 self.repo_name = repo_name
31 self.repo_name = repo_name
32 self.auth = auth
32 self.auth = auth
33 self.oid = oid
33 self.oid = oid
34 self.obj_size = obj_size
34 self.obj_size = obj_size
35 self.obj_data = obj_data
35 self.obj_data = obj_data
36 self.obj_href = obj_href
36 self.obj_href = obj_href
37 self.obj_verify_href = obj_verify_href
37 self.obj_verify_href = obj_verify_href
38
38
39 def get_store(self, mode=None):
39 def get_store(self, mode=None):
40 return self.current_store
40 return self.current_store
41
41
42 def get_auth(self):
42 def get_auth(self):
43 """returns auth header for re-use in upload/download"""
43 """returns auth header for re-use in upload/download"""
44 return " ".join(self.auth)
44 return " ".join(self.auth)
45
45
46 def download(self):
46 def download(self):
47
47
48 store = self.get_store()
48 store = self.get_store()
49 response = None
49 response = None
50 has_errors = None
50 has_errors = None
51
51
52 if not store.has_oid():
52 if not store.has_oid():
53 # error reply back to client that something is wrong with dl
53 # error reply back to client that something is wrong with dl
54 err_msg = 'object: {} does not exist in store'.format(store.oid)
54 err_msg = 'object: {} does not exist in store'.format(store.oid)
55 has_errors = OrderedDict(
55 has_errors = OrderedDict(
56 error=OrderedDict(
56 error=OrderedDict(
57 code=404,
57 code=404,
58 message=err_msg
58 message=err_msg
59 )
59 )
60 )
60 )
61
61
62 download_action = OrderedDict(
62 download_action = OrderedDict(
63 href=self.obj_href,
63 href=self.obj_href,
64 header=OrderedDict([("Authorization", self.get_auth())])
64 header=OrderedDict([("Authorization", self.get_auth())])
65 )
65 )
66 if not has_errors:
66 if not has_errors:
67 response = OrderedDict(download=download_action)
67 response = OrderedDict(download=download_action)
68 return response, has_errors
68 return response, has_errors
69
69
70 def upload(self, skip_existing=True):
70 def upload(self, skip_existing=True):
71 """
71 """
72 Write upload action for git-lfs server
72 Write upload action for git-lfs server
73 """
73 """
74
74
75 store = self.get_store()
75 store = self.get_store()
76 response = None
76 response = None
77 has_errors = None
77 has_errors = None
78
78
79 # verify if we have the OID before, if we do, reply with empty
79 # verify if we have the OID before, if we do, reply with empty
80 if store.has_oid():
80 if store.has_oid():
81 log.debug('LFS: store already has oid %s', store.oid)
81 log.debug('LFS: store already has oid %s', store.oid)
82 if skip_existing:
82
83 # validate size
84 size_match = store.size_oid() == self.obj_size
85 if not size_match:
86 log.warning('LFS: size mismatch for oid:%s', self.oid)
87 elif skip_existing:
83 log.debug('LFS: skipping further action as oid is existing')
88 log.debug('LFS: skipping further action as oid is existing')
84 return response, has_errors
89 return response, has_errors
85
90
86 upload_action = OrderedDict(
91 upload_action = OrderedDict(
87 href=self.obj_href,
92 href=self.obj_href,
88 header=OrderedDict([("Authorization", self.get_auth())])
93 header=OrderedDict([("Authorization", self.get_auth())])
89 )
94 )
90 if not has_errors:
95 if not has_errors:
91 response = OrderedDict(upload=upload_action)
96 response = OrderedDict(upload=upload_action)
92 # if specified in handler, return the verification endpoint
97 # if specified in handler, return the verification endpoint
93 if self.obj_verify_href:
98 if self.obj_verify_href:
94 verify_action = OrderedDict(
99 verify_action = OrderedDict(
95 href=self.obj_verify_href,
100 href=self.obj_verify_href,
96 header=OrderedDict([("Authorization", self.get_auth())])
101 header=OrderedDict([("Authorization", self.get_auth())])
97 )
102 )
98 response['verify'] = verify_action
103 response['verify'] = verify_action
99 return response, has_errors
104 return response, has_errors
100
105
101 def exec_operation(self, operation, *args, **kwargs):
106 def exec_operation(self, operation, *args, **kwargs):
102 handler = getattr(self, operation)
107 handler = getattr(self, operation)
103 log.debug('LFS: handling request using %s handler', handler)
108 log.debug('LFS: handling request using %s handler', handler)
104 return handler(*args, **kwargs)
109 return handler(*args, **kwargs)
105
110
106
111
107 class LFSOidStore(object):
112 class LFSOidStore(object):
108
113
109 def __init__(self, oid, repo, store_location=None):
114 def __init__(self, oid, repo, store_location=None):
110 self.oid = oid
115 self.oid = oid
111 self.repo = repo
116 self.repo = repo
112 self.store_path = store_location or self.get_default_store()
117 self.store_path = store_location or self.get_default_store()
113 self.tmp_oid_path = os.path.join(self.store_path, oid + '.tmp')
118 self.tmp_oid_path = os.path.join(self.store_path, oid + '.tmp')
114 self.oid_path = os.path.join(self.store_path, oid)
119 self.oid_path = os.path.join(self.store_path, oid)
115 self.fd = None
120 self.fd = None
116
121
117 def get_engine(self, mode):
122 def get_engine(self, mode):
118 """
123 """
119 engine = .get_engine(mode='wb')
124 engine = .get_engine(mode='wb')
120 with engine as f:
125 with engine as f:
121 f.write('...')
126 f.write('...')
122 """
127 """
123
128
124 class StoreEngine(object):
129 class StoreEngine(object):
125 def __init__(self, mode, store_path, oid_path, tmp_oid_path):
130 def __init__(self, mode, store_path, oid_path, tmp_oid_path):
126 self.mode = mode
131 self.mode = mode
127 self.store_path = store_path
132 self.store_path = store_path
128 self.oid_path = oid_path
133 self.oid_path = oid_path
129 self.tmp_oid_path = tmp_oid_path
134 self.tmp_oid_path = tmp_oid_path
130
135
131 def __enter__(self):
136 def __enter__(self):
132 if not os.path.isdir(self.store_path):
137 if not os.path.isdir(self.store_path):
133 os.makedirs(self.store_path)
138 os.makedirs(self.store_path)
134
139
135 # TODO(marcink): maybe write metadata here with size/oid ?
140 # TODO(marcink): maybe write metadata here with size/oid ?
136 fd = open(self.tmp_oid_path, self.mode)
141 fd = open(self.tmp_oid_path, self.mode)
137 self.fd = fd
142 self.fd = fd
138 return fd
143 return fd
139
144
140 def __exit__(self, exc_type, exc_value, traceback):
145 def __exit__(self, exc_type, exc_value, traceback):
141 # close tmp file, and rename to final destination
146 # close tmp file, and rename to final destination
142 self.fd.close()
147 self.fd.close()
143 shutil.move(self.tmp_oid_path, self.oid_path)
148 shutil.move(self.tmp_oid_path, self.oid_path)
144
149
145 return StoreEngine(
150 return StoreEngine(
146 mode, self.store_path, self.oid_path, self.tmp_oid_path)
151 mode, self.store_path, self.oid_path, self.tmp_oid_path)
147
152
148 def get_default_store(self):
153 def get_default_store(self):
149 """
154 """
150 Default store, consistent with defaults of Mercurial large files store
155 Default store, consistent with defaults of Mercurial large files store
151 which is /home/username/.cache/largefiles
156 which is /home/username/.cache/largefiles
152 """
157 """
153 user_home = os.path.expanduser("~")
158 user_home = os.path.expanduser("~")
154 return os.path.join(user_home, '.cache', 'lfs-store')
159 return os.path.join(user_home, '.cache', 'lfs-store')
155
160
156 def has_oid(self):
161 def has_oid(self):
157 return os.path.exists(os.path.join(self.store_path, self.oid))
162 return os.path.exists(os.path.join(self.store_path, self.oid))
158
163
159 def size_oid(self):
164 def size_oid(self):
160 size = -1
165 size = -1
161
166
162 if self.has_oid():
167 if self.has_oid():
163 oid = os.path.join(self.store_path, self.oid)
168 oid = os.path.join(self.store_path, self.oid)
164 size = os.stat(oid).st_size
169 size = os.stat(oid).st_size
165
170
166 return size
171 return size
General Comments 0
You need to be logged in to leave comments. Login now