##// END OF EJS Templates
file-store: always calculate sha256 and metadata.
marcink -
r3455:4171fa2a default
parent child Browse files
Show More
@@ -21,6 +21,7 b''
21 import os
21 import os
22 import time
22 import time
23 import shutil
23 import shutil
24 import hashlib
24
25
25 from rhodecode.lib.ext_json import json
26 from rhodecode.lib.ext_json import json
26 from rhodecode.apps.file_store import utils
27 from rhodecode.apps.file_store import utils
@@ -63,6 +64,21 b' class LocalFileStorage(object):'
63 def _sub_store_from_filename(cls, filename):
64 def _sub_store_from_filename(cls, filename):
64 return filename[:2]
65 return filename[:2]
65
66
67 @classmethod
68 def calculate_path_hash(cls, file_path):
69 """
70 Efficient calculation of file_path sha256 sum
71
72 :param file_path:
73 :return: sha256sum
74 """
75 digest = hashlib.sha256()
76 with open(file_path, 'rb') as f:
77 for chunk in iter(lambda: f.read(1024 * 100), b""):
78 digest.update(chunk)
79
80 return digest.hexdigest()
81
66 def __init__(self, base_path, extension_groups=None):
82 def __init__(self, base_path, extension_groups=None):
67
83
68 """
84 """
@@ -134,7 +150,7 b' class LocalFileStorage(object):'
134 return ext.lower() in extensions
150 return ext.lower() in extensions
135
151
136 def save_file(self, file_obj, filename, directory=None, extensions=None,
152 def save_file(self, file_obj, filename, directory=None, extensions=None,
137 metadata=None, **kwargs):
153 extra_metadata=None, **kwargs):
138 """
154 """
139 Saves a file object to the uploads location.
155 Saves a file object to the uploads location.
140 Returns the resolved filename, i.e. the directory +
156 Returns the resolved filename, i.e. the directory +
@@ -144,8 +160,7 b' class LocalFileStorage(object):'
144 :param filename: original filename
160 :param filename: original filename
145 :param directory: relative path of sub-directory
161 :param directory: relative path of sub-directory
146 :param extensions: iterable of allowed extensions, if not default
162 :param extensions: iterable of allowed extensions, if not default
147 :param metadata: JSON metadata to store next to the file with .meta suffix
163 :param extra_metadata: extra JSON metadata to store next to the file with .meta suffix
148 :returns: modified filename
149 """
164 """
150
165
151 extensions = extensions or self.extensions
166 extensions = extensions or self.extensions
@@ -163,24 +178,32 b' class LocalFileStorage(object):'
163
178
164 filename = utils.uid_filename(filename)
179 filename = utils.uid_filename(filename)
165
180
181 # resolve also produces special sub-dir for file optimized store
166 filename, path = self.resolve_name(filename, dest_directory)
182 filename, path = self.resolve_name(filename, dest_directory)
183 stored_file_dir = os.path.dirname(path)
167
184
168 file_obj.seek(0)
185 file_obj.seek(0)
169
186
170 with open(path, "wb") as dest:
187 with open(path, "wb") as dest:
171 shutil.copyfileobj(file_obj, dest)
188 shutil.copyfileobj(file_obj, dest)
172
189
173 if metadata:
190 metadata = {}
174 size = os.stat(path).st_size
191 if extra_metadata:
175 metadata.update(
192 metadata = extra_metadata
176 {"size": size,
193
177 "time": time.time(),
194 size = os.stat(path).st_size
178 "meta_ver": METADATA_VER})
195 file_hash = self.calculate_path_hash(path)
179
196
180 stored_file_path = os.path.dirname(path)
197 metadata.update(
181 filename_meta = filename + '.meta'
198 {"filename": filename,
182 with open(os.path.join(stored_file_path, filename_meta), "wb") as dest_meta:
199 "size": size,
183 dest_meta.write(json.dumps(metadata))
200 "time": time.time(),
201 "sha256": file_hash,
202 "meta_ver": METADATA_VER})
203
204 filename_meta = filename + '.meta'
205 with open(os.path.join(stored_file_dir, filename_meta), "wb") as dest_meta:
206 dest_meta.write(json.dumps(metadata))
184
207
185 if directory:
208 if directory:
186 filename = os.path.join(directory, filename)
209 filename = os.path.join(directory, filename)
@@ -58,7 +58,7 b' class TestFileStoreViews(TestController)'
58 f.write(content)
58 f.write(content)
59
59
60 with open(filesystem_file, 'rb') as f:
60 with open(filesystem_file, 'rb') as f:
61 fid, metadata = store.save_file(f, fid, metadata={'filename': fid})
61 fid, metadata = store.save_file(f, fid, extra_metadata={'filename': fid})
62
62
63 else:
63 else:
64 status = 404
64 status = 404
@@ -63,13 +63,12 b' class FileStoreView(BaseAppView):'
63 filename = file_obj.filename
63 filename = file_obj.filename
64
64
65 metadata = {
65 metadata = {
66 'filename': filename,
67 'size': '', # filled by save_file
68 'user_uploaded': {'username': self._rhodecode_user.username,
66 'user_uploaded': {'username': self._rhodecode_user.username,
69 'user_id': self._rhodecode_user.user_id,
67 'user_id': self._rhodecode_user.user_id,
70 'ip': self._rhodecode_user.ip_addr}}
68 'ip': self._rhodecode_user.ip_addr}}
71 try:
69 try:
72 store_fid, metadata = self.storage.save_file(file_obj.file, filename, metadata=metadata)
70 store_fid, metadata = self.storage.save_file(
71 file_obj.file, filename, extra_metadata=metadata)
73 except FileNotAllowedException:
72 except FileNotAllowedException:
74 return {'store_fid': None,
73 return {'store_fid': None,
75 'access_path': None,
74 'access_path': None,
General Comments 0
You need to be logged in to leave comments. Login now