file-store: always calculate sha256 and metadata.

rhodecode/apps/file_store/local_store.py

@@ -21,6 +21,7 b''
 import os
 import time
 import shutil
+import hashlib
 
 from rhodecode.lib.ext_json import json
 from rhodecode.apps.file_store import utils
@@ -63,6 +64,21 b' class LocalFileStorage(object):'
     def _sub_store_from_filename(cls, filename):
         return filename[:2]
 
+    @classmethod
+    def calculate_path_hash(cls, file_path):
+        """
+        Efficient calculation of file_path sha256 sum
+
+        :param file_path:
+        :return: sha256sum
+        """
+        digest = hashlib.sha256()
+        with open(file_path, 'rb') as f:
+            for chunk in iter(lambda: f.read(1024 * 100), b""):
+                digest.update(chunk)
+
+        return digest.hexdigest()
+
     def __init__(self, base_path, extension_groups=None):
 
         """
@@ -134,7 +150,7 b' class LocalFileStorage(object):'
         return ext.lower() in extensions
 
     def save_file(self, file_obj, filename, directory=None, extensions=None,
-                  metadata=None, **kwargs):
+                  extra_metadata=None, **kwargs):
         """
         Saves a file object to the uploads location.
         Returns the resolved filename, i.e. the directory +
@@ -144,8 +160,7 b' class LocalFileStorage(object):'
         :param filename: original filename
         :param directory: relative path of sub-directory
         :param extensions: iterable of allowed extensions, if not default
-        :param metadata: JSON metadata to store next to the file with .meta suffix
-        :returns: modified filename
+        :param extra_metadata: extra JSON metadata to store next to the file with .meta suffix
         """
 
         extensions = extensions or self.extensions
@@ -163,24 +178,32 b' class LocalFileStorage(object):'
 
         filename = utils.uid_filename(filename)
 
+        # resolve also produces special sub-dir for file optimized store
         filename, path = self.resolve_name(filename, dest_directory)
+        stored_file_dir = os.path.dirname(path)
 
         file_obj.seek(0)
 
         with open(path, "wb") as dest:
             shutil.copyfileobj(file_obj, dest)
 
-        if metadata:
-            size = os.stat(path).st_size
-            metadata.update(
-                {"size": size,
-                 "time": time.time(),
-                 "meta_ver": METADATA_VER})
+        metadata = {}
+        if extra_metadata:
+            metadata = extra_metadata
+
+        size = os.stat(path).st_size
+        file_hash = self.calculate_path_hash(path)
 
-            stored_file_path = os.path.dirname(path)
-            filename_meta = filename + '.meta'
-            with open(os.path.join(stored_file_path, filename_meta), "wb") as dest_meta:
-                dest_meta.write(json.dumps(metadata))
+        metadata.update(
+            {"filename": filename,
+             "size": size,
+             "time": time.time(),
+             "sha256": file_hash,
+             "meta_ver": METADATA_VER})
+
+        filename_meta = filename + '.meta'
+        with open(os.path.join(stored_file_dir, filename_meta), "wb") as dest_meta:
+            dest_meta.write(json.dumps(metadata))
 
         if directory:
             filename = os.path.join(directory, filename)

rhodecode/apps/file_store/tests/test_upload_file.py

@@ -58,7 +58,7 b' class TestFileStoreViews(TestController)'
                 f.write(content)
 
             with open(filesystem_file, 'rb') as f:
-                fid, metadata = store.save_file(f, fid, metadata={'filename': fid})
+                fid, metadata = store.save_file(f, fid, extra_metadata={'filename': fid})
 
         else:
             status = 404

rhodecode/apps/file_store/views.py

@@ -63,13 +63,12 b' class FileStoreView(BaseAppView):'
         filename = file_obj.filename
 
         metadata = {
-            'filename': filename,
-            'size': '',  # filled by save_file
             'user_uploaded': {'username': self._rhodecode_user.username,
                               'user_id': self._rhodecode_user.user_id,
                               'ip': self._rhodecode_user.ip_addr}}
         try:
-            store_fid, metadata = self.storage.save_file(file_obj.file, filename, metadata=metadata)
+            store_fid, metadata = self.storage.save_file(
+                file_obj.file, filename, extra_metadata=metadata)
         except FileNotAllowedException:
             return {'store_fid': None,
                     'access_path': None,