file-store: always calculate sha256 and metadata.

rhodecode/apps/file_store/local_store.py

@@ -1,188 +1,211 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import time
 import shutil
+import hashlib
 
 from rhodecode.lib.ext_json import json
 from rhodecode.apps.file_store import utils
 from rhodecode.apps.file_store.extensions import resolve_extensions
 from rhodecode.apps.file_store.exceptions import FileNotAllowedException
 
 METADATA_VER = 'v1'
 
 
 class LocalFileStorage(object):
 
     @classmethod
     def resolve_name(cls, name, directory):
         """
         Resolves a unique name and the correct path. If a filename
         for that path already exists then a numeric prefix with values > 0 will be
         added, for example test.jpg -> test-1.jpg etc. initially file would have 0 prefix.
 
         :param name: base name of file
         :param directory: absolute directory path
         """
 
         basename, ext = os.path.splitext(name)
         counter = 0
         while True:
             name = '%s-%d%s' % (basename, counter, ext)
 
             # sub_store prefix to optimize disk usage, e.g some_path/ab/final_file
             sub_store = cls._sub_store_from_filename(basename)
             sub_store_path = os.path.join(directory, sub_store)
             if not os.path.exists(sub_store_path):
                 os.makedirs(sub_store_path)
 
             path = os.path.join(sub_store_path, name)
             if not os.path.exists(path):
                 return name, path
             counter += 1
 
     @classmethod
     def _sub_store_from_filename(cls, filename):
         return filename[:2]
 
+    @classmethod
+    def calculate_path_hash(cls, file_path):
+        """
+        Efficient calculation of file_path sha256 sum
+
+        :param file_path:
+        :return: sha256sum
+        """
+        digest = hashlib.sha256()
+        with open(file_path, 'rb') as f:
+            for chunk in iter(lambda: f.read(1024 * 100), b""):
+                digest.update(chunk)
+
+        return digest.hexdigest()
+
     def __init__(self, base_path, extension_groups=None):
 
         """
         Local file storage
 
         :param base_path: the absolute base path where uploads are stored
         :param extension_groups: extensions string
         """
 
         extension_groups = extension_groups or ['any']
         self.base_path = base_path
         self.extensions = resolve_extensions([], groups=extension_groups)
 
     def store_path(self, filename):
         """
         Returns absolute file path of the filename, joined to the
         base_path.
 
         :param filename: base name of file
         """
         sub_store = self._sub_store_from_filename(filename)
         return os.path.join(self.base_path, sub_store, filename)
 
     def delete(self, filename):
         """
         Deletes the filename. Filename is resolved with the
         absolute path based on base_path. If file does not exist,
         returns **False**, otherwise **True**
 
         :param filename: base name of file
         """
         if self.exists(filename):
             os.remove(self.store_path(filename))
             return True
         return False
 
     def exists(self, filename):
         """
         Checks if file exists. Resolves filename's absolute
         path based on base_path.
 
         :param filename: base name of file
         """
         return os.path.exists(self.store_path(filename))
 
     def filename_allowed(self, filename, extensions=None):
         """Checks if a filename has an allowed extension
 
         :param filename: base name of file
         :param extensions: iterable of extensions (or self.extensions)
         """
         _, ext = os.path.splitext(filename)
         return self.extension_allowed(ext, extensions)
 
     def extension_allowed(self, ext, extensions=None):
         """
         Checks if an extension is permitted. Both e.g. ".jpg" and
         "jpg" can be passed in. Extension lookup is case-insensitive.
 
         :param ext: extension to check
         :param extensions: iterable of extensions to validate against (or self.extensions)
         """
 
         extensions = extensions or self.extensions
         if not extensions:
             return True
         if ext.startswith('.'):
             ext = ext[1:]
         return ext.lower() in extensions
 
     def save_file(self, file_obj, filename, directory=None, extensions=None,
-                  metadata=None, **kwargs):
+                  extra_metadata=None, **kwargs):
         """
         Saves a file object to the uploads location.
         Returns the resolved filename, i.e. the directory +
         the (randomized/incremented) base name.
 
         :param file_obj: **cgi.FieldStorage** object (or similar)
         :param filename: original filename
         :param directory: relative path of sub-directory
         :param extensions: iterable of allowed extensions, if not default
-        :param metadata: JSON metadata to store next to the file with .meta suffix
-        :returns: modified filename
+        :param extra_metadata: extra JSON metadata to store next to the file with .meta suffix
         """
 
         extensions = extensions or self.extensions
 
         if not self.filename_allowed(filename, extensions):
             raise FileNotAllowedException()
 
         if directory:
             dest_directory = os.path.join(self.base_path, directory)
         else:
             dest_directory = self.base_path
 
         if not os.path.exists(dest_directory):
             os.makedirs(dest_directory)
 
         filename = utils.uid_filename(filename)
 
+        # resolve also produces special sub-dir for file optimized store
         filename, path = self.resolve_name(filename, dest_directory)
+        stored_file_dir = os.path.dirname(path)
 
         file_obj.seek(0)
 
         with open(path, "wb") as dest:
             shutil.copyfileobj(file_obj, dest)
 
-        if metadata:
+        metadata = {}
+        if extra_metadata:
+            metadata = extra_metadata
+
             size = os.stat(path).st_size
+        file_hash = self.calculate_path_hash(path)
+
             metadata.update(
-                {"size": size,
+            {"filename": filename,
+             "size": size,
                  "time": time.time(),
+             "sha256": file_hash,
                  "meta_ver": METADATA_VER})
 
-            stored_file_path = os.path.dirname(path)
             filename_meta = filename + '.meta'
-            with open(os.path.join(stored_file_path, filename_meta), "wb") as dest_meta:
+        with open(os.path.join(stored_file_dir, filename_meta), "wb") as dest_meta:
                 dest_meta.write(json.dumps(metadata))
 
         if directory:
             filename = os.path.join(directory, filename)
 
         return filename, metadata

rhodecode/apps/file_store/tests/test_upload_file.py

@@ -1,111 +1,111 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import os
 import pytest
 
 from rhodecode.lib.ext_json import json
 from rhodecode.tests import TestController
 from rhodecode.apps.file_store import utils, config_keys
 
 
 def route_path(name, params=None, **kwargs):
     import urllib
 
     base_url = {
         'upload_file': '/_file_store/upload',
         'download_file': '/_file_store/download/{fid}',
 
     }[name].format(**kwargs)
 
     if params:
         base_url = '{}?{}'.format(base_url, urllib.urlencode(params))
     return base_url
 
 
 class TestFileStoreViews(TestController):
 
     @pytest.mark.parametrize("fid, content, exists", [
         ('abcde-0.jpg', "xxxxx", True),
         ('abcde-0.exe', "1234567", True),
         ('abcde-0.jpg', "xxxxx", False),
     ])
     def test_get_files_from_store(self, fid, content, exists, tmpdir):
         self.log_user()
         store_path = self.app._pyramid_settings[config_keys.store_path]
 
         if exists:
             status = 200
             store = utils.get_file_storage({config_keys.store_path: store_path})
             filesystem_file = os.path.join(str(tmpdir), fid)
             with open(filesystem_file, 'wb') as f:
                 f.write(content)
 
             with open(filesystem_file, 'rb') as f:
-                fid, metadata = store.save_file(f, fid, metadata={'filename': fid})
+                fid, metadata = store.save_file(f, fid, extra_metadata={'filename': fid})
 
         else:
             status = 404
 
         response = self.app.get(route_path('download_file', fid=fid), status=status)
 
         if exists:
             assert response.text == content
             file_store_path = os.path.dirname(store.resolve_name(fid, store_path)[1])
             metadata_file = os.path.join(file_store_path, fid + '.meta')
             assert os.path.exists(metadata_file)
             with open(metadata_file, 'rb') as f:
                 json_data = json.loads(f.read())
 
             assert json_data
             assert 'size' in json_data
 
     def test_upload_files_without_content_to_store(self):
         self.log_user()
         response = self.app.post(
             route_path('upload_file'),
             params={'csrf_token': self.csrf_token},
             status=200)
 
         assert response.json == {
             u'error': u'store_file data field is missing',
             u'access_path': None,
             u'store_fid': None}
 
     def test_upload_files_bogus_content_to_store(self):
         self.log_user()
         response = self.app.post(
             route_path('upload_file'),
             params={'csrf_token': self.csrf_token, 'store_file': 'bogus'},
             status=200)
 
         assert response.json == {
             u'error': u'filename cannot be read from the data field',
             u'access_path': None,
             u'store_fid': None}
 
     def test_upload_content_to_store(self):
         self.log_user()
         response = self.app.post(
             route_path('upload_file'),
             upload_files=[('store_file', 'myfile.txt', 'SOME CONTENT')],
             params={'csrf_token': self.csrf_token},
             status=200)
 
         assert response.json['store_fid']

rhodecode/apps/file_store/views.py

@@ -1,96 +1,95 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import logging
 
 from pyramid.view import view_config
 from pyramid.response import FileResponse
 from pyramid.httpexceptions import HTTPFound, HTTPNotFound
 
 from rhodecode.apps._base import BaseAppView
 from rhodecode.apps.file_store import utils
 from rhodecode.apps.file_store.exceptions import (
     FileNotAllowedException,FileOverSizeException)
 
 from rhodecode.lib import helpers as h
 from rhodecode.lib import audit_logger
 from rhodecode.lib.auth import (CSRFRequired, NotAnonymous)
 
 log = logging.getLogger(__name__)
 
 
 class FileStoreView(BaseAppView):
     upload_key = 'store_file'
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         self.storage = utils.get_file_storage(self.request.registry.settings)
         return c
 
     @NotAnonymous()
     @CSRFRequired()
     @view_config(route_name='upload_file', request_method='POST', renderer='json_ext')
     def upload_file(self):
         self.load_default_context()
         file_obj = self.request.POST.get(self.upload_key)
 
         if file_obj is None:
             return {'store_fid': None,
                     'access_path': None,
                     'error': '{} data field is missing'.format(self.upload_key)}
 
         if not hasattr(file_obj, 'filename'):
             return {'store_fid': None,
                     'access_path': None,
                     'error': 'filename cannot be read from the data field'}
 
         filename = file_obj.filename
 
         metadata = {
-            'filename': filename,
-            'size': '',  # filled by save_file
             'user_uploaded': {'username': self._rhodecode_user.username,
                               'user_id': self._rhodecode_user.user_id,
                               'ip': self._rhodecode_user.ip_addr}}
         try:
-            store_fid, metadata = self.storage.save_file(file_obj.file, filename, metadata=metadata)
+            store_fid, metadata = self.storage.save_file(
+                file_obj.file, filename, extra_metadata=metadata)
         except FileNotAllowedException:
             return {'store_fid': None,
                     'access_path': None,
                     'error': 'File {} is not allowed.'.format(filename)}
 
         except FileOverSizeException:
             return {'store_fid': None,
                     'access_path': None,
                     'error': 'File {} is exceeding allowed limit.'.format(filename)}
 
         return {'store_fid': store_fid,
                 'access_path': h.route_path('download_file', fid=store_fid)}
 
     @view_config(route_name='download_file')
     def download_file(self):
         self.load_default_context()
         file_uid = self.request.matchdict['fid']
         log.debug('Requesting FID:%s from store %s', file_uid, self.storage)
         if not self.storage.exists(file_uid):
             log.debug('File with FID:%s not found in the store', file_uid)
             raise HTTPNotFound()
 
         file_path = self.storage.store_path(file_uid)
         return FileResponse(file_path)