Deduplicated file hash calculation method

boards/models/attachment/__init__.py

@@ -1,71 +1,63 b''
 import hashlib
 import os
 import time
 
 from random import random
 
 from django.db import models
+from boards import utils
 
 from boards.models.attachment.viewers import get_viewers, AbstractViewer
 
 
 FILES_DIRECTORY = 'files/'
 FILE_EXTENSION_DELIMITER = '.'
 
 
 class AttachmentManager(models.Manager):
     def create_with_hash(self, file):
-        file_hash = self.get_hash(file)
+        file_hash = utils.get_file_hash(file)
         existing = self.filter(hash=file_hash)
         if len(existing) > 0:
             attachment = existing[0]
         else:
             file_type = file.name.split(FILE_EXTENSION_DELIMITER)[-1].lower()
-            attachment = Attachment.objects.create(file=file,
-                    mimetype=file_type, hash=file_hash)
+            attachment = Attachment.objects.create(
+                file=file, mimetype=file_type, hash=file_hash)
 
         return attachment
 
-    def get_hash(self, file):
-        """
-        Gets hash of an file.
-        """
-        md5 = hashlib.md5()
-        for chunk in file.chunks():
-            md5.update(chunk)
-        return md5.hexdigest()
-
 
 class Attachment(models.Model):
     objects = AttachmentManager()
 
     # TODO Dedup the method
     def _update_filename(self, filename):
         """
         Gets unique filename
         """
 
         # TODO Use something other than random number in file name
         new_name = '{}{}.{}'.format(
             str(int(time.mktime(time.gmtime()))),
             str(int(random() * 1000)),
             filename.split(FILE_EXTENSION_DELIMITER)[-1:][0])
 
         return os.path.join(FILES_DIRECTORY, new_name)
 
     file = models.FileField(upload_to=_update_filename)
     mimetype = models.CharField(max_length=50)
     hash = models.CharField(max_length=36)
 
     def get_view(self):
         file_viewer = None
         for viewer in get_viewers():
             if viewer.supports(self.mimetype):
                 file_viewer = viewer(self.file, self.mimetype)
                 break
         if file_viewer is None:
             file_viewer = AbstractViewer(self.file, self.mimetype)
 
         return file_viewer.get_view()
 
 

boards/models/image.py

@@ -1,121 +1,112 b''
 import hashlib
 import os
 from random import random
 import time
 
 from django.db import models
 from django.template.defaultfilters import filesizeformat
 
-from boards import thumbs
+from boards import thumbs, utils
 import boards
 from boards.models.base import Viewable
 
 __author__ = 'neko259'
 
 
 IMAGE_THUMB_SIZE = (200, 150)
 IMAGES_DIRECTORY = 'images/'
 FILE_EXTENSION_DELIMITER = '.'
 HASH_LENGTH = 36
 
 CSS_CLASS_IMAGE = 'image'
 CSS_CLASS_THUMB = 'thumb'
 
 
 class PostImageManager(models.Manager):
     def create_with_hash(self, image):
-        image_hash = self.get_hash(image)
+        image_hash = utils.get_file_hash(image)
         existing = self.filter(hash=image_hash)
         if len(existing) > 0:
             post_image = existing[0]
         else:
             post_image = PostImage.objects.create(image=image)
 
         return post_image
 
-    def get_hash(self, image):
-        """
-        Gets hash of an image.
-        """
-        md5 = hashlib.md5()
-        for chunk in image.chunks():
-            md5.update(chunk)
-        return md5.hexdigest()
-
     def get_random_images(self, count, include_archived=False, tags=None):
         images = self.filter(post_images__thread__archived=include_archived)
         if tags is not None:
             images = images.filter(post_images__threads__tags__in=tags)
         return images.order_by('?')[:count]
 
 
 class PostImage(models.Model, Viewable):
     objects = PostImageManager()
 
     class Meta:
         app_label = 'boards'
         ordering = ('id',)
 
     def _update_image_filename(self, filename):
         """
         Gets unique image filename
         """
 
         # TODO Use something other than random number in file name
         new_name = '{}{}.{}'.format(
             str(int(time.mktime(time.gmtime()))),
             str(int(random() * 1000)),
             filename.split(FILE_EXTENSION_DELIMITER)[-1:][0])
 
         return os.path.join(IMAGES_DIRECTORY, new_name)
 
     width = models.IntegerField(default=0)
     height = models.IntegerField(default=0)
 
     pre_width = models.IntegerField(default=0)
     pre_height = models.IntegerField(default=0)
 
     image = thumbs.ImageWithThumbsField(upload_to=_update_image_filename,
                                         blank=True, sizes=(IMAGE_THUMB_SIZE,),
                                         width_field='width',
                                         height_field='height',
                                         preview_width_field='pre_width',
                                         preview_height_field='pre_height')
     hash = models.CharField(max_length=HASH_LENGTH)
 
     def save(self, *args, **kwargs):
         """
         Saves the model and computes the image hash for deduplication purposes.
         """
 
         if not self.pk and self.image:
-            self.hash = PostImage.objects.get_hash(self.image)
+            self.hash = utils.get_file_hash(self.image)
         super(PostImage, self).save(*args, **kwargs)
 
     def __str__(self):
         return self.image.url
 
     def get_view(self):
         metadata = '{}, {}'.format(self.image.name.split('.')[-1],
                                    filesizeformat(self.image.size))
         return '<div class="{}">' \
                '<a class="{}" href="{full}">' \
                '<img class="post-image-preview"' \
                ' src="{}"' \
                ' alt="{}"' \
                ' width="{}"' \
                ' height="{}"' \
                ' data-width="{}"' \
                ' data-height="{}" />' \
                '</a>' \
                '<div class="image-metadata">{image_meta}</div>' \
                '</div>'\
             .format(CSS_CLASS_IMAGE, CSS_CLASS_THUMB,
                     self.image.url_200x150,
                     str(self.hash), str(self.pre_width),
                     str(self.pre_height), str(self.width), str(self.height),
                     full=self.image.url, image_meta=metadata)
 
     def get_random_associated_post(self):
         posts = boards.models.Post.objects.filter(images__in=[self])
         return posts.order_by('?').first()

boards/utils.py

@@ -1,84 +1,92 b''
 """
 This module contains helper functions and helper classes.
 """
+import hashlib
 import time
 import hmac
 
 from django.core.cache import cache
 from django.db.models import Model
 
 from django.utils import timezone
 
 from neboard import settings
 
 
 CACHE_KEY_DELIMITER = '_'
 PERMISSION_MODERATE = 'moderation'
 
 def get_client_ip(request):
     x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
     if x_forwarded_for:
         ip = x_forwarded_for.split(',')[-1].strip()
     else:
         ip = request.META.get('REMOTE_ADDR')
     return ip
 
 
 # TODO The output format is not epoch because it includes microseconds
 def datetime_to_epoch(datetime):
     return int(time.mktime(timezone.localtime(
         datetime,timezone.get_current_timezone()).timetuple())
                * 1000000 + datetime.microsecond)
 
 
 def get_websocket_token(user_id='', timestamp=''):
     """
     Create token to validate information provided by new connection.
     """
 
     sign = hmac.new(settings.CENTRIFUGE_PROJECT_SECRET.encode())
     sign.update(settings.CENTRIFUGE_PROJECT_ID.encode())
     sign.update(user_id.encode())
     sign.update(timestamp.encode())
     token = sign.hexdigest()
 
     return token
 
 
 def cached_result(key_method=None):
     """
     Caches method result in the Django's cache system, persisted by object name,
     object name and model id if object is a Django model.
     """
     def _cached_result(function):
         def inner_func(obj, *args, **kwargs):
             # TODO Include method arguments to the cache key
             cache_key_params = [obj.__class__.__name__, function.__name__]
             if isinstance(obj, Model):
                 cache_key_params.append(str(obj.id))
 
             if key_method is not None:
                 cache_key_params += [str(arg) for arg in key_method(obj)]
 
             cache_key = CACHE_KEY_DELIMITER.join(cache_key_params)
 
             persisted_result = cache.get(cache_key)
             if persisted_result is not None:
                 result = persisted_result
             else:
                 result = function(obj, *args, **kwargs)
                 cache.set(cache_key, result)
 
             return result
 
         return inner_func
     return _cached_result
 
 
 def is_moderator(request):
     try:
         moderate = request.user.has_perm(PERMISSION_MODERATE)
     except AttributeError:
         moderate = False
 
-    return moderate
  No newline at end of file
+    return moderate
+
+
+def get_file_hash(file) -> str:
+    md5 = hashlib.md5()
+    for chunk in file.chunks():
+        md5.update(chunk)
+    return md5.hexdigest()