# HG changeset patch # User neko259 # Date 2016-05-21 10:14:24 # Node ID 0eb7ac3c51909d0b4220c24b8e8b532328587354 # Parent c2a3fd807f3491c97d8b1dfaa2fb847397d3012d Store images as regular attachments instead of separate model diff --git a/boards/abstracts/attachment_alias.py b/boards/abstracts/attachment_alias.py --- a/boards/abstracts/attachment_alias.py +++ b/boards/abstracts/attachment_alias.py @@ -1,5 +1,6 @@ from boards.abstracts.settingsmanager import SessionSettingsManager -from boards.models import PostImage +from boards.models import Attachment + class AttachmentAlias: def get_image(alias): @@ -17,7 +18,7 @@ class SessionAttachmentAlias(AttachmentA class ModelAttachmentAlias(AttachmentAlias): def get_image(self, alias): - return PostImage.objects.filter(alias=alias).first() + return Attachment.objects.filter(alias=alias).first() def get_image_by_alias(alias, session): diff --git a/boards/abstracts/constants.py b/boards/abstracts/constants.py new file mode 100644 --- /dev/null +++ b/boards/abstracts/constants.py @@ -0,0 +1,1 @@ +FILE_DIRECTORY = 'files/' diff --git a/boards/admin.py b/boards/admin.py --- a/boards/admin.py +++ b/boards/admin.py @@ -1,8 +1,8 @@ +from boards.models.attachment import FILE_TYPES_IMAGE from django.contrib import admin from django.utils.translation import ugettext_lazy as _ from django.core.urlresolvers import reverse -from django.db.models import F -from boards.models import Post, Tag, Ban, Thread, Banner, PostImage, KeyPair, GlobalId +from boards.models import Post, Tag, Ban, Thread, Banner, Attachment, KeyPair, GlobalId @admin.register(Post) @@ -40,11 +40,11 @@ class PostAdmin(admin.ModelAdmin): self.message_user(request, _('{} posters were banned, {} messages were hidden').format(bans, hidden)) def linked_images(self, obj: Post): - images = obj.images.all() + images = obj.attachments.filter(mimetype__in=FILE_TYPES_IMAGE) image_urls = [''.format( reverse('admin:%s_%s_change' % (image._meta.app_label, image._meta.model_name), - args=[image.id]), image.image.url_200x150) for image in images] + args=[image.id]), image.file.url_200x150) for image in images] return ', '.join(image_urls) linked_images.allow_tags = True @@ -142,8 +142,8 @@ class BannerAdmin(admin.ModelAdmin): list_display = ('title', 'text') -@admin.register(PostImage) -class PostImageAdmin(admin.ModelAdmin): +@admin.register(Attachment) +class AttachmentAdmin(admin.ModelAdmin): search_fields = ('alias',) diff --git a/boards/management/commands/cleanfiles.py b/boards/management/commands/cleanfiles.py --- a/boards/management/commands/cleanfiles.py +++ b/boards/management/commands/cleanfiles.py @@ -1,46 +1,36 @@ import os +from boards.abstracts.constants import FILE_DIRECTORY from django.core.management import BaseCommand from django.db import transaction from boards.models import Attachment -from boards.models.attachment import FILES_DIRECTORY -from boards.models.image import IMAGES_DIRECTORY, PostImage, IMAGE_THUMB_SIZE from neboard.settings import MEDIA_ROOT __author__ = 'neko259' +THUMB_SIZE = (200, 150) + class Command(BaseCommand): help = 'Remove files whose models were deleted' @transaction.atomic def handle(self, *args, **options): - count = 0 - thumb_prefix = '.{}x{}'.format(*IMAGE_THUMB_SIZE) - - model_files = os.listdir(MEDIA_ROOT + IMAGES_DIRECTORY) - for file in model_files: - image_name = file if thumb_prefix not in file else file.replace(thumb_prefix, '') - found = PostImage.objects.filter( - image=IMAGES_DIRECTORY + image_name).exists() - - if not found: - print('Missing {}'.format(image_name)) - os.remove(MEDIA_ROOT + IMAGES_DIRECTORY + file) - count += 1 - print('Deleted {} image files.'.format(count)) + thumb_prefix = '.{}x{}'.format(*THUMB_SIZE) count = 0 - model_files = os.listdir(MEDIA_ROOT + FILES_DIRECTORY) + model_files = os.listdir(MEDIA_ROOT + FILE_DIRECTORY) for file in model_files: - found = Attachment.objects.filter(file=FILES_DIRECTORY + file)\ + model_filename = file if thumb_prefix not in file else file.replace( + thumb_prefix, '') + found = Attachment.objects.filter(file=FILE_DIRECTORY + model_filename)\ .exists() if not found: print('Missing {}'.format(file)) - os.remove(MEDIA_ROOT + FILES_DIRECTORY + file) + os.remove(MEDIA_ROOT + FILE_DIRECTORY + file) count += 1 print('Deleted {} attachment files.'.format(count)) diff --git a/boards/migrations/0001_initial.py b/boards/migrations/0001_initial.py --- a/boards/migrations/0001_initial.py +++ b/boards/migrations/0001_initial.py @@ -2,7 +2,6 @@ from __future__ import unicode_literals from django.db import models, migrations -import boards.models.image import boards.models.base import boards.thumbs diff --git a/boards/migrations/0047_attachment_alias.py b/boards/migrations/0047_attachment_alias.py new file mode 100644 --- /dev/null +++ b/boards/migrations/0047_attachment_alias.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-05-21 07:43 +from __future__ import unicode_literals + +from boards.signals import generate_thumb +from boards.utils import get_extension +from django.db import migrations, models + + +class Migration(migrations.Migration): + + def images_to_attachments(apps, schema_editor): + PostImage = apps.get_model('boards', 'PostImage') + Attachment = apps.get_model('boards', 'Attachment') + + count = 0 + images = PostImage.objects.all() + for image in images: + file_type = get_extension(image.image.name) + attachment = Attachment.objects.create( + file=image.image.file, mimetype=file_type, hash=image.hash, + alias=image.alias) + generate_thumb(attachment) + count += 1 + print('Processed {} of {} images'.format(count, len(images))) + for post in image.post_images.all(): + post.attachments.add(attachment) + + image.image.close() + + dependencies = [ + ('boards', '0046_auto_20160520_2307'), + ] + + operations = [ + migrations.AddField( + model_name='attachment', + name='alias', + field=models.TextField(blank=True, null=True, unique=True), + ), + migrations.RunPython(images_to_attachments), + ] diff --git a/boards/migrations/0048_remove_post_images.py b/boards/migrations/0048_remove_post_images.py new file mode 100644 --- /dev/null +++ b/boards/migrations/0048_remove_post_images.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-05-21 08:25 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('boards', '0047_attachment_alias'), + ] + + operations = [ + migrations.RemoveField( + model_name='post', + name='images', + ), + ] diff --git a/boards/migrations/0049_delete_postimage.py b/boards/migrations/0049_delete_postimage.py new file mode 100644 --- /dev/null +++ b/boards/migrations/0049_delete_postimage.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-05-21 08:29 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('boards', '0048_remove_post_images'), + ] + + operations = [ + migrations.DeleteModel( + name='PostImage', + ), + ] diff --git a/boards/models/__init__.py b/boards/models/__init__.py --- a/boards/models/__init__.py +++ b/boards/models/__init__.py @@ -5,7 +5,6 @@ STATUS_ARCHIVE = 'archived' from boards.models.sync_key import KeyPair from boards.models.signature import GlobalId, Signature -from boards.models.image import PostImage from boards.models.attachment import Attachment from boards.models.thread import Thread from boards.models.post import Post diff --git a/boards/models/attachment/__init__.py b/boards/models/attachment/__init__.py --- a/boards/models/attachment/__init__.py +++ b/boards/models/attachment/__init__.py @@ -1,8 +1,12 @@ +import boards +from boards.models import STATUS_ARCHIVE +from django.core.files.images import get_image_dimensions from django.db import models from boards import utils -from boards.models.attachment.viewers import get_viewers, AbstractViewer -from boards.utils import get_upload_filename, get_file_mimetype, get_extension +from boards.models.attachment.viewers import get_viewers, AbstractViewer, \ + FILE_TYPES_IMAGE +from boards.utils import get_upload_filename, get_extension, cached_result class AttachmentManager(models.Manager): @@ -19,6 +23,13 @@ class AttachmentManager(models.Manager): return attachment + def get_random_images(self, count, tags=None): + images = self.filter(mimetype__in=FILE_TYPES_IMAGE).exclude( + post_attachments__thread__status=STATUS_ARCHIVE) + if tags is not None: + images = images.filter(post_attachments__threads__tags__in=tags) + return images.order_by('?')[:count] + class Attachment(models.Model): objects = AttachmentManager() @@ -26,6 +37,7 @@ class Attachment(models.Model): file = models.FileField(upload_to=get_upload_filename) mimetype = models.CharField(max_length=50) hash = models.CharField(max_length=36) + alias = models.TextField(unique=True, null=True, blank=True) def get_view(self): file_viewer = None @@ -40,3 +52,27 @@ class Attachment(models.Model): def __str__(self): return self.file.url + + def get_random_associated_post(self): + posts = boards.models.Post.objects.filter(attachments__in=[self]) + return posts.order_by('?').first() + + @cached_result() + def get_size(self): + if self.mimetype in FILE_TYPES_IMAGE: + return get_image_dimensions(self.file) + else: + return 200, 150 + + def get_thumb_url(self): + split = self.file.url.rsplit('.', 1) + w, h = 200, 150 + return '%s.%sx%s.%s' % (split[0], w, h, split[1]) + + @cached_result() + def get_preview_size(self): + if self.mimetype in FILE_TYPES_IMAGE: + preview_path = self.file.path.replace('.', '.200x150.') + return get_image_dimensions(preview_path) + else: + return 200, 150 diff --git a/boards/models/attachment/viewers.py b/boards/models/attachment/viewers.py --- a/boards/models/attachment/viewers.py +++ b/boards/models/attachment/viewers.py @@ -1,3 +1,4 @@ +from django.core.files.images import get_image_dimensions from django.template.defaultfilters import filesizeformat from django.contrib.staticfiles.templatetags.staticfiles import static @@ -15,6 +16,13 @@ FILE_TYPES_AUDIO = ( 'mp3', 'opus', ) +FILE_TYPES_IMAGE = ( + 'jpeg', + 'jpg', + 'png', + 'bmp', + 'gif', +) PLAIN_FILE_FORMATS = { 'pdf': 'pdf', @@ -22,6 +30,9 @@ PLAIN_FILE_FORMATS = { 'txt': 'txt', } +CSS_CLASS_IMAGE = 'image' +CSS_CLASS_THUMB = 'thumb' + def get_viewers(): return AbstractViewer.__subclasses__() @@ -83,3 +94,35 @@ class SvgViewer(AbstractViewer): return ''\ ''\ ''.format(self.file.url, self.file.url) + + +class ImageViewer(AbstractViewer): + @staticmethod + def supports(file_type): + return file_type in FILE_TYPES_IMAGE + + def get_format_view(self): + metadata = '{}, {}'.format(self.file.name.split('.')[-1], + filesizeformat(self.file.size)) + width, height = get_image_dimensions(self.file.file) + preview_path = self.file.path.replace('.', '.200x150.') + pre_width, pre_height = get_image_dimensions(preview_path) + + split = self.file.url.rsplit('.', 1) + w, h = 200, 150 + thumb_url = '%s.%sx%s.%s' % (split[0], w, h, split[1]) + + return '' \ + '' \ + '' \ + .format(CSS_CLASS_THUMB, + thumb_url, + str(pre_width), + str(pre_height), str(width), str(height), + full=self.file.url, image_meta=metadata) + diff --git a/boards/models/image.py b/boards/models/image.py deleted file mode 100644 --- a/boards/models/image.py +++ /dev/null @@ -1,92 +0,0 @@ -from django.core.files.images import get_image_dimensions -from django.db import models -from django.template.defaultfilters import filesizeformat - -from boards import thumbs, utils -import boards -from boards.models.base import Viewable -from boards.models import STATUS_ARCHIVE -from boards.utils import get_upload_filename - - -__author__ = 'neko259' - - -IMAGE_THUMB_SIZE = (200, 150) -HASH_LENGTH = 36 - -CSS_CLASS_IMAGE = 'image' -CSS_CLASS_THUMB = 'thumb' - - -class PostImageManager(models.Manager): - def create_with_hash(self, image): - image_hash = utils.get_file_hash(image) - existing = self.filter(hash=image_hash) - if len(existing) > 0: - post_image = existing[0] - else: - post_image = PostImage.objects.create(image=image) - - return post_image - - def get_random_images(self, count, tags=None): - images = self.exclude(post_images__thread__status=STATUS_ARCHIVE) - if tags is not None: - images = images.filter(post_images__threads__tags__in=tags) - return images.order_by('?')[:count] - - -class PostImage(models.Model, Viewable): - objects = PostImageManager() - - class Meta: - app_label = 'boards' - ordering = ('id',) - - image = thumbs.ImageWithThumbsField(upload_to=get_upload_filename, - blank=True, sizes=(IMAGE_THUMB_SIZE,)) - hash = models.CharField(max_length=HASH_LENGTH) - alias = models.TextField(unique=True, null=True, blank=True) - - def save(self, *args, **kwargs): - """ - Saves the model and computes the image hash for deduplication purposes. - """ - - if not self.pk and self.image: - self.hash = utils.get_file_hash(self.image) - super(PostImage, self).save(*args, **kwargs) - - def __str__(self): - return self.image.url - - def get_view(self): - metadata = '{}, {}'.format(self.image.name.split('.')[-1], - filesizeformat(self.image.size)) - width, height = get_image_dimensions(self.image.file) - preview_path = self.image.path.replace('.', '.200x150.') - pre_width, pre_height = get_image_dimensions(preview_path) - return '
' \ - '' \ - '{}' \ - '' \ - '' \ - '
'\ - .format(CSS_CLASS_IMAGE, CSS_CLASS_THUMB, - self.image.url_200x150, - str(self.hash), str(pre_width), - str(pre_height), str(width), str(height), - full=self.image.url, image_meta=metadata) - - def get_random_associated_post(self): - posts = boards.models.Post.objects.filter(images__in=[self]) - return posts.order_by('?').first() diff --git a/boards/models/post/__init__.py b/boards/models/post/__init__.py --- a/boards/models/post/__init__.py +++ b/boards/models/post/__init__.py @@ -3,7 +3,8 @@ import uuid import re from boards import settings from boards.abstracts.tripcode import Tripcode -from boards.models import PostImage, Attachment, KeyPair, GlobalId +from boards.models import Attachment, KeyPair, GlobalId +from boards.models.attachment import FILE_TYPES_IMAGE from boards.models.base import Viewable from boards.models.post.export import get_exporter, DIFF_TYPE_JSON from boards.models.post.manager import PostManager @@ -27,8 +28,6 @@ APP_LABEL_BOARDS = 'boards' BAN_REASON_AUTO = 'Auto' -IMAGE_THUMB_SIZE = (200, 150) - TITLE_MAX_LENGTH = 200 REGEX_REPLY = re.compile(r'\[post\](\d+)\[/post\]') @@ -74,8 +73,6 @@ class Post(models.Model, Viewable): text = TextField(blank=True, null=True) _text_rendered = TextField(blank=True, null=True, editable=False) - images = models.ManyToManyField(PostImage, null=True, blank=True, - related_name='post_images', db_index=True) attachments = models.ManyToManyField(Attachment, null=True, blank=True, related_name='attachment_posts') @@ -212,8 +209,8 @@ class Post(models.Model, Viewable): def get_search_view(self, *args, **kwargs): return self.get_view(need_op_data=True, *args, **kwargs) - def get_first_image(self) -> PostImage: - return self.images.earliest('id') + def get_first_image(self) -> Attachment: + return self.attachments.filter(mimetype__in=FILE_TYPES_IMAGE).earliest('id') def set_global_id(self, key_pair=None): """ diff --git a/boards/models/post/manager.py b/boards/models/post/manager.py --- a/boards/models/post/manager.py +++ b/boards/models/post/manager.py @@ -11,19 +11,11 @@ import boards from boards.models.user import Ban from boards.mdx_neboard import Parser -from boards.models import PostImage, Attachment +from boards.models import Attachment from boards import utils __author__ = 'neko259' -IMAGE_TYPES = ( - 'jpeg', - 'jpg', - 'png', - 'bmp', - 'gif', -) - POSTS_PER_DAY_RANGE = 7 NO_IP = '0.0.0.0' @@ -186,8 +178,4 @@ class PostManager(models.Manager): list(map(thread.tags.add, tags)) def _add_file_to_post(self, file, post): - file_type = file.name.split('.')[-1].lower() - if file_type in IMAGE_TYPES: - post.images.add(PostImage.objects.create_with_hash(file)) - else: - post.attachments.add(Attachment.objects.create_with_hash(file)) + post.attachments.add(Attachment.objects.create_with_hash(file)) diff --git a/boards/models/post/sync.py b/boards/models/post/sync.py --- a/boards/models/post/sync.py +++ b/boards/models/post/sync.py @@ -72,16 +72,11 @@ class SyncManager: global_id = post.global_id - images = post.images.all() attachments = post.attachments.all() if global_id.content: model.append(et.fromstring(global_id.content)) - if len(images) > 0 or len(attachments) > 0: + if len(attachments) > 0: attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS) - for image in images: - SyncManager._attachment_to_xml( - None, attachment_refs, image.image.file, - image.hash, image.image.url) for file in attachments: SyncManager._attachment_to_xml( None, attachment_refs, file.file.file, @@ -116,14 +111,10 @@ class SyncManager: tripcode = et.SubElement(content_tag, TAG_TRIPCODE) tripcode.text = post.tripcode - if len(images) > 0 or len(attachments) > 0: + if len(attachments) > 0: attachments_tag = et.SubElement(content_tag, TAG_ATTACHMENTS) attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS) - for image in images: - SyncManager._attachment_to_xml( - attachments_tag, attachment_refs, image.image.file, - image.hash, image.image.url) for file in attachments: SyncManager._attachment_to_xml( attachments_tag, attachment_refs, file.file.file, diff --git a/boards/models/tag.py b/boards/models/tag.py --- a/boards/models/tag.py +++ b/boards/models/tag.py @@ -1,10 +1,11 @@ import hashlib +from boards.models.attachment import FILE_TYPES_IMAGE from django.template.loader import render_to_string from django.db import models from django.db.models import Count from django.core.urlresolvers import reverse -from boards.models import PostImage +from boards.models import Attachment from boards.models.base import Viewable from boards.models.thread import STATUS_ACTIVE, STATUS_BUMPLIMIT, STATUS_ARCHIVE from boards.utils import cached_result @@ -107,8 +108,9 @@ class Tag(models.Model, Viewable): return self.description def get_random_image_post(self, status=[STATUS_ACTIVE, STATUS_BUMPLIMIT]): - posts = boards.models.Post.objects.annotate(images_count=Count( - 'images')).filter(images_count__gt=0, threads__tags__in=[self]) + posts = boards.models.Post.objects.filter(attachments__mimetype__in=FILE_TYPES_IMAGE)\ + .annotate(images_count=Count( + 'attachments')).filter(images_count__gt=0, threads__tags__in=[self]) if status is not None: posts = posts.filter(thread__status__in=status) return posts.order_by('?').first() @@ -143,5 +145,6 @@ class Tag(models.Model, Viewable): return self.children def get_images(self): - return PostImage.objects.filter(post_images__thread__tags__in=[self])\ - .order_by('-post_images__pub_time') \ No newline at end of file + return Attachment.objects.filter( + post_attachments__thread__tags__in=[self]).filter( + mimetype__in=FILE_TYPES_IMAGE).order_by('-post_images__pub_time') \ No newline at end of file diff --git a/boards/models/thread.py b/boards/models/thread.py --- a/boards/models/thread.py +++ b/boards/models/thread.py @@ -1,5 +1,6 @@ import logging from adjacent import Client +from boards.models.attachment import FILE_TYPES_IMAGE from django.db.models import Count, Sum, QuerySet, Q from django.utils import timezone @@ -138,8 +139,10 @@ class Thread(models.Model): @cached_result(key_method=_get_cache_key) def get_images_count(self) -> int: - return self.get_replies().annotate(images_count=Count( - 'images')).aggregate(Sum('images_count'))['images_count__sum'] + return self.get_replies().filter( + attachments__mimetype__in=FILE_TYPES_IMAGE)\ + .annotate(images_count=Count( + 'attachments')).aggregate(Sum('images_count'))['images_count__sum'] def can_bump(self) -> bool: """ @@ -181,7 +184,7 @@ class Thread(models.Model): """ query = self.multi_replies.order_by('pub_time').prefetch_related( - 'images', 'thread', 'attachments') + 'thread', 'attachments') if view_fields_only: query = query.defer('poster_ip') return query @@ -193,9 +196,9 @@ class Thread(models.Model): """ Gets replies that have at least one image attached """ - - return self.get_replies(view_fields_only).annotate(images_count=Count( - 'images')).filter(images_count__gt=0) + return self.get_replies(view_fields_only).filter( + attachments__mimetype__in=FILE_TYPES_IMAGE).annotate(images_count=Count( + 'attachments')).filter(images_count__gt=0) def get_opening_post(self, only_id=False) -> Post: """ diff --git a/boards/signals.py b/boards/signals.py --- a/boards/signals.py +++ b/boards/signals.py @@ -1,7 +1,9 @@ import re +from boards import thumbs from boards.mdx_neboard import get_parser -from boards.models import Post, GlobalId +from boards.models import Post, GlobalId, Attachment +from boards.models.attachment.viewers import FILE_TYPES_IMAGE from boards.models.post import REGEX_NOTIFICATION, REGEX_REPLY,\ REGEX_GLOBAL_REPLY from boards.models.post.manager import post_import_deps @@ -12,6 +14,9 @@ from django.dispatch import receiver from django.utils import timezone +THUMB_SIZES = ((200, 150),) + + @receiver(post_save, sender=Post) def connect_replies(instance, **kwargs): for reply_number in re.finditer(REGEX_REPLY, instance.get_raw_text()): @@ -90,3 +95,22 @@ def update_thread_on_delete(instance, ** def delete_global_id(instance, **kwargs): if instance.global_id and instance.global_id.id: instance.global_id.delete() + + +@receiver(post_save, sender=Attachment) +def generate_thumb(instance, **kwargs): + if instance.mimetype in FILE_TYPES_IMAGE: + for size in THUMB_SIZES: + (w, h) = size + split = instance.file.name.rsplit('.', 1) + thumb_name = '%s.%sx%s.%s' % (split[0], w, h, split[1]) + + if not instance.file.storage.exists(thumb_name): + # you can use another thumbnailing function if you like + thumb_content = thumbs.generate_thumb(instance.file, size, split[1]) + + thumb_name_ = instance.file.storage.save(thumb_name, thumb_content) + + if not thumb_name == thumb_name_: + raise ValueError( + 'There is already a file named %s' % thumb_name_) diff --git a/boards/templates/boards/all_threads.html b/boards/templates/boards/all_threads.html --- a/boards/templates/boards/all_threads.html +++ b/boards/templates/boards/all_threads.html @@ -40,11 +40,11 @@
{% if random_image_post %}
- {% with image=random_image_post.images.first %} + {% with image=random_image_post.get_first_image %} {{ random_image_post.id }} {% endwith %}
diff --git a/boards/templates/boards/settings.html b/boards/templates/boards/settings.html --- a/boards/templates/boards/settings.html +++ b/boards/templates/boards/settings.html @@ -26,7 +26,7 @@ {% endif %} {% for image in image_aliases %} - {{ image.alias }}:
+
{{ image.alias }}: {{ image.get_view|safe }}
{% endfor %}
diff --git a/boards/templates/boards/thread_gallery.html b/boards/templates/boards/thread_gallery.html --- a/boards/templates/boards/thread_gallery.html +++ b/boards/templates/boards/thread_gallery.html @@ -23,7 +23,7 @@ {% autoescape off %} {{ image.get_view }}