diff --git a/boards/config/default_settings.ini b/boards/config/default_settings.ini --- a/boards/config/default_settings.ini +++ b/boards/config/default_settings.ini @@ -9,7 +9,7 @@ CacheTimeout = 600 [Forms] # Max post length in characters MaxTextLength = 30000 -MaxImageSize = 8000000 +MaxFileSize = 8000000 LimitPostingSpeed = false [Messages] diff --git a/boards/forms.py b/boards/forms.py --- a/boards/forms.py +++ b/boards/forms.py @@ -18,14 +18,6 @@ import boards.settings as board_settings HEADER_CONTENT_LENGTH = 'content-length' HEADER_CONTENT_TYPE = 'content-type' -CONTENT_TYPE_IMAGE = ( - 'image/jpeg', - 'image/jpg', - 'image/png', - 'image/gif', - 'image/bmp', -) - REGEX_TAGS = re.compile(r'^[\w\s\d]+$', re.UNICODE) VETERAN_POSTING_DELAY = 5 @@ -47,7 +39,7 @@ ERROR_SPEED = _('Please wait %s seconds TAG_MAX_LENGTH = 20 -IMAGE_DOWNLOAD_CHUNK_BYTES = 100000 +FILE_DOWNLOAD_CHUNK_BYTES = 100000 HTTP_RESULT_OK = 200 @@ -144,10 +136,10 @@ class PostForm(NeboardForm): ATTRIBUTE_ROWS: TEXTAREA_ROWS, }), required=False, label=LABEL_TEXT) - image = forms.ImageField(required=False, label=_('Image'), + file = forms.FileField(required=False, label=_('File'), widget=forms.ClearableFileInput( - attrs={'accept': 'image/*'})) - image_url = forms.CharField(required=False, label=_('Image URL'), + attrs={'accept': 'file/*'})) + file_url = forms.CharField(required=False, label=_('File URL'), widget=forms.TextInput( attrs={ATTRIBUTE_PLACEHOLDER: 'http://example.com/image.png'})) @@ -181,27 +173,27 @@ class PostForm(NeboardForm): 'characters') % str(max_length)) return text - def clean_image(self): - image = self.cleaned_data['image'] + def clean_file(self): + file = self.cleaned_data['file'] - if image: - self.validate_image_size(image.size) + if file: + self.validate_file_size(file.size) - return image + return file - def clean_image_url(self): - url = self.cleaned_data['image_url'] + def clean_file_url(self): + url = self.cleaned_data['file_url'] - image = None + file = None if url: - image = self._get_image_from_url(url) + file = self._get_file_from_url(url) - if not image: + if not file: raise forms.ValidationError(_('Invalid URL')) else: - self.validate_image_size(image.size) + self.validate_file_size(file.size) - return image + return file def clean_threads(self): threads_str = self.cleaned_data['threads'] @@ -230,27 +222,27 @@ class PostForm(NeboardForm): raise forms.ValidationError('A human cannot enter a hidden field') if not self.errors: - self._clean_text_image() + self._clean_text_file() if not self.errors and self.session: self._validate_posting_speed() return cleaned_data - def get_image(self): + def get_file(self): """ - Gets image from file or URL. + Gets file from form or URL. """ - image = self.cleaned_data['image'] - return image if image else self.cleaned_data['image_url'] + file = self.cleaned_data['file'] + return file or self.cleaned_data['file_url'] - def _clean_text_image(self): + def _clean_text_file(self): text = self.cleaned_data.get('text') - image = self.get_image() + file = self.get_file() - if (not text) and (not image): - error_message = _('Either text or image must be entered.') + if (not text) and (not file): + error_message = _('Either text or file must be entered.') self._errors['text'] = self.error_class([error_message]) def _validate_posting_speed(self): @@ -284,16 +276,16 @@ class PostForm(NeboardForm): if can_post: self.session[LAST_POST_TIME] = now - def validate_image_size(self, size: int): - max_size = board_settings.get_int('Forms', 'MaxImageSize') + def validate_file_size(self, size: int): + max_size = board_settings.get_int('Forms', 'MaxFileSize') if size > max_size: raise forms.ValidationError( - _('Image must be less than %s bytes') + _('File must be less than %s bytes') % str(max_size)) - def _get_image_from_url(self, url: str) -> SimpleUploadedFile: + def _get_file_from_url(self, url: str) -> SimpleUploadedFile: """ - Gets an image file from URL. + Gets an file file from URL. """ img_temp = None @@ -302,30 +294,29 @@ class PostForm(NeboardForm): # Verify content headers response_head = requests.head(url, verify=False) content_type = response_head.headers[HEADER_CONTENT_TYPE].split(';')[0] - if content_type in CONTENT_TYPE_IMAGE: - length_header = response_head.headers.get(HEADER_CONTENT_LENGTH) - if length_header: - length = int(length_header) - self.validate_image_size(length) - # Get the actual content into memory - response = requests.get(url, verify=False, stream=True) + length_header = response_head.headers.get(HEADER_CONTENT_LENGTH) + if length_header: + length = int(length_header) + self.validate_file_size(length) + # Get the actual content into memory + response = requests.get(url, verify=False, stream=True) - # Download image, stop if the size exceeds limit - size = 0 - content = b'' - for chunk in response.iter_content(IMAGE_DOWNLOAD_CHUNK_BYTES): - size += len(chunk) - self.validate_image_size(size) - content += chunk + # Download file, stop if the size exceeds limit + size = 0 + content = b'' + for chunk in response.iter_content(file_DOWNLOAD_CHUNK_BYTES): + size += len(chunk) + self.validate_file_size(size) + content += chunk - if response.status_code == HTTP_RESULT_OK and content: - # Set a dummy file name that will be replaced - # anyway, just keep the valid extension - filename = 'image.' + content_type.split('/')[1] - img_temp = SimpleUploadedFile(filename, content, - content_type) + if response.status_code == HTTP_RESULT_OK and content: + # Set a dummy file name that will be replaced + # anyway, just keep the valid extension + filename = 'file.' + content_type.split('/')[1] + img_temp = SimpleUploadedFile(filename, content, + content_type) except Exception: - # Just return no image + # Just return no file pass return img_temp @@ -369,7 +360,7 @@ class ThreadForm(PostForm): class SettingsForm(NeboardForm): theme = forms.ChoiceField(choices=settings.THEMES, label=_('Theme')) - image_viewer = forms.ChoiceField(choices=settings.IMAGE_VIEWERS, label=_('Image view mode')) + image_viewer = forms.ChoiceField(choices=settings.IMAGE_VIEWERS, label=_('image view mode')) username = forms.CharField(label=_('User name'), required=False) timezone = forms.ChoiceField(choices=get_timezones(), label=_('Time zone')) diff --git a/boards/migrations/0022_auto_20150812_1819.py b/boards/migrations/0022_auto_20150812_1819.py new file mode 100644 --- /dev/null +++ b/boards/migrations/0022_auto_20150812_1819.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('boards', '0021_tag_description'), + ] + + operations = [ + migrations.AlterField( + model_name='thread', + name='tags', + field=models.ManyToManyField(related_name='thread_tags', to='boards.Tag'), + ), + ] diff --git a/boards/migrations/0023_auto_20150818_1026.py b/boards/migrations/0023_auto_20150818_1026.py new file mode 100644 --- /dev/null +++ b/boards/migrations/0023_auto_20150818_1026.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations +import boards.models.attachment + + +class Migration(migrations.Migration): + + dependencies = [ + ('boards', '0022_auto_20150812_1819'), + ] + + operations = [ + migrations.CreateModel( + name='Attachment', + fields=[ + ('id', models.AutoField(verbose_name='ID', primary_key=True, serialize=False, auto_created=True)), + ('file', models.FileField(upload_to=boards.models.attachment.Attachment._update_filename)), + ('mimetype', models.CharField(max_length=50)), + ('hash', models.CharField(max_length=36)), + ], + ), + migrations.AddField( + model_name='post', + name='attachments', + field=models.ManyToManyField(blank=True, null=True, related_name='attachment_posts', to='boards.Attachment'), + ), + ] diff --git a/boards/models/__init__.py b/boards/models/__init__.py --- a/boards/models/__init__.py +++ b/boards/models/__init__.py @@ -1,6 +1,7 @@ __author__ = 'neko259' from boards.models.image import PostImage +from boards.models.attachment import Attachment from boards.models.thread import Thread from boards.models.post import Post from boards.models.tag import Tag diff --git a/boards/models/attachment/__init__.py b/boards/models/attachment/__init__.py new file mode 100644 --- /dev/null +++ b/boards/models/attachment/__init__.py @@ -0,0 +1,75 @@ +import hashlib +import os +import time + +from random import random + +from django.db import models + +from boards.models.attachment.viewers import AbstractViewer, WebmViewer + + +FILES_DIRECTORY = 'files/' +FILE_EXTENSION_DELIMITER = '.' + +VIEWERS = ( + WebmViewer, +) + + +class AttachmentManager(models.Manager): + def create_with_hash(self, file): + file_hash = self.get_hash(file) + existing = self.filter(hash=file_hash) + if len(existing) > 0: + attachment = existing[0] + else: + file_type = file.name.split(FILE_EXTENSION_DELIMITER)[-1].lower() + attachment = Attachment.objects.create(file=file, + mimetype=file_type, hash=file_hash) + + return attachment + + def get_hash(self, file): + """ + Gets hash of an file. + """ + md5 = hashlib.md5() + for chunk in file.chunks(): + md5.update(chunk) + return md5.hexdigest() + + +class Attachment(models.Model): + objects = AttachmentManager() + + # TODO Dedup the method + def _update_filename(self, filename): + """ + Gets unique filename + """ + + # TODO Use something other than random number in file name + new_name = '{}{}.{}'.format( + str(int(time.mktime(time.gmtime()))), + str(int(random() * 1000)), + filename.split(FILE_EXTENSION_DELIMITER)[-1:][0]) + + return os.path.join(FILES_DIRECTORY, new_name) + + file = models.FileField(upload_to=_update_filename) + mimetype = models.CharField(max_length=50) + hash = models.CharField(max_length=36) + + def get_view(self): + file_viewer = None + for viewer in VIEWERS: + if viewer.supports(self.mimetype): + file_viewer = viewer(self.file, self.mimetype) + break + if file_viewer is None: + file_viewer = AbstractViewer(self.file, self.mimetype) + + return file_viewer.get_view() + + diff --git a/boards/models/attachment/viewers.py b/boards/models/attachment/viewers.py new file mode 100644 --- /dev/null +++ b/boards/models/attachment/viewers.py @@ -0,0 +1,39 @@ +from django.template.defaultfilters import filesizeformat + + +class AbstractViewer: + def __init__(self, file, file_type): + self.file = file + self.file_type = file_type + + @staticmethod + def get_viewer(file_type, file): + for viewer in VIEWERS: + if viewer.supports(file_type): + return viewer(file) + return AbstractViewer(file) + + @staticmethod + def supports(file_type): + return true + + def get_view(self): + return '
'\ + ''\ + ''\ + '
{}, {}
'\ + '
'.format(self.file.url, self.file_type, + filesizeformat(self.file.size)) + + +class WebmViewer(AbstractViewer): + @staticmethod + def supports(file_type): + return file_type == 'webm' + + def get_view(self): + return '
'\ + '
'.format(self.file.url) + diff --git a/boards/models/image.py b/boards/models/image.py --- a/boards/models/image.py +++ b/boards/models/image.py @@ -61,15 +61,13 @@ class PostImage(models.Model, Viewable): Gets unique image filename """ - path = IMAGES_DIRECTORY - # TODO Use something other than random number in file name new_name = '{}{}.{}'.format( str(int(time.mktime(time.gmtime()))), str(int(random() * 1000)), filename.split(FILE_EXTENSION_DELIMITER)[-1:][0]) - return os.path.join(path, new_name) + return os.path.join(IMAGES_DIRECTORY, new_name) width = models.IntegerField(default=0) height = models.IntegerField(default=0) diff --git a/boards/models/post/__init__.py b/boards/models/post/__init__.py --- a/boards/models/post/__init__.py +++ b/boards/models/post/__init__.py @@ -13,7 +13,7 @@ from django.utils import timezone from boards import settings from boards.mdx_neboard import Parser -from boards.models import PostImage +from boards.models import PostImage, Attachment from boards.models.base import Viewable from boards import utils from boards.models.post.export import get_exporter, DIFF_TYPE_JSON @@ -62,10 +62,17 @@ POST_VIEW_PARAMS = ( REFMAP_STR = '>>{}' +IMAGE_TYPES = ( + 'jpeg', + 'jpg', + 'png', + 'bmp', +) + class PostManager(models.Manager): @transaction.atomic - def create_post(self, title: str, text: str, image=None, thread=None, + def create_post(self, title: str, text: str, file=None, thread=None, ip=NO_IP, tags: list=None, opening_posts: list=None): """ Creates new post @@ -105,8 +112,13 @@ class PostManager(models.Manager): logger.info('Created post {} by {}'.format(post, post.poster_ip)) - if image: - post.images.add(PostImage.objects.create_with_hash(image)) + # TODO Move this to other place + if file: + file_type = file.name.split('.')[-1].lower() + if file_type in IMAGE_TYPES: + post.images.add(PostImage.objects.create_with_hash(file)) + else: + post.attachments.add(Attachment.objects.create_with_hash(file)) post.build_url() post.connect_replies() @@ -169,6 +181,8 @@ class Post(models.Model, Viewable): images = models.ManyToManyField(PostImage, null=True, blank=True, related_name='post_images', db_index=True) + attachments = models.ManyToManyField(Attachment, null=True, blank=True, + related_name='attachment_posts') poster_ip = models.GenericIPAddressField() diff --git a/boards/templates/boards/post.html b/boards/templates/boards/post.html --- a/boards/templates/boards/post.html +++ b/boards/templates/boards/post.html @@ -51,12 +51,19 @@ supports multiple. {% endcomment %} {% if post.images.exists %} - {% with post.images.all.0 as image %} + {% with post.images.first as image %} {% autoescape off %} {{ image.get_view }} {% endautoescape %} {% endwith %} {% endif %} + {% if post.attachments.exists %} + {% with post.attachments.first as file %} + {% autoescape off %} + {{ file.get_view }} + {% endautoescape %} + {% endwith %} + {% endif %} {% comment %} Post message (text) {% endcomment %} diff --git a/boards/views/all_threads.py b/boards/views/all_threads.py --- a/boards/views/all_threads.py +++ b/boards/views/all_threads.py @@ -141,7 +141,7 @@ class AllThreadsView(PostMixin, BaseBoar title = data[FORM_TITLE] text = data[FORM_TEXT] - image = form.get_image() + file = form.get_file() threads = data[FORM_THREADS] text = self._remove_invalid_links(text) @@ -150,7 +150,7 @@ class AllThreadsView(PostMixin, BaseBoar tags = self.parse_tags_string(tag_strings) - post = Post.objects.create_post(title=title, text=text, image=image, + post = Post.objects.create_post(title=title, text=text, file=file, ip=ip, tags=tags, opening_posts=threads) # This is required to update the threads to which posts we have replied diff --git a/boards/views/thread/thread.py b/boards/views/thread/thread.py --- a/boards/views/thread/thread.py +++ b/boards/views/thread/thread.py @@ -102,14 +102,14 @@ class ThreadView(BaseBoardView, PostMixi title = data[FORM_TITLE] text = data[FORM_TEXT] - image = form.get_image() + file = form.get_file() threads = data[FORM_THREADS] text = self._remove_invalid_links(text) post_thread = opening_post.get_thread() - post = Post.objects.create_post(title=title, text=text, image=image, + post = Post.objects.create_post(title=title, text=text, file=file, thread=post_thread, ip=ip, opening_posts=threads) post.notify_clients()