Download webm videos from youtube

boards/models/attachment/downloaders.py

@@ -0,0 +1,83 b''
+import os
+import re
+
+from django.core.files.uploadedfile import SimpleUploadedFile
+from pytube import YouTube
+import requests
+
+from boards.utils import validate_file_size
+
+YOUTUBE_VIDEO_FORMAT = 'webm'
+
+HTTP_RESULT_OK = 200
+
+HEADER_CONTENT_LENGTH = 'content-length'
+HEADER_CONTENT_TYPE = 'content-type'
+
+FILE_DOWNLOAD_CHUNK_BYTES = 100000
+
+YOUTUBE_URL = re.compile(r'https?://www\.youtube\.com/watch\?v=\w+')
+
+
+class Downloader:
+    @staticmethod
+    def handles(url: str) -> bool:
+        return False
+
+    @staticmethod
+    def download(url: str):
+        # Verify content headers
+        response_head = requests.head(url, verify=False)
+        content_type = response_head.headers[HEADER_CONTENT_TYPE].split(';')[0]
+        length_header = response_head.headers.get(HEADER_CONTENT_LENGTH)
+        if length_header:
+            length = int(length_header)
+            validate_file_size(length)
+        # Get the actual content into memory
+        response = requests.get(url, verify=False, stream=True)
+
+        # Download file, stop if the size exceeds limit
+        size = 0
+        content = b''
+        for chunk in response.iter_content(FILE_DOWNLOAD_CHUNK_BYTES):
+            size += len(chunk)
+            validate_file_size(size)
+            content += chunk
+
+        if response.status_code == HTTP_RESULT_OK and content:
+            # Set a dummy file name that will be replaced
+            # anyway, just keep the valid extension
+            filename = 'file.' + content_type.split('/')[1]
+            return SimpleUploadedFile(filename, content, content_type)
+
+
+class YouTubeDownloader(Downloader):
+    @staticmethod
+    def download(url: str):
+        yt = YouTube()
+        yt.from_url(url)
+        videos = yt.filter(YOUTUBE_VIDEO_FORMAT)
+        if len(videos) > 0:
+            video = videos[0]
+            filename = '{}.{}'.format(video.filename, video.extension)
+            try:
+                video.download(on_progress=YouTubeDownloader.on_progress)
+
+                file = open(filename, 'rb')
+                content = file.read()
+                file.close()
+
+                os.remove(filename)
+                return SimpleUploadedFile(filename, content, video.extension)
+            except Exception as e:
+                if os.path.isfile(filename):
+                    os.remove(filename)
+                raise e
+
+    @staticmethod
+    def handles(url: str) -> bool:
+        return YOUTUBE_URL.match(url)
+
+    @staticmethod
+    def on_progress(bytes, file_size, start):
+        validate_file_size(file_size)

boards/forms.py

@@ -1,395 +1,364 b''
 import hashlib
 import re
 import time
 
 import pytz
 from django import forms
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.core.exceptions import ObjectDoesNotExist
 from django.forms.util import ErrorList
 from django.utils.translation import ugettext_lazy as _
-import requests
 
 from boards.mdx_neboard import formatters
+from boards.models.attachment.downloaders import Downloader
 from boards.models.post import TITLE_MAX_LENGTH
 from boards.models import Tag, Post
+from boards.utils import validate_file_size
 from neboard import settings
 import boards.settings as board_settings
 import neboard
 
-HEADER_CONTENT_LENGTH = 'content-length'
-HEADER_CONTENT_TYPE = 'content-type'
-
 REGEX_TAGS = re.compile(r'^[\w\s\d]+$', re.UNICODE)
 
 VETERAN_POSTING_DELAY = 5
 
 ATTRIBUTE_PLACEHOLDER = 'placeholder'
 ATTRIBUTE_ROWS = 'rows'
 
 LAST_POST_TIME = 'last_post_time'
 LAST_LOGIN_TIME = 'last_login_time'
 TEXT_PLACEHOLDER = _('Type message here. Use formatting panel for more advanced usage.')
 TAGS_PLACEHOLDER = _('music images i_dont_like_tags')
 
 LABEL_TITLE = _('Title')
 LABEL_TEXT = _('Text')
 LABEL_TAG = _('Tag')
 LABEL_SEARCH = _('Search')
 
 ERROR_SPEED = _('Please wait %s seconds before sending message')
 
 TAG_MAX_LENGTH = 20
 
-FILE_DOWNLOAD_CHUNK_BYTES = 100000
-
-HTTP_RESULT_OK = 200
-
 TEXTAREA_ROWS = 4
 
 
 def get_timezones():
     timezones = []
     for tz in pytz.common_timezones:
         timezones.append((tz, tz),)
     return timezones
 
 
 class FormatPanel(forms.Textarea):
     """
     Panel for text formatting. Consists of buttons to add different tags to the
     form text area.
     """
 
     def render(self, name, value, attrs=None):
         output = '<div id="mark-panel">'
         for formatter in formatters:
             output += '<span class="mark_btn"' + \
                       ' onClick="addMarkToMsg(\'' + formatter.format_left + \
                       '\', \'' + formatter.format_right + '\')">' + \
                       formatter.preview_left + formatter.name + \
                       formatter.preview_right + '</span>'
 
         output += '</div>'
         output += super(FormatPanel, self).render(name, value, attrs=None)
 
         return output
 
 
 class PlainErrorList(ErrorList):
     def __unicode__(self):
         return self.as_text()
 
     def as_text(self):
         return ''.join(['(!) %s ' % e for e in self])
 
 
 class NeboardForm(forms.Form):
     """
     Form with neboard-specific formatting.
     """
 
     def as_div(self):
         """
         Returns this form rendered as HTML <as_div>s.
         """
 
         return self._html_output(
             # TODO Do not show hidden rows in the list here
             normal_row='<div class="form-row">'
                        '<div class="form-label">'
                        '%(label)s'
                        '</div>'
                        '<div class="form-input">'
                        '%(field)s'
                        '</div>'
                        '</div>'
                        '<div class="form-row">'
                        '%(help_text)s'
                        '</div>',
             error_row='<div class="form-row">'
                       '<div class="form-label"></div>'
                       '<div class="form-errors">%s</div>'
                       '</div>',
             row_ender='</div>',
             help_text_html='%s',
             errors_on_separate_row=True)
 
     def as_json_errors(self):
         errors = []
 
         for name, field in list(self.fields.items()):
             if self[name].errors:
                 errors.append({
                     'field': name,
                     'errors': self[name].errors.as_text(),
                 })
 
         return errors
 
 
 class PostForm(NeboardForm):
 
     title = forms.CharField(max_length=TITLE_MAX_LENGTH, required=False,
                             label=LABEL_TITLE,
                             widget=forms.TextInput(
                                 attrs={ATTRIBUTE_PLACEHOLDER:
                                            'test#tripcode'}))
     text = forms.CharField(
         widget=FormatPanel(attrs={
             ATTRIBUTE_PLACEHOLDER: TEXT_PLACEHOLDER,
             ATTRIBUTE_ROWS: TEXTAREA_ROWS,
         }),
         required=False, label=LABEL_TEXT)
     file = forms.FileField(required=False, label=_('File'),
                              widget=forms.ClearableFileInput(
                                  attrs={'accept': 'file/*'}))
     file_url = forms.CharField(required=False, label=_('File URL'),
                                 widget=forms.TextInput(
                                     attrs={ATTRIBUTE_PLACEHOLDER:
                                                'http://example.com/image.png'}))
 
     # This field is for spam prevention only
     email = forms.CharField(max_length=100, required=False, label=_('e-mail'),
                             widget=forms.TextInput(attrs={
                                 'class': 'form-email'}))
     threads = forms.CharField(required=False, label=_('Additional threads'),
             widget=forms.TextInput(attrs={ATTRIBUTE_PLACEHOLDER:
                 '123 456 789'}))
 
     session = None
     need_to_ban = False
 
     def clean_title(self):
         title = self.cleaned_data['title']
         if title:
             if len(title) > TITLE_MAX_LENGTH:
                 raise forms.ValidationError(_('Title must have less than %s '
                                               'characters') %
                                             str(TITLE_MAX_LENGTH))
         return title
 
     def clean_text(self):
         text = self.cleaned_data['text'].strip()
         if text:
             max_length = board_settings.get_int('Forms', 'MaxTextLength')
             if len(text) > max_length:
                 raise forms.ValidationError(_('Text must have less than %s '
                                               'characters') % str(max_length))
         return text
 
     def clean_file(self):
         file = self.cleaned_data['file']
 
         if file:
-            self.validate_file_size(file.size)
+            validate_file_size(file.size)
 
         return file
 
     def clean_file_url(self):
         url = self.cleaned_data['file_url']
 
         file = None
         if url:
             file = self._get_file_from_url(url)
 
             if not file:
                 raise forms.ValidationError(_('Invalid URL'))
             else:
-                self.validate_file_size(file.size)
+                validate_file_size(file.size)
 
         return file
 
     def clean_threads(self):
         threads_str = self.cleaned_data['threads']
 
         if len(threads_str) > 0:
             threads_id_list = threads_str.split(' ')
 
             threads = list()
 
             for thread_id in threads_id_list:
                 try:
                     thread = Post.objects.get(id=int(thread_id))
                     if not thread.is_opening() or thread.get_thread().archived:
                         raise ObjectDoesNotExist()
                     threads.append(thread)
                 except (ObjectDoesNotExist, ValueError):
                     raise forms.ValidationError(_('Invalid additional thread list'))
 
             return threads
 
     def clean(self):
         cleaned_data = super(PostForm, self).clean()
 
         if cleaned_data['email']:
             self.need_to_ban = True
             raise forms.ValidationError('A human cannot enter a hidden field')
 
         if not self.errors:
             self._clean_text_file()
 
         if not self.errors and self.session:
             self._validate_posting_speed()
 
         return cleaned_data
 
     def get_file(self):
         """
         Gets file from form or URL.
         """
 
         file = self.cleaned_data['file']
         return file or self.cleaned_data['file_url']
 
     def get_tripcode(self):
         title = self.cleaned_data['title']
         if title is not None and '#' in title:
             code = title.split('#', maxsplit=1)[1] + neboard.settings.SECRET_KEY
             return hashlib.md5(code.encode()).hexdigest()
 
     def get_title(self):
         title = self.cleaned_data['title']
         if title is not None and '#' in title:
             return title.split('#', maxsplit=1)[0]
         else:
             return title
 
     def _clean_text_file(self):
         text = self.cleaned_data.get('text')
         file = self.get_file()
 
         if (not text) and (not file):
             error_message = _('Either text or file must be entered.')
             self._errors['text'] = self.error_class([error_message])
 
     def _validate_posting_speed(self):
         can_post = True
 
         posting_delay = settings.POSTING_DELAY
 
         if board_settings.get_bool('Forms', 'LimitPostingSpeed'):
             now = time.time()
 
             current_delay = 0
             need_delay = False
 
             if not LAST_POST_TIME in self.session:
                 self.session[LAST_POST_TIME] = now
 
                 need_delay = True
             else:
                 last_post_time = self.session.get(LAST_POST_TIME)
                 current_delay = int(now - last_post_time)
 
                 need_delay = current_delay < posting_delay
 
             if need_delay:
                 error_message = ERROR_SPEED % str(posting_delay
                         - current_delay)
                 self._errors['text'] = self.error_class([error_message])
 
                 can_post = False
 
             if can_post:
                 self.session[LAST_POST_TIME] = now
 
-    def validate_file_size(self, size: int):
-        max_size = board_settings.get_int('Forms', 'MaxFileSize')
-        if size > max_size:
-            raise forms.ValidationError(
-                _('File must be less than %s bytes')
-                % str(max_size))
-
     def _get_file_from_url(self, url: str) -> SimpleUploadedFile:
         """
         Gets an file file from URL.
         """
 
         img_temp = None
 
         try:
-            # Verify content headers
-            response_head = requests.head(url, verify=False)
-            content_type = response_head.headers[HEADER_CONTENT_TYPE].split(';')[0]
-            length_header = response_head.headers.get(HEADER_CONTENT_LENGTH)
-            if length_header:
-                length = int(length_header)
-                self.validate_file_size(length)
-            # Get the actual content into memory
-            response = requests.get(url, verify=False, stream=True)
-
-            # Download file, stop if the size exceeds limit
-            size = 0
-            content = b''
-            for chunk in response.iter_content(FILE_DOWNLOAD_CHUNK_BYTES):
-                size += len(chunk)
-                self.validate_file_size(size)
-                content += chunk
-
-            if response.status_code == HTTP_RESULT_OK and content:
-                # Set a dummy file name that will be replaced
-                # anyway, just keep the valid extension
-                filename = 'file.' + content_type.split('/')[1]
-                img_temp = SimpleUploadedFile(filename, content,
-                                              content_type)
+            for downloader in Downloader.__subclasses__():
+                if downloader.handles(url):
+                    return downloader.download(url)
+            # If nobody of the specific downloaders handles this, use generic
+            # one
+            return Downloader.download(url)
+        except forms.ValidationError as e:
+            raise e
         except Exception as e:
             # Just return no file
             pass
 
-        return img_temp
-
 
 class ThreadForm(PostForm):
 
     tags = forms.CharField(
         widget=forms.TextInput(attrs={ATTRIBUTE_PLACEHOLDER: TAGS_PLACEHOLDER}),
         max_length=100, label=_('Tags'), required=True)
 
     def clean_tags(self):
         tags = self.cleaned_data['tags'].strip()
 
         if not tags or not REGEX_TAGS.match(tags):
             raise forms.ValidationError(
                 _('Inappropriate characters in tags.'))
 
         required_tag_exists = False
         for tag in tags.split():
             try:
                 Tag.objects.get(name=tag.strip().lower(), required=True)
                 required_tag_exists = True
                 break
             except ObjectDoesNotExist:
                 pass
 
         if not required_tag_exists:
             all_tags = Tag.objects.filter(required=True)
             raise forms.ValidationError(
                 _('Need at least one section.'))
 
         return tags
 
     def clean(self):
         cleaned_data = super(ThreadForm, self).clean()
 
         return cleaned_data
 
 
 class SettingsForm(NeboardForm):
 
     theme = forms.ChoiceField(choices=settings.THEMES, label=_('Theme'))
     image_viewer = forms.ChoiceField(choices=settings.IMAGE_VIEWERS, label=_('Image view mode'))
     username = forms.CharField(label=_('User name'), required=False)
     timezone = forms.ChoiceField(choices=get_timezones(), label=_('Time zone'))
 
     def clean_username(self):
         username = self.cleaned_data['username']
 
         if username and not REGEX_TAGS.match(username):
             raise forms.ValidationError(_('Inappropriate characters.'))
 
         return username
 
 
 class SearchForm(NeboardForm):
     query = forms.CharField(max_length=500, label=LABEL_SEARCH, required=False)

boards/utils.py

@@ -1,92 +1,103 b''
 """
 This module contains helper functions and helper classes.
 """
 import hashlib
 import time
 import hmac
 
 from django.core.cache import cache
 from django.db.models import Model
+from django import forms
 
 from django.utils import timezone
+from django.utils.translation import ugettext_lazy as _
+import boards
 
 from neboard import settings
 
 
 CACHE_KEY_DELIMITER = '_'
 PERMISSION_MODERATE = 'moderation'
 
 def get_client_ip(request):
     x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
     if x_forwarded_for:
         ip = x_forwarded_for.split(',')[-1].strip()
     else:
         ip = request.META.get('REMOTE_ADDR')
     return ip
 
 
 # TODO The output format is not epoch because it includes microseconds
 def datetime_to_epoch(datetime):
     return int(time.mktime(timezone.localtime(
         datetime,timezone.get_current_timezone()).timetuple())
                * 1000000 + datetime.microsecond)
 
 
 def get_websocket_token(user_id='', timestamp=''):
     """
     Create token to validate information provided by new connection.
     """
 
     sign = hmac.new(settings.CENTRIFUGE_PROJECT_SECRET.encode())
     sign.update(settings.CENTRIFUGE_PROJECT_ID.encode())
     sign.update(user_id.encode())
     sign.update(timestamp.encode())
     token = sign.hexdigest()
 
     return token
 
 
 def cached_result(key_method=None):
     """
     Caches method result in the Django's cache system, persisted by object name,
     object name and model id if object is a Django model.
     """
     def _cached_result(function):
         def inner_func(obj, *args, **kwargs):
             # TODO Include method arguments to the cache key
             cache_key_params = [obj.__class__.__name__, function.__name__]
             if isinstance(obj, Model):
                 cache_key_params.append(str(obj.id))
 
             if key_method is not None:
                 cache_key_params += [str(arg) for arg in key_method(obj)]
 
             cache_key = CACHE_KEY_DELIMITER.join(cache_key_params)
 
             persisted_result = cache.get(cache_key)
             if persisted_result is not None:
                 result = persisted_result
             else:
                 result = function(obj, *args, **kwargs)
                 cache.set(cache_key, result)
 
             return result
 
         return inner_func
     return _cached_result
 
 
 def is_moderator(request):
     try:
         moderate = request.user.has_perm(PERMISSION_MODERATE)
     except AttributeError:
         moderate = False
 
     return moderate
 
 
 def get_file_hash(file) -> str:
     md5 = hashlib.md5()
     for chunk in file.chunks():
         md5.update(chunk)
     return md5.hexdigest()
+
+
+def validate_file_size(size: int):
+    max_size = boards.settings.get_int('Forms', 'MaxFileSize')
+    if size > max_size:
+        raise forms.ValidationError(
+            _('File must be less than %s bytes')
+            % str(max_size))

requirements.txt

@@ -1,8 +1,9 b''
+pytube
 requests
 adjacent
 django-haystack
 pillow
 django>=1.8
 bbcode
 django-debug-toolbar
 pytz