Added fetch sources to fetch external information into threads as posts

boards/migrations/0067_threadsource.py

@@ -0,0 +1,27 b''
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11 on 2017-11-21 11:29
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('boards', '0066_auto_20171025_1148'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ThreadSource',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.TextField()),
+                ('timestamp', models.DateTimeField()),
+                ('source', models.TextField()),
+                ('source_type', models.CharField(choices=[('RSS', 'RSS')], max_length=100)),
+                ('thread', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='boards.Thread')),
+            ],
+        ),
+    ]

boards/models/source.py

@@ -0,0 +1,66 b''
+import feedparser
+import logging
+
+from time import mktime
+from datetime import datetime
+
+from django.db import models, transaction
+from django.utils.dateparse import parse_datetime
+from django.utils.timezone import utc
+from django.utils import timezone
+from boards.models import Post
+from boards.models.post import TITLE_MAX_LENGTH
+
+
+SOURCE_TYPE_MAX_LENGTH = 100
+SOURCE_TYPE_RSS = 'RSS'
+TYPE_CHOICES = (
+        (SOURCE_TYPE_RSS, SOURCE_TYPE_RSS),
+)
+
+
+class ThreadSource(models.Model):
+    class Meta:
+        app_label = 'boards'
+
+    name = models.TextField()
+    thread = models.ForeignKey('Thread')
+    timestamp = models.DateTimeField()
+    source = models.TextField()
+    source_type = models.CharField(max_length=SOURCE_TYPE_MAX_LENGTH,
+            choices=TYPE_CHOICES)
+
+    def __str__(self):
+        return self.name
+
+    @transaction.atomic
+    def fetch_latest_posts(self):
+        """Creates new posts with the info fetched since the timestamp."""
+        logger = logging.getLogger('boards.source')
+
+        if self.thread.is_archived():
+            logger.error('The thread {} is archived, please try another one'.format(self.thread))
+        else:
+            start_timestamp = timezone.localtime(self.timestamp)
+            last_timestamp = start_timestamp
+            if self.thread.is_bumplimit():
+                logger.warn('The thread {} has reached its bumplimit, please create a new one'.format(self.thread))
+            if self.source_type == SOURCE_TYPE_RSS:
+                feed = feedparser.parse(self.source)
+                items = sorted(feed.entries, key=lambda entry: entry.published_parsed)
+                for item in items:
+                    title = item.title[:TITLE_MAX_LENGTH]
+                    timestamp = datetime.fromtimestamp(mktime(item.published_parsed), tz=utc)
+                    if not timestamp:
+                        logger.error('Invalid timestamp {} for {}'.format(item.published, title))
+                    else:
+                        if timestamp > last_timestamp:
+                            last_timestamp = timestamp
+
+                        if timestamp > start_timestamp:
+                            Post.objects.create_post(title=title, text=item.description, thread=self.thread, file_urls=[item.link])
+                            logger.info('Fetched item {} from {} into thread {}'.format(
+                                title, self.name, self.thread))
+            self.timestamp = last_timestamp
+            self.save(update_fields=['timestamp'])
+

boards/admin.py

@@ -1,181 +1,188 b''
 from boards.abstracts.sticker_factory import StickerFactory
 from boards.models.attachment import FILE_TYPES_IMAGE, AttachmentSticker, \
     StickerPack
 from django.contrib import admin
 from django.utils.translation import ugettext_lazy as _
 from django.core.urlresolvers import reverse
 from boards.models import Post, Tag, Ban, Thread, Banner, Attachment, \
         KeyPair, GlobalId, TagAlias
+from boards.models.source import ThreadSource
 
 
 @admin.register(Post)
 class PostAdmin(admin.ModelAdmin):
 
     list_display = ('id', 'title', 'text', 'poster_ip', 'linked_images',
             'foreign', 'tags')
     list_filter = ('pub_time',)
     search_fields = ('id', 'title', 'text', 'poster_ip')
     exclude = ('referenced_posts', 'refmap', 'images', 'global_id')
     readonly_fields = ('poster_ip', 'thread', 'linked_images',
             'attachments', 'uid', 'url', 'pub_time', 'opening', 'linked_global_id',
             'foreign', 'tags')
 
     def ban_poster(self, request, queryset):
         bans = 0
         for post in queryset:
             poster_ip = post.poster_ip
             ban, created = Ban.objects.get_or_create(ip=poster_ip)
             if created:
                 bans += 1
         self.message_user(request, _('{} posters were banned').format(bans))
 
     def ban_latter_with_delete(self, request, queryset):
         bans = 0
         hidden = 0
         for post in queryset:
             poster_ip = post.poster_ip
             ban, created = Ban.objects.get_or_create(ip=poster_ip)
             if created:
                 bans += 1
             posts = Post.objects.filter(poster_ip=poster_ip, id__gte=post.id)
             hidden += posts.count()
             posts.delete()
         self.message_user(request, _('{} posters were banned, {} messages were removed.').format(bans, hidden))
     ban_latter_with_delete.short_description = _('Ban user and delete posts starting from this one and later')
 
     def linked_images(self, obj: Post):
         images = obj.attachments.filter(mimetype__in=FILE_TYPES_IMAGE)
         image_urls = ['<a href="{}"><img src="{}" /></a>'.format(
             reverse('admin:%s_%s_change' % (image._meta.app_label,
                                             image._meta.model_name),
                     args=[image.id]), image.get_thumb_url()) for image in images]
         return ', '.join(image_urls)
     linked_images.allow_tags = True
 
     def linked_global_id(self, obj: Post):
         global_id = obj.global_id
         if global_id is not None:
             return '<a href="{}">{}</a>'.format(
                 reverse('admin:%s_%s_change' % (global_id._meta.app_label,
                                                 global_id._meta.model_name),
                         args=[global_id.id]), str(global_id))
     linked_global_id.allow_tags = True
 
     def tags(self, obj: Post):
         return ', '.join([tag.get_name() for tag in obj.get_tags()])
 
     def save_model(self, request, obj, form, change):
         obj.save()
         obj.clear_cache()
 
     def foreign(self, obj: Post):
         return obj is not None and obj.global_id is not None and\
                 not obj.global_id.is_local()
 
     actions = ['ban_poster', 'ban_latter_with_delete']
 
 
 @admin.register(Tag)
 class TagAdmin(admin.ModelAdmin):
     def thread_count(self, obj: Tag) -> int:
         return obj.get_thread_count()
 
     def display_children(self, obj: Tag):
         return ', '.join([str(child) for child in obj.get_children().all()])
 
     def name(self, obj: Tag):
         return obj.get_name()
 
     def save_model(self, request, obj, form, change):
         super().save_model(request, obj, form, change)
         for thread in obj.get_threads().all():
             thread.refresh_tags()
 
     list_display = ('name', 'thread_count', 'display_children')
     search_fields = ('id',)
     readonly_fields = ('name',)
 
 
 @admin.register(TagAlias)
 class TagAliasAdmin(admin.ModelAdmin):
     list_display = ('locale', 'name', 'parent')
     list_filter = ('locale',)
     search_fields = ('name',)
 
 
 @admin.register(Thread)
 class ThreadAdmin(admin.ModelAdmin):
 
     def title(self, obj: Thread) -> str:
         return obj.get_opening_post().get_title()
 
     def reply_count(self, obj: Thread) -> int:
         return obj.get_reply_count()
 
     def ip(self, obj: Thread):
         return obj.get_opening_post().poster_ip
 
     def display_tags(self, obj: Thread):
         return ', '.join([str(tag) for tag in obj.get_tags().all()])
 
     def op(self, obj: Thread):
         return obj.get_opening_post_id()
 
     # Save parent tags when editing tags
     def save_related(self, request, form, formsets, change):
         super().save_related(request, form, formsets, change)
         form.instance.refresh_tags()
 
     def save_model(self, request, obj, form, change):
         op = obj.get_opening_post()
         obj.save()
         op.clear_cache()
 
     list_display = ('id', 'op', 'title', 'reply_count', 'status', 'ip',
             'display_tags')
     list_filter = ('bump_time', 'status')
     search_fields = ('id', 'title')
     filter_horizontal = ('tags',)
 
 
 @admin.register(KeyPair)
 class KeyPairAdmin(admin.ModelAdmin):
     list_display = ('public_key', 'primary')
     list_filter = ('primary',)
     search_fields = ('public_key',)
 
 
 @admin.register(Ban)
 class BanAdmin(admin.ModelAdmin):
     list_display = ('ip', 'can_read')
     list_filter = ('can_read',)
     search_fields = ('ip',)
 
 
 @admin.register(Banner)
 class BannerAdmin(admin.ModelAdmin):
     list_display = ('title', 'text')
 
 
 @admin.register(Attachment)
 class AttachmentAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'mimetype', 'file', 'url')
 
 
 @admin.register(AttachmentSticker)
 class AttachmentStickerAdmin(admin.ModelAdmin):
     search_fields = ('name',)
 
 
 @admin.register(StickerPack)
 class StickerPackAdmin(admin.ModelAdmin):
     search_fields = ('name',)
 
 
 @admin.register(GlobalId)
 class GlobalIdAdmin(admin.ModelAdmin):
     def is_linked(self, obj):
         return Post.objects.filter(global_id=obj).exists()
 
     list_display = ('__str__', 'is_linked',)
     readonly_fields = ('content',)
+
+
[e-mail](ThreadSource)
+class ThreadSourceAdmin(admin.ModelAdmin):
+    search_fields = ('name', 'source')
+

boards/config/default_settings.ini

@@ -1,49 +1,50 b''
 [Version]
 Version = 4.6.0 Lucy
 SiteName = Neboard DEV
 
 [Cache]
 # Timeout for caching, if cache is used
 CacheTimeout = 600
 
 [Forms]
 # Max post length in characters
 MaxTextLength = 30000
 MaxFileSize = 8000000
 LimitFirstPosting = true
 LimitPostingSpeed = false
 PowDifficulty = 0
 # Delay in seconds
 PostingDelay = 30
 Autoban = false
 DefaultTag = test
 MaxFileCount = 1
 AdditionalSpoilerSpaces = false
 
 [Messages]
 # Thread bumplimit
 MaxPostsPerThread = 10
 ThreadArchiveDays = 300
 AnonymousMode = false
 
 [View]
 DefaultTheme = md
 DefaultImageViewer = simple
 LastRepliesCount = 3
 ThreadsPerPage = 3
 PostsPerPage = 10
 ImagesPerPageGallery = 20
 MaxFavoriteThreads = 20
 MaxLandingThreads = 20
 Themes=md:Mystic Dark,md_centered:Mystic Dark (centered),sw:Snow White,pg:Photon Grey,ad:Amanita Dark,iw:Inocibe White
 ImageViewers=simple:Simple,popup:Popup
 
 [Storage]
 # Enable archiving threads instead of deletion when the thread limit is reached
 ArchiveThreads = true
 
 [RSS]
 MaxItems = 20
 
 [External]
 ImageSearchHost=
+SourceFetcherTripcode=

boards/models/thread.py

@@ -1,322 +1,325 b''
 import logging
 from datetime import timedelta
 
 from django.db import models, transaction
 from django.db.models import Count, Sum, QuerySet, Q
 from django.utils import timezone
 
 import boards
 from boards import settings
 from boards.models import STATUS_BUMPLIMIT, STATUS_ACTIVE, STATUS_ARCHIVE
 from boards.models.attachment import FILE_TYPES_IMAGE
 from boards.models.post import Post
 from boards.models.tag import Tag, DEFAULT_LOCALE, TagAlias
 from boards.utils import cached_result, datetime_to_epoch
 
 FAV_THREAD_NO_UPDATES = -1
 
 
 __author__ = 'neko259'
 
 
 logger = logging.getLogger(__name__)
 
 
 WS_NOTIFICATION_TYPE_NEW_POST = 'new_post'
 WS_NOTIFICATION_TYPE = 'notification_type'
 
 WS_CHANNEL_THREAD = "thread:"
 
 STATUS_CHOICES = (
         (STATUS_ACTIVE, STATUS_ACTIVE),
         (STATUS_BUMPLIMIT, STATUS_BUMPLIMIT),
         (STATUS_ARCHIVE, STATUS_ARCHIVE),
 )
 
 
 class ThreadManager(models.Manager):
     def process_old_threads(self):
         """
          Preserves maximum thread count. If there are too many threads,
          archive or delete the old ones.
          """
         old_time_delta = settings.get_int('Messages', 'ThreadArchiveDays')
         old_time = timezone.now() - timedelta(days=old_time_delta)
         old_ops = Post.objects.filter(opening=True, pub_time__lte=old_time).exclude(thread__status=STATUS_ARCHIVE)
 
         for op in old_ops:
             thread = op.get_thread()
             if settings.get_bool('Storage', 'ArchiveThreads'):
                 self._archive_thread(thread)
             else:
                 thread.delete()
             logger.info('Processed old thread {}'.format(thread))
 
 
     def _archive_thread(self, thread):
         thread.status = STATUS_ARCHIVE
         thread.last_edit_time = timezone.now()
         thread.update_posts_time()
         thread.save(update_fields=['last_edit_time', 'status'])
 
     def get_new_posts(self, datas):
         query = None
         # TODO Use classes instead of dicts
         for data in datas:
             if data['last_id'] != FAV_THREAD_NO_UPDATES:
                 q = (Q(id=data['op'].get_thread_id())
                      & Q(replies__id__gt=data['last_id']))
                 if query is None:
                     query = q
                 else:
                     query = query | q
         if query is not None:
             return self.filter(query).annotate(
                 new_post_count=Count('replies'))
 
     def get_new_post_count(self, datas):
         new_posts = self.get_new_posts(datas)
         return new_posts.aggregate(total_count=Count('replies'))\
                 ['total_count'] if new_posts else 0
 
 
 def get_thread_max_posts():
     return settings.get_int('Messages', 'MaxPostsPerThread')
 
 
 class Thread(models.Model):
     objects = ThreadManager()
 
     class Meta:
         app_label = 'boards'
 
     tags = models.ManyToManyField('Tag', related_name='thread_tags')
     bump_time = models.DateTimeField(db_index=True)
     last_edit_time = models.DateTimeField()
     max_posts = models.IntegerField(default=get_thread_max_posts)
     status = models.CharField(max_length=50, default=STATUS_ACTIVE,
             choices=STATUS_CHOICES, db_index=True)
     monochrome = models.BooleanField(default=False)
     stickerpack = models.BooleanField(default=False)
 
     def get_tags(self) -> QuerySet:
         """
         Gets a sorted tag list.
         """
 
         return self.tags.filter(aliases__in=TagAlias.objects.filter_localized(parent__thread_tags=self)).order_by('aliases__name')
 
     def bump(self):
         """
         Bumps (moves to up) thread if possible.
         """
 
         if self.can_bump():
             self.bump_time = self.last_edit_time
 
             self.update_bump_status()
 
             logger.info('Bumped thread %d' % self.id)
 
     def has_post_limit(self) -> bool:
         return self.max_posts > 0
 
     def update_bump_status(self, exclude_posts=None):
         if self.has_post_limit() and self.get_reply_count() >= self.max_posts:
             self.status = STATUS_BUMPLIMIT
             self.update_posts_time(exclude_posts=exclude_posts)
 
     def _get_cache_key(self):
         return [datetime_to_epoch(self.last_edit_time)]
 
     @cached_result(key_method=_get_cache_key)
     def get_reply_count(self) -> int:
         return self.get_replies().count()
 
     @cached_result(key_method=_get_cache_key)
     def get_images_count(self) -> int:
         return self.get_replies().filter(
             attachments__mimetype__in=FILE_TYPES_IMAGE)\
             .annotate(images_count=Count(
             'attachments')).aggregate(Sum('images_count'))['images_count__sum'] or 0
 
     @cached_result(key_method=_get_cache_key)
     def get_attachment_count(self) -> int:
         return self.get_replies().annotate(attachment_count=Count('attachments'))\
                 .aggregate(Sum('attachment_count'))['attachment_count__sum'] or 0
 
     def can_bump(self) -> bool:
         """
         Checks if the thread can be bumped by replying to it.
         """
 
         return self.get_status() == STATUS_ACTIVE
 
     def get_last_replies(self) -> QuerySet:
         """
         Gets several last replies, not including opening post
         """
 
         last_replies_count = settings.get_int('View', 'LastRepliesCount')
 
         if last_replies_count > 0:
             reply_count = self.get_reply_count()
 
             if reply_count > 0:
                 reply_count_to_show = min(last_replies_count,
                                           reply_count - 1)
                 replies = self.get_replies()
                 last_replies = replies[reply_count - reply_count_to_show:]
 
                 return last_replies
 
     def get_skipped_replies_count(self) -> int:
         """
         Gets number of posts between opening post and last replies.
         """
         reply_count = self.get_reply_count()
         last_replies_count = min(settings.get_int('View', 'LastRepliesCount'),
                                  reply_count - 1)
         return reply_count - last_replies_count - 1
 
     # TODO Remove argument, it is not used
     def get_replies(self, view_fields_only=True) -> QuerySet:
         """
         Gets sorted thread posts
         """
         query = self.replies.order_by('pub_time').prefetch_related(
             'attachments')
         return query
 
     def get_viewable_replies(self) -> QuerySet:
         """
         Gets replies with only fields that are used for viewing.
         """
         return self.get_replies().defer('text', 'last_edit_time')
 
     def get_top_level_replies(self) -> QuerySet:
         return self.get_replies().exclude(refposts__threads__in=[self])
 
     def get_replies_with_images(self, view_fields_only=False) -> QuerySet:
         """
         Gets replies that have at least one image attached
         """
         return self.get_replies(view_fields_only).filter(
             attachments__mimetype__in=FILE_TYPES_IMAGE).annotate(images_count=Count(
             'attachments')).filter(images_count__gt=0)
 
     def get_opening_post(self, only_id=False) -> Post:
         """
         Gets the first post of the thread
         """
 
         query = self.get_replies().filter(opening=True)
         if only_id:
             query = query.only('id')
         opening_post = query.first()
 
         return opening_post
 
     @cached_result()
     def get_opening_post_id(self) -> int:
         """
         Gets ID of the first thread post.
         """
 
         return self.get_opening_post(only_id=True).id
 
     def get_pub_time(self):
         """
         Gets opening post's pub time because thread does not have its own one.
         """
 
         return self.get_opening_post().pub_time
 
     def __str__(self):
-        return 'T#{}'.format(self.id)
+        return 'T#{}/{}'.format(self.id, self.get_opening_post())
 
     def get_tag_url_list(self) -> list:
         return boards.models.Tag.objects.get_tag_url_list(self.get_tags().all())
 
     def update_posts_time(self, exclude_posts=None):
         last_edit_time = self.last_edit_time
 
         for post in self.replies.all():
             if exclude_posts is None or post not in exclude_posts:
                 # Manual update is required because uids are generated on save
                 post.last_edit_time = last_edit_time
                 post.save(update_fields=['last_edit_time'])
 
     def get_absolute_url(self):
         return self.get_opening_post().get_absolute_url()
 
     def get_required_tags(self):
         return self.get_tags().filter(required=True)
 
     def get_sections_str(self):
         return Tag.objects.get_tag_url_list(self.get_required_tags())
 
     def get_replies_newer(self, post_id):
         return self.get_replies().filter(id__gt=post_id)
 
     def is_archived(self):
         return self.get_status() == STATUS_ARCHIVE
 
+    def is_bumplimit(self):
+        return self.get_status() == STATUS_BUMPLIMIT
+
     def get_status(self):
         return self.status
 
     def is_monochrome(self):
         return self.monochrome
 
     def is_stickerpack(self):
         return self.stickerpack
 
     # If tags have parent, add them to the tag list
     @transaction.atomic
     def refresh_tags(self):
         for tag in self.get_tags().all():
             parents = tag.get_all_parents()
             if len(parents) > 0:
                 self.tags.add(*parents)
 
     def get_reply_tree(self):
         replies = self.get_replies().prefetch_related('refposts')
         tree = []
         for reply in replies:
             parents = reply.refposts.all()
 
             found_parent = False
             searching_for_index = False
 
             if len(parents) > 0:
                 index = 0
                 parent_depth = 0
 
                 indexes_to_insert = []
 
                 for depth, element in tree:
                     index += 1
 
                     # If this element is next after parent on the same level,
                     # insert child before it
                     if searching_for_index and depth <= parent_depth:
                         indexes_to_insert.append((index - 1, parent_depth))
                         searching_for_index = False
 
                     if element in parents:
                         found_parent = True
                         searching_for_index = True
                         parent_depth = depth
 
             if not found_parent:
                 tree.append((0, reply))
             else:
                 if searching_for_index:
                     tree.append((parent_depth + 1, reply))
 
                 offset = 0
                 for last_index, parent_depth in indexes_to_insert:
                     tree.insert(last_index + offset, (parent_depth + 1, reply))
                     offset += 1
 
         return tree

boards/settings.py

@@ -1,24 +1,29 b''
 import configparser
 
 
+CONFIG_DEFAULT_SETTINGS = 'boards/config/default_settings.ini'
+CONFIG_SETTINGS = 'boards/config/settings.ini'
+
+
 config = configparser.ConfigParser()
-config.read('boards/config/default_settings.ini')
-config.read('boards/config/settings.ini')
+config.read(CONFIG_DEFAULT_SETTINGS)
+config.read(CONFIG_SETTINGS)
 
 
 def get(section, name):
     return config[section][name]
 
 
 def get_int(section, name):
     return int(get(section, name))
 
 
 def get_bool(section, name):
     return get(section, name) == 'true'
 
 
 def get_list_dict(section, name):
     str_dict = get(section, name)
     return [item.split(':') for item in str_dict.split(',')]
 
+

requirements.txt

@@ -1,11 +1,12 b''
 python-magic
 httplib2
 simplejson
 pytube
 requests
 pillow
 django>=1.8
 bbcode
 django-debug-toolbar
 pytz
 ecdsa
+feedparser