Added fetch sources to fetch external information into threads as posts

boards/migrations/0067_threadsource.py

@@ -0,0 +1,27 b''
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11 on 2017-11-21 11:29
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('boards', '0066_auto_20171025_1148'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ThreadSource',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.TextField()),
+                ('timestamp', models.DateTimeField()),
+                ('source', models.TextField()),
+                ('source_type', models.CharField(choices=[('RSS', 'RSS')], max_length=100)),
+                ('thread', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='boards.Thread')),
+            ],
+        ),
+    ]

boards/models/source.py

@@ -0,0 +1,66 b''
+import feedparser
+import logging
+
+from time import mktime
+from datetime import datetime
+
+from django.db import models, transaction
+from django.utils.dateparse import parse_datetime
+from django.utils.timezone import utc
+from django.utils import timezone
+from boards.models import Post
+from boards.models.post import TITLE_MAX_LENGTH
+
+
+SOURCE_TYPE_MAX_LENGTH = 100
+SOURCE_TYPE_RSS = 'RSS'
+TYPE_CHOICES = (
+        (SOURCE_TYPE_RSS, SOURCE_TYPE_RSS),
+)
+
+
+class ThreadSource(models.Model):
+    class Meta:
+        app_label = 'boards'
+
+    name = models.TextField()
+    thread = models.ForeignKey('Thread')
+    timestamp = models.DateTimeField()
+    source = models.TextField()
+    source_type = models.CharField(max_length=SOURCE_TYPE_MAX_LENGTH,
+            choices=TYPE_CHOICES)
+
+    def __str__(self):
+        return self.name
+
+    @transaction.atomic
+    def fetch_latest_posts(self):
+        """Creates new posts with the info fetched since the timestamp."""
+        logger = logging.getLogger('boards.source')
+
+        if self.thread.is_archived():
+            logger.error('The thread {} is archived, please try another one'.format(self.thread))
+        else:
+            start_timestamp = timezone.localtime(self.timestamp)
+            last_timestamp = start_timestamp
+            if self.thread.is_bumplimit():
+                logger.warn('The thread {} has reached its bumplimit, please create a new one'.format(self.thread))
+            if self.source_type == SOURCE_TYPE_RSS:
+                feed = feedparser.parse(self.source)
+                items = sorted(feed.entries, key=lambda entry: entry.published_parsed)
+                for item in items:
+                    title = item.title[:TITLE_MAX_LENGTH]
+                    timestamp = datetime.fromtimestamp(mktime(item.published_parsed), tz=utc)
+                    if not timestamp:
+                        logger.error('Invalid timestamp {} for {}'.format(item.published, title))
+                    else:
+                        if timestamp > last_timestamp:
+                            last_timestamp = timestamp
+
+                        if timestamp > start_timestamp:
+                            Post.objects.create_post(title=title, text=item.description, thread=self.thread, file_urls=[item.link])
+                            logger.info('Fetched item {} from {} into thread {}'.format(
+                                title, self.name, self.thread))
+            self.timestamp = last_timestamp
+            self.save(update_fields=['timestamp'])
+

boards/admin.py

@@ -6,6 +6,7 b' from django.utils.translation import uge'
 from django.core.urlresolvers import reverse
 from boards.models import Post, Tag, Ban, Thread, Banner, Attachment, \
         KeyPair, GlobalId, TagAlias
+from boards.models.source import ThreadSource
 
 
 @admin.register(Post)
@@ -179,3 +180,9 b' class GlobalIdAdmin(admin.ModelAdmin):'
 
     list_display = ('__str__', 'is_linked',)
     readonly_fields = ('content',)
+
+
[e-mail](ThreadSource)
+class ThreadSourceAdmin(admin.ModelAdmin):
+    search_fields = ('name', 'source')
+

boards/config/default_settings.ini

@@ -47,3 +47,4 b' MaxItems = 20'
 
 [External]
 ImageSearchHost=
+SourceFetcherTripcode=

boards/models/thread.py

@@ -233,7 +233,7 b' class Thread(models.Model):'
         return self.get_opening_post().pub_time
 
     def __str__(self):
-        return 'T#{}'.format(self.id)
+        return 'T#{}/{}'.format(self.id, self.get_opening_post())
 
     def get_tag_url_list(self) -> list:
         return boards.models.Tag.objects.get_tag_url_list(self.get_tags().all())
@@ -262,6 +262,9 b' class Thread(models.Model):'
     def is_archived(self):
         return self.get_status() == STATUS_ARCHIVE
 
+    def is_bumplimit(self):
+        return self.get_status() == STATUS_BUMPLIMIT
+
     def get_status(self):
         return self.status
 

boards/settings.py

@@ -1,9 +1,13 b''
 import configparser
 
 
+CONFIG_DEFAULT_SETTINGS = 'boards/config/default_settings.ini'
+CONFIG_SETTINGS = 'boards/config/settings.ini'
+
+
 config = configparser.ConfigParser()
-config.read('boards/config/default_settings.ini')
-config.read('boards/config/settings.ini')
+config.read(CONFIG_DEFAULT_SETTINGS)
+config.read(CONFIG_SETTINGS)
 
 
 def get(section, name):
@@ -22,3 +26,4 b' def get_list_dict(section, name):'
     str_dict = get(section, name)
     return [item.split(':') for item in str_dict.split(',')]
 
+

requirements.txt

@@ -9,3 +9,4 b' bbcode'
 django-debug-toolbar
 pytz
 ecdsa
+feedparser