diff --git a/boards/admin.py b/boards/admin.py --- a/boards/admin.py +++ b/boards/admin.py @@ -6,6 +6,7 @@ from django.utils.translation import uge from django.core.urlresolvers import reverse from boards.models import Post, Tag, Ban, Thread, Banner, Attachment, \ KeyPair, GlobalId, TagAlias +from boards.models.source import ThreadSource @admin.register(Post) @@ -179,3 +180,9 @@ class GlobalIdAdmin(admin.ModelAdmin): list_display = ('__str__', 'is_linked',) readonly_fields = ('content',) + + +@admin.register(ThreadSource) +class ThreadSourceAdmin(admin.ModelAdmin): + search_fields = ('name', 'source') + diff --git a/boards/config/default_settings.ini b/boards/config/default_settings.ini --- a/boards/config/default_settings.ini +++ b/boards/config/default_settings.ini @@ -47,3 +47,4 @@ MaxItems = 20 [External] ImageSearchHost= +SourceFetcherTripcode= diff --git a/boards/migrations/0067_threadsource.py b/boards/migrations/0067_threadsource.py new file mode 100644 --- /dev/null +++ b/boards/migrations/0067_threadsource.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2017-11-21 11:29 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('boards', '0066_auto_20171025_1148'), + ] + + operations = [ + migrations.CreateModel( + name='ThreadSource', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.TextField()), + ('timestamp', models.DateTimeField()), + ('source', models.TextField()), + ('source_type', models.CharField(choices=[('RSS', 'RSS')], max_length=100)), + ('thread', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='boards.Thread')), + ], + ), + ] diff --git a/boards/models/source.py b/boards/models/source.py new file mode 100644 --- /dev/null +++ b/boards/models/source.py @@ -0,0 +1,66 @@ +import feedparser +import logging + +from time import mktime +from datetime import datetime + +from django.db import models, transaction +from django.utils.dateparse import parse_datetime +from django.utils.timezone import utc +from django.utils import timezone +from boards.models import Post +from boards.models.post import TITLE_MAX_LENGTH + + +SOURCE_TYPE_MAX_LENGTH = 100 +SOURCE_TYPE_RSS = 'RSS' +TYPE_CHOICES = ( + (SOURCE_TYPE_RSS, SOURCE_TYPE_RSS), +) + + +class ThreadSource(models.Model): + class Meta: + app_label = 'boards' + + name = models.TextField() + thread = models.ForeignKey('Thread') + timestamp = models.DateTimeField() + source = models.TextField() + source_type = models.CharField(max_length=SOURCE_TYPE_MAX_LENGTH, + choices=TYPE_CHOICES) + + def __str__(self): + return self.name + + @transaction.atomic + def fetch_latest_posts(self): + """Creates new posts with the info fetched since the timestamp.""" + logger = logging.getLogger('boards.source') + + if self.thread.is_archived(): + logger.error('The thread {} is archived, please try another one'.format(self.thread)) + else: + start_timestamp = timezone.localtime(self.timestamp) + last_timestamp = start_timestamp + if self.thread.is_bumplimit(): + logger.warn('The thread {} has reached its bumplimit, please create a new one'.format(self.thread)) + if self.source_type == SOURCE_TYPE_RSS: + feed = feedparser.parse(self.source) + items = sorted(feed.entries, key=lambda entry: entry.published_parsed) + for item in items: + title = item.title[:TITLE_MAX_LENGTH] + timestamp = datetime.fromtimestamp(mktime(item.published_parsed), tz=utc) + if not timestamp: + logger.error('Invalid timestamp {} for {}'.format(item.published, title)) + else: + if timestamp > last_timestamp: + last_timestamp = timestamp + + if timestamp > start_timestamp: + Post.objects.create_post(title=title, text=item.description, thread=self.thread, file_urls=[item.link]) + logger.info('Fetched item {} from {} into thread {}'.format( + title, self.name, self.thread)) + self.timestamp = last_timestamp + self.save(update_fields=['timestamp']) + diff --git a/boards/models/thread.py b/boards/models/thread.py --- a/boards/models/thread.py +++ b/boards/models/thread.py @@ -233,7 +233,7 @@ class Thread(models.Model): return self.get_opening_post().pub_time def __str__(self): - return 'T#{}'.format(self.id) + return 'T#{}/{}'.format(self.id, self.get_opening_post()) def get_tag_url_list(self) -> list: return boards.models.Tag.objects.get_tag_url_list(self.get_tags().all()) @@ -262,6 +262,9 @@ class Thread(models.Model): def is_archived(self): return self.get_status() == STATUS_ARCHIVE + def is_bumplimit(self): + return self.get_status() == STATUS_BUMPLIMIT + def get_status(self): return self.status diff --git a/boards/settings.py b/boards/settings.py --- a/boards/settings.py +++ b/boards/settings.py @@ -1,9 +1,13 @@ import configparser +CONFIG_DEFAULT_SETTINGS = 'boards/config/default_settings.ini' +CONFIG_SETTINGS = 'boards/config/settings.ini' + + config = configparser.ConfigParser() -config.read('boards/config/default_settings.ini') -config.read('boards/config/settings.ini') +config.read(CONFIG_DEFAULT_SETTINGS) +config.read(CONFIG_SETTINGS) def get(section, name): @@ -22,3 +26,4 @@ def get_list_dict(section, name): str_dict = get(section, name) return [item.split(':') for item in str_dict.split(',')] + diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ bbcode django-debug-toolbar pytz ecdsa +feedparser