source.py
68 lines
| 2.6 KiB
| text/x-python
|
PythonLexer
|
r1968 | import feedparser | ||
import logging | ||||
|
r1970 | import calendar | ||
|
r1968 | |||
from time import mktime | ||||
from datetime import datetime | ||||
from django.db import models, transaction | ||||
from django.utils.dateparse import parse_datetime | ||||
from django.utils.timezone import utc | ||||
from django.utils import timezone | ||||
from boards.models import Post | ||||
from boards.models.post import TITLE_MAX_LENGTH | ||||
SOURCE_TYPE_MAX_LENGTH = 100 | ||||
SOURCE_TYPE_RSS = 'RSS' | ||||
TYPE_CHOICES = ( | ||||
(SOURCE_TYPE_RSS, SOURCE_TYPE_RSS), | ||||
) | ||||
class ThreadSource(models.Model): | ||||
class Meta: | ||||
app_label = 'boards' | ||||
name = models.TextField() | ||||
thread = models.ForeignKey('Thread') | ||||
timestamp = models.DateTimeField() | ||||
source = models.TextField() | ||||
source_type = models.CharField(max_length=SOURCE_TYPE_MAX_LENGTH, | ||||
choices=TYPE_CHOICES) | ||||
def __str__(self): | ||||
return self.name | ||||
@transaction.atomic | ||||
def fetch_latest_posts(self): | ||||
"""Creates new posts with the info fetched since the timestamp.""" | ||||
logger = logging.getLogger('boards.source') | ||||
if self.thread.is_archived(): | ||||
logger.error('The thread {} is archived, please try another one'.format(self.thread)) | ||||
else: | ||||
|
r1969 | start_timestamp = self.timestamp | ||
|
r1968 | last_timestamp = start_timestamp | ||
|
r1969 | logger.info('Start timestamp is {}'.format(start_timestamp)) | ||
|
r1968 | if self.thread.is_bumplimit(): | ||
logger.warn('The thread {} has reached its bumplimit, please create a new one'.format(self.thread)) | ||||
if self.source_type == SOURCE_TYPE_RSS: | ||||
feed = feedparser.parse(self.source) | ||||
items = sorted(feed.entries, key=lambda entry: entry.published_parsed) | ||||
for item in items: | ||||
title = item.title[:TITLE_MAX_LENGTH] | ||||
|
r1970 | timestamp = datetime.fromtimestamp(calendar.timegm(item.published_parsed), tz=utc) | ||
|
r1968 | if not timestamp: | ||
logger.error('Invalid timestamp {} for {}'.format(item.published, title)) | ||||
else: | ||||
if timestamp > last_timestamp: | ||||
last_timestamp = timestamp | ||||
if timestamp > start_timestamp: | ||||
Post.objects.create_post(title=title, text=item.description, thread=self.thread, file_urls=[item.link]) | ||||
logger.info('Fetched item {} from {} into thread {}'.format( | ||||
title, self.name, self.thread)) | ||||
|
r1969 | logger.info('New timestamp is {}'.format(last_timestamp)) | ||
|
r1968 | self.timestamp = last_timestamp | ||
self.save(update_fields=['timestamp']) | ||||