##// END OF EJS Templates
Properly parse feed timestamp
neko259 -
r1970:075e04c3 default
parent child Browse files
Show More
@@ -1,67 +1,68 b''
1 1 import feedparser
2 2 import logging
3 import calendar
3 4
4 5 from time import mktime
5 6 from datetime import datetime
6 7
7 8 from django.db import models, transaction
8 9 from django.utils.dateparse import parse_datetime
9 10 from django.utils.timezone import utc
10 11 from django.utils import timezone
11 12 from boards.models import Post
12 13 from boards.models.post import TITLE_MAX_LENGTH
13 14
14 15
15 16 SOURCE_TYPE_MAX_LENGTH = 100
16 17 SOURCE_TYPE_RSS = 'RSS'
17 18 TYPE_CHOICES = (
18 19 (SOURCE_TYPE_RSS, SOURCE_TYPE_RSS),
19 20 )
20 21
21 22
22 23 class ThreadSource(models.Model):
23 24 class Meta:
24 25 app_label = 'boards'
25 26
26 27 name = models.TextField()
27 28 thread = models.ForeignKey('Thread')
28 29 timestamp = models.DateTimeField()
29 30 source = models.TextField()
30 31 source_type = models.CharField(max_length=SOURCE_TYPE_MAX_LENGTH,
31 32 choices=TYPE_CHOICES)
32 33
33 34 def __str__(self):
34 35 return self.name
35 36
36 37 @transaction.atomic
37 38 def fetch_latest_posts(self):
38 39 """Creates new posts with the info fetched since the timestamp."""
39 40 logger = logging.getLogger('boards.source')
40 41
41 42 if self.thread.is_archived():
42 43 logger.error('The thread {} is archived, please try another one'.format(self.thread))
43 44 else:
44 45 start_timestamp = self.timestamp
45 46 last_timestamp = start_timestamp
46 47 logger.info('Start timestamp is {}'.format(start_timestamp))
47 48 if self.thread.is_bumplimit():
48 49 logger.warn('The thread {} has reached its bumplimit, please create a new one'.format(self.thread))
49 50 if self.source_type == SOURCE_TYPE_RSS:
50 51 feed = feedparser.parse(self.source)
51 52 items = sorted(feed.entries, key=lambda entry: entry.published_parsed)
52 53 for item in items:
53 54 title = item.title[:TITLE_MAX_LENGTH]
54 timestamp = datetime.fromtimestamp(mktime(item.published_parsed), tz=utc)
55 timestamp = datetime.fromtimestamp(calendar.timegm(item.published_parsed), tz=utc)
55 56 if not timestamp:
56 57 logger.error('Invalid timestamp {} for {}'.format(item.published, title))
57 58 else:
58 59 if timestamp > last_timestamp:
59 60 last_timestamp = timestamp
60 61 if timestamp > start_timestamp:
61 62 Post.objects.create_post(title=title, text=item.description, thread=self.thread, file_urls=[item.link])
62 63 logger.info('Fetched item {} from {} into thread {}'.format(
63 64 title, self.name, self.thread))
64 65 logger.info('New timestamp is {}'.format(last_timestamp))
65 66 self.timestamp = last_timestamp
66 67 self.save(update_fields=['timestamp'])
67 68
General Comments 0
You need to be logged in to leave comments. Login now