Show More
@@ -1,67 +1,68 b'' | |||
|
1 | 1 | import feedparser |
|
2 | 2 | import logging |
|
3 | import calendar | |
|
3 | 4 | |
|
4 | 5 | from time import mktime |
|
5 | 6 | from datetime import datetime |
|
6 | 7 | |
|
7 | 8 | from django.db import models, transaction |
|
8 | 9 | from django.utils.dateparse import parse_datetime |
|
9 | 10 | from django.utils.timezone import utc |
|
10 | 11 | from django.utils import timezone |
|
11 | 12 | from boards.models import Post |
|
12 | 13 | from boards.models.post import TITLE_MAX_LENGTH |
|
13 | 14 | |
|
14 | 15 | |
|
15 | 16 | SOURCE_TYPE_MAX_LENGTH = 100 |
|
16 | 17 | SOURCE_TYPE_RSS = 'RSS' |
|
17 | 18 | TYPE_CHOICES = ( |
|
18 | 19 | (SOURCE_TYPE_RSS, SOURCE_TYPE_RSS), |
|
19 | 20 | ) |
|
20 | 21 | |
|
21 | 22 | |
|
22 | 23 | class ThreadSource(models.Model): |
|
23 | 24 | class Meta: |
|
24 | 25 | app_label = 'boards' |
|
25 | 26 | |
|
26 | 27 | name = models.TextField() |
|
27 | 28 | thread = models.ForeignKey('Thread') |
|
28 | 29 | timestamp = models.DateTimeField() |
|
29 | 30 | source = models.TextField() |
|
30 | 31 | source_type = models.CharField(max_length=SOURCE_TYPE_MAX_LENGTH, |
|
31 | 32 | choices=TYPE_CHOICES) |
|
32 | 33 | |
|
33 | 34 | def __str__(self): |
|
34 | 35 | return self.name |
|
35 | 36 | |
|
36 | 37 | @transaction.atomic |
|
37 | 38 | def fetch_latest_posts(self): |
|
38 | 39 | """Creates new posts with the info fetched since the timestamp.""" |
|
39 | 40 | logger = logging.getLogger('boards.source') |
|
40 | 41 | |
|
41 | 42 | if self.thread.is_archived(): |
|
42 | 43 | logger.error('The thread {} is archived, please try another one'.format(self.thread)) |
|
43 | 44 | else: |
|
44 | 45 | start_timestamp = self.timestamp |
|
45 | 46 | last_timestamp = start_timestamp |
|
46 | 47 | logger.info('Start timestamp is {}'.format(start_timestamp)) |
|
47 | 48 | if self.thread.is_bumplimit(): |
|
48 | 49 | logger.warn('The thread {} has reached its bumplimit, please create a new one'.format(self.thread)) |
|
49 | 50 | if self.source_type == SOURCE_TYPE_RSS: |
|
50 | 51 | feed = feedparser.parse(self.source) |
|
51 | 52 | items = sorted(feed.entries, key=lambda entry: entry.published_parsed) |
|
52 | 53 | for item in items: |
|
53 | 54 | title = item.title[:TITLE_MAX_LENGTH] |
|
54 |
timestamp = datetime.fromtimestamp( |
|
|
55 | timestamp = datetime.fromtimestamp(calendar.timegm(item.published_parsed), tz=utc) | |
|
55 | 56 | if not timestamp: |
|
56 | 57 | logger.error('Invalid timestamp {} for {}'.format(item.published, title)) |
|
57 | 58 | else: |
|
58 | 59 | if timestamp > last_timestamp: |
|
59 | 60 | last_timestamp = timestamp |
|
60 | 61 | if timestamp > start_timestamp: |
|
61 | 62 | Post.objects.create_post(title=title, text=item.description, thread=self.thread, file_urls=[item.link]) |
|
62 | 63 | logger.info('Fetched item {} from {} into thread {}'.format( |
|
63 | 64 | title, self.name, self.thread)) |
|
64 | 65 | logger.info('New timestamp is {}'.format(last_timestamp)) |
|
65 | 66 | self.timestamp = last_timestamp |
|
66 | 67 | self.save(update_fields=['timestamp']) |
|
67 | 68 |
General Comments 0
You need to be logged in to leave comments.
Login now