Administration for sync sources. Now sync is available from the admin site instead of manual command line arguments. Still need to invoke sync_with_server by cron/timer

boards/migrations/0071_syncsource.py

@@ -0,0 +1,24 b''
+# Generated by Django 2.0.8 on 2018-08-05 17:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('boards', '0070_auto_20171225_1149'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SyncSource',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.TextField()),
+                ('timestamp', models.DateTimeField(blank=True, null=True)),
+                ('url', models.TextField()),
+                ('tags', models.TextField(blank=True)),
+                ('query_split_limit', models.IntegerField()),
+            ],
+        ),
+    ]

boards/admin.py

@@ -1,193 +1,197 b''
 from django.contrib import admin
 from django.urls import reverse
 from django.utils.translation import ugettext_lazy as _
 
 from boards.models import Post, Tag, Ban, Thread, Banner, Attachment, \
     KeyPair, GlobalId, TagAlias, STATUS_ACTIVE
 from boards.models.attachment import FILE_TYPES_IMAGE, AttachmentSticker, \
     StickerPack
-from boards.models.source import ThreadSource
+from boards.models.source import ThreadSource, SyncSource
 
 
 @admin.register(Post)
 class PostAdmin(admin.ModelAdmin):
 
     list_display = ('id', 'title', 'text', 'poster_ip', 'linked_images',
             'foreign', 'tags')
     list_filter = ('pub_time',)
     search_fields = ('id', 'title', 'text', 'poster_ip')
     exclude = ('referenced_posts', 'refmap', 'images', 'global_id')
     readonly_fields = ('poster_ip', 'thread', 'linked_images',
             'attachments', 'uid', 'url', 'pub_time', 'opening', 'linked_global_id',
             'foreign', 'tags')
 
     def ban_poster(self, request, queryset):
         bans = 0
         for post in queryset:
             poster_ip = post.poster_ip
             ban, created = Ban.objects.get_or_create(ip=poster_ip)
             if created:
                 bans += 1
         self.message_user(request, _('{} posters were banned').format(bans))
 
     def ban_latter_with_delete(self, request, queryset):
         bans = 0
         hidden = 0
         for post in queryset:
             poster_ip = post.poster_ip
             ban, created = Ban.objects.get_or_create(ip=poster_ip)
             if created:
                 bans += 1
             posts = Post.objects.filter(poster_ip=poster_ip, id__gte=post.id)
             hidden += posts.count()
             posts.delete()
         self.message_user(request, _('{} posters were banned, {} messages were removed.').format(bans, hidden))
     ban_latter_with_delete.short_description = _('Ban user and delete posts starting from this one and later')
 
     def linked_images(self, obj: Post):
         images = obj.attachments.filter(mimetype__in=FILE_TYPES_IMAGE)
         image_urls = ['<a href="{}"><img src="{}" /></a>'.format(
             reverse('admin:%s_%s_change' % (image._meta.app_label,
                                             image._meta.model_name),
                     args=[image.id]), image.get_thumb_url()) for image in images]
         return ', '.join(image_urls)
     linked_images.allow_tags = True
 
     def linked_global_id(self, obj: Post):
         global_id = obj.global_id
         if global_id is not None:
             return '<a href="{}">{}</a>'.format(
                 reverse('admin:%s_%s_change' % (global_id._meta.app_label,
                                                 global_id._meta.model_name),
                         args=[global_id.id]), str(global_id))
     linked_global_id.allow_tags = True
 
     def tags(self, obj: Post):
         return ', '.join([tag.get_name() for tag in obj.get_tags()])
 
     def save_model(self, request, obj, form, change):
         obj.save()
         obj.clear_cache()
 
     def foreign(self, obj: Post):
         return obj is not None and obj.global_id is not None and\
                 not obj.global_id.is_local()
 
     actions = ['ban_poster', 'ban_latter_with_delete']
 
 
 @admin.register(Tag)
 class TagAdmin(admin.ModelAdmin):
     def thread_count(self, obj: Tag) -> int:
         return obj.get_thread_count()
 
     def display_children(self, obj: Tag):
         return ', '.join([str(child) for child in obj.get_children().all()])
 
     def name(self, obj: Tag):
         return obj.get_name()
 
     def save_model(self, request, obj, form, change):
         super().save_model(request, obj, form, change)
         for thread in obj.get_threads().all():
             thread.refresh_tags()
 
     list_display = ('name', 'thread_count', 'display_children')
     search_fields = ('id',)
     readonly_fields = ('name',)
 
 
 @admin.register(TagAlias)
 class TagAliasAdmin(admin.ModelAdmin):
     list_display = ('locale', 'name', 'parent')
     list_filter = ('locale',)
     search_fields = ('name',)
 
 
 @admin.register(Thread)
 class ThreadAdmin(admin.ModelAdmin):
 
     def title(self, obj: Thread) -> str:
         return obj.get_opening_post().get_title()
 
     def reply_count(self, obj: Thread) -> int:
         return obj.get_reply_count()
 
     def ip(self, obj: Thread):
         return obj.get_opening_post().poster_ip
 
     def display_tags(self, obj: Thread):
         return ', '.join([str(tag) for tag in obj.get_tags().all()])
 
     def op(self, obj: Thread):
         return obj.get_opening_post_id()
 
     # Save parent tags when editing tags
     def save_related(self, request, form, formsets, change):
         super().save_related(request, form, formsets, change)
         form.instance.refresh_tags()
 
     def save_model(self, request, obj, form, change):
         op = obj.get_opening_post()
         obj.save()
         op.clear_cache()
 
     list_display = ('id', 'op', 'title', 'reply_count', 'status', 'ip',
             'display_tags')
     list_filter = ('bump_time', 'status')
     search_fields = ('id', 'title')
     filter_horizontal = ('tags',)
 
 
 @admin.register(KeyPair)
 class KeyPairAdmin(admin.ModelAdmin):
     list_display = ('public_key', 'primary')
     list_filter = ('primary',)
     search_fields = ('public_key',)
 
 
 @admin.register(Ban)
 class BanAdmin(admin.ModelAdmin):
     list_display = ('ip', 'can_read')
     list_filter = ('can_read',)
     search_fields = ('ip',)
 
 
 @admin.register(Banner)
 class BannerAdmin(admin.ModelAdmin):
     list_display = ('title', 'text')
 
 
 @admin.register(Attachment)
 class AttachmentAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'mimetype', 'file', 'url')
 
 
 @admin.register(AttachmentSticker)
 class AttachmentStickerAdmin(admin.ModelAdmin):
     search_fields = ('name',)
 
 
 @admin.register(StickerPack)
 class StickerPackAdmin(admin.ModelAdmin):
     search_fields = ('name',)
 
 
 @admin.register(GlobalId)
 class GlobalIdAdmin(admin.ModelAdmin):
     def is_linked(self, obj):
         return Post.objects.filter(global_id=obj).exists()
 
     list_display = ('__str__', 'is_linked',)
     readonly_fields = ('content',)
 
 
 @admin.register(ThreadSource)
 class ThreadSourceAdmin(admin.ModelAdmin):
     search_fields = ('name', 'source')
 
     def formfield_for_foreignkey(self, db_field, request, **kwargs):
         if db_field.name == 'thread':
             kwargs['queryset'] = Thread.objects.filter(status=STATUS_ACTIVE)
         return super().formfield_for_foreignkey(db_field, request, **kwargs)
 
+
[e-mail](SyncSource)
+class SyncSourceAdmin(admin.ModelAdmin):
+    search_fields = ('name',)
  No newline at end of file

boards/management/commands/sync_with_server.py

@@ -1,140 +1,13 b''
-import re
-import logging
-import xml.etree.ElementTree as ET
+from django.core.management import BaseCommand
 
-import httplib2
-from django.core.management import BaseCommand
-from django.utils.dateparse import parse_datetime
-
-from boards.models import GlobalId, KeyPair
-from boards.models.post.sync import SyncManager, TAG_ID, TAG_UPDATE_TIME
+from boards.models.source import SyncSource
 
 __author__ = 'neko259'
 
 
-REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)')
-
-
 class Command(BaseCommand):
-    help = 'Send a sync or get request to the server.'
-
-    def add_arguments(self, parser):
-        parser.add_argument('url', type=str, help='Server root url')
-        parser.add_argument('--global-id', type=str, default='',
-                            help='Post global ID')
-        parser.add_argument('--split-query', type=int, default=1,
-                            help='Split GET query into separate by the given'
-                                 ' number of posts in one')
-        parser.add_argument('--thread', type=int,
-                            help='Get posts of one specific thread')
-        parser.add_argument('--tags', type=str,
-                help='Get posts of the tags, comma-separated')
-        parser.add_argument('--time-from', type=str,
-                help='Get posts from the given timestamp')
+    help = 'Send a sync or get request to the servers.'
 
     def handle(self, *args, **options):
-        logger = logging.getLogger('boards.sync')
-
-        url = options.get('url')
-
-        list_url = url + 'api/sync/list/'
-        get_url = url + 'api/sync/get/'
-        file_url = url[:-1]
-
-        global_id_str = options.get('global_id')
-        if global_id_str:
-            match = REGEX_GLOBAL_ID.match(global_id_str)
-            if match:
-                key_type = match.group(1)
-                key = match.group(2)
-                local_id = match.group(3)
-
-                global_id = GlobalId(key_type=key_type, key=key,
-                                     local_id=local_id)
-
-                xml = SyncManager.generate_request_get([global_id])
-                h = httplib2.Http()
-                response, content = h.request(get_url, method="POST", body=xml)
-
-                SyncManager.parse_response_get(content, file_url)
-            else:
-                raise Exception('Invalid global ID')
-        else:
-            logger.info('Running LIST request...')
-            h = httplib2.Http()
-
-            tags = []
-            tags_str = options.get('tags')
-            if tags_str:
-                tags = tags_str.split(',')
-
-            timestamp_str = options.get('time_from')
-            timestamp = None
-            if timestamp_str:
-                timestamp = parse_datetime(timestamp_str)
-                if not timestamp:
-                    raise Exception('Timestamp {} cannot be parsed'.format(
-                        timestamp_str))
-
-            xml = SyncManager.generate_request_list(
-                opening_post=options.get('thread'), tags=tags,
-                timestamp_from=timestamp).encode()
-            response, content = h.request(list_url, method="POST", body=xml)
-            if response.status != 200:
-                raise Exception('Server returned error {}'.format(response.status))
-
-            logger.info('Processing response...')
-
-            root = ET.fromstring(content)
-            status = root.findall('status')[0].text
-            if status == 'success':
-                ids_to_sync = list()
-
-                models = root.findall('models')[0]
-                for model in models:
-                    self.add_to_sync_list(ids_to_sync, logger, model)
-                logger.info('Starting sync...')
-
-                if len(ids_to_sync) > 0:
-                    limit = options.get('split_query', len(ids_to_sync))
-                    for offset in range(0, len(ids_to_sync), limit):
-                        xml = SyncManager.generate_request_get(ids_to_sync[offset:offset + limit])
-                        h = httplib2.Http()
-                        logger.info('Running GET request...')
-                        response, content = h.request(get_url, method="POST", body=xml)
-                        logger.info('Processing response...')
-
-                        SyncManager.parse_response_get(content, file_url)
-
-                        logger.info('Sync completed successfully')
-                else:
-                    logger.info('Nothing to get, everything synced')
-            else:
-                raise Exception('Invalid response status')
-
-    def add_to_sync_list(self, ids_to_sync, logger, model):
-        tag_id = model.find(TAG_ID)
-        global_id, exists = GlobalId.from_xml_element(tag_id)
-        from_this_board = self._is_from_this_board(global_id)
-        if from_this_board:
-            # If the post originates from this board, no need to process
-            # it again, nobody else could modify it
-            logger.debug('NO SYNC Processed post {}'.format(global_id))
-        else:
-            tag_update_time = model.find(TAG_UPDATE_TIME)
-            if tag_update_time:
-                update_time = tag_update_time.text
-            else:
-                update_time = None
-            if not exists or update_time is None or global_id.post.last_edit_time < parse_datetime(
-                    update_time):
-                logger.debug('SYNC Processed post {}'.format(global_id))
-                ids_to_sync.append(global_id)
-            else:
-                logger.debug('NO SYNC Processed post {}'.format(global_id))
-
-    def _is_from_this_board(self, global_id):
-        from_this_board = KeyPair.objects.filter(
-            key_type=global_id.key_type,
-            public_key=global_id.key).exists()
-        return from_this_board
+        for source in SyncSource.objects.all():
+            source.run_sync()

boards/models/source.py

@@ -1,85 +1,195 b''
 import feedparser
+import httplib2
 import logging
 import calendar
+import xml.etree.ElementTree as ET
 
 from time import mktime
 from datetime import datetime
 
 from django.db import models, transaction
 from django.utils.dateparse import parse_datetime
 from django.utils.timezone import utc
 from django.utils import timezone
 from django.utils.html import strip_tags
 
-from boards.models import Post
+from boards.models import Post, GlobalId, KeyPair
 from boards.models.post import TITLE_MAX_LENGTH
+from boards.models.post.sync import SyncManager, TAG_ID, TAG_UPDATE_TIME
 from boards.settings import SECTION_EXTERNAL
 from boards.utils import get_tripcode_from_text
 from boards import settings
 
+DELIMITER_TAGS = ','
 
 SOURCE_TYPE_MAX_LENGTH = 100
 SOURCE_TYPE_RSS = 'RSS'
 TYPE_CHOICES = (
         (SOURCE_TYPE_RSS, SOURCE_TYPE_RSS),
 )
 
 
 class ThreadSource(models.Model):
     class Meta:
         app_label = 'boards'
 
     name = models.TextField()
     thread = models.ForeignKey('Thread', on_delete=models.CASCADE)
     timestamp = models.DateTimeField()
     source = models.TextField()
     source_type = models.CharField(max_length=SOURCE_TYPE_MAX_LENGTH,
             choices=TYPE_CHOICES)
 
     def __str__(self):
         return self.name
 
     @transaction.atomic
     def fetch_latest_posts(self):
         """Creates new posts with the info fetched since the timestamp."""
         logger = logging.getLogger('boards.source')
 
         if self.thread.is_archived():
             logger.error('The thread {} is archived, please try another one'.format(self.thread))
         else:
             tripcode = get_tripcode_from_text(
                     settings.get(SECTION_EXTERNAL, 'SourceFetcherTripcode'))
             start_timestamp = self.timestamp
             last_timestamp = start_timestamp
             logger.info('Start timestamp is {}'.format(start_timestamp))
             if self.thread.is_bumplimit():
                 logger.warn('The thread {} has reached its bumplimit, please create a new one'.format(self.thread))
             if self.source_type == SOURCE_TYPE_RSS:
                 feed = feedparser.parse(self.source)
                 items = sorted(feed.entries, key=lambda entry: entry.published_parsed)
                 for item in items:
                     title = self.strip_title(item.title, TITLE_MAX_LENGTH)
                     timestamp = datetime.fromtimestamp(calendar.timegm(item.published_parsed), tz=utc)
                     if not timestamp:
                         logger.error('Invalid timestamp {} for {}'.format(item.published, title))
                     else:
                         if timestamp > last_timestamp:
                             last_timestamp = timestamp
                         if timestamp > start_timestamp:
                             Post.objects.create_post(title=title, text=self.parse_text(item.description),
                                     thread=self.thread, file_urls=[item.link], tripcode=tripcode)
                             logger.info('Fetched item {} from {} into thread {}'.format(
                                 title, self.name, self.thread))
             logger.info('New timestamp is {}'.format(last_timestamp))
             self.timestamp = last_timestamp
             self.save(update_fields=['timestamp'])
 
     def parse_text(self, text):
         return strip_tags(text)
 
     def strip_title(self, title, max_length):
         result = title
         if len(title) > max_length:
             result = title[:max_length - 1] + '…'
         return result
 
+
+class SyncSource(models.Model):
+    class Meta:
+        app_label = 'boards'
+
+    name = models.TextField()
+    timestamp = models.DateTimeField(blank=True, null=True)
+    url = models.TextField()
+    tags = models.TextField(blank=True)
+    query_split_limit = models.IntegerField()
+
+    def __str__(self):
+        return self.name
+
+    @transaction.atomic
+    def run_sync(self):
+        logger = logging.getLogger('boards.sync')
+
+        tags = []
+        if self.tags:
+            tags = self.tags.split(DELIMITER_TAGS)
+
+        timestamp = None
+        if self.timestamp:
+            timestamp = self.timestamp
+
+        new_timestamp = timezone.now()
+
+        list_url = '{}api/sync/list/'.format(self.url)
+        get_url = '{}api/sync/get/'.format(self.url)
+        file_url = self.url[:-1]
+
+        xml = SyncManager.generate_request_list(
+            tags=tags,
+            timestamp_from=timestamp).encode()
+
+        logger.info('Running LIST request for {}...'.format(self.name))
+        h = httplib2.Http()
+        response, content = h.request(list_url, method="POST", body=xml)
+        if response.status != 200:
+            raise Exception('Server returned error {}'.format(response.status))
+
+        logger.info('Processing response...')
+
+        root = ET.fromstring(content)
+        status = root.findall('status')[0].text
+        if status == 'success':
+            ids_to_sync = list()
+
+            models = root.findall('models')[0]
+            for model in models:
+                self.add_to_sync_list(ids_to_sync, logger, model)
+            logger.info('Starting sync...')
+
+            if len(ids_to_sync) > 0:
+                if self.query_split_limit > 0:
+                    limit = min(self.query_split_limit, len(ids_to_sync))
+                else:
+                    limit = len(ids_to_sync)
+
+                for offset in range(0, len(ids_to_sync), limit):
+                    xml = SyncManager.generate_request_get(
+                        ids_to_sync[offset:offset + limit])
+                    h = httplib2.Http()
+                    logger.info('Running GET request...')
+                    response, content = h.request(get_url, method="POST",
+                                                  body=xml)
+                    logger.info('Processing response...')
+
+                    SyncManager.parse_response_get(content, file_url)
+
+                    logger.info('Sync completed successfully for {}'.format(self.name))
+            else:
+                logger.info('Nothing to get for {}, everything synced'.format(self.name))
+        else:
+            raise Exception('Invalid response status')
+
+        self.timestamp = new_timestamp
+        self.save(update_fields=['timestamp'])
+
+    def add_to_sync_list(self, ids_to_sync, logger, model):
+        tag_id = model.find(TAG_ID)
+        global_id, exists = GlobalId.from_xml_element(tag_id)
+        from_this_board = self._is_from_this_board(global_id)
+        if from_this_board:
+            # If the post originates from this board, no need to process
+            # it again, nobody else could modify it
+            logger.debug('NO SYNC Processed post {}'.format(global_id))
+        else:
+            tag_update_time = model.find(TAG_UPDATE_TIME)
+            if tag_update_time:
+                update_time = tag_update_time.text
+            else:
+                update_time = None
+            if not exists or update_time is None or global_id.post.last_edit_time < parse_datetime(
+                    update_time):
+                logger.debug('SYNC Processed post {}'.format(global_id))
+                ids_to_sync.append(global_id)
+            else:
+                logger.debug('NO SYNC Processed post {}'.format(global_id))
+
+    def _is_from_this_board(self, global_id):
+        from_this_board = KeyPair.objects.filter(
+            key_type=global_id.key_type,
+            public_key=global_id.key).exists()
+        return from_this_board