Download attached filed to the post during sync

boards/admin.py

@@ -1,7 +1,7 b''
 from django.contrib import admin
 from django.utils.translation import ugettext_lazy as _
 from django.core.urlresolvers import reverse
-from boards.models import Post, Tag, Ban, Thread, Banner, PostImage, KeyPair
+from boards.models import Post, Tag, Ban, Thread, Banner, PostImage, KeyPair, GlobalId
 
 
 @admin.register(Post)
@@ -109,3 +109,16 b' class BanAdmin(admin.ModelAdmin):'
 @admin.register(Banner)
 class BannerAdmin(admin.ModelAdmin):
     list_display = ('title', 'text')
+
+
[e-mail](PostImage)
+class PostImageAdmin(admin.ModelAdmin):
+    search_fields = ('alias',)
+
+
[e-mail](GlobalId)
+class GlobalIdAdmin(admin.ModelAdmin):
+    def is_linked(self, obj):
+        return Post.objects.filter(global_id=obj).exists()
+
+    list_display = ('__str__', 'is_linked',)
  No newline at end of file

boards/forms.py

@@ -15,7 +15,7 b' from django.utils import timezone'
 from boards.abstracts.settingsmanager import get_settings_manager
 from boards.abstracts.attachment_alias import get_image_by_alias
 from boards.mdx_neboard import formatters
-from boards.models.attachment.downloaders import Downloader
+from boards.models.attachment.downloaders import download
 from boards.models.post import TITLE_MAX_LENGTH
 from boards.models import Tag, Post
 from boards.utils import validate_file_size, get_file_mimetype, \
@@ -375,12 +375,7 b' class PostForm(NeboardForm):'
         img_temp = None
 
         try:
-            for downloader in Downloader.__subclasses__():
-                if downloader.handles(url):
-                    return downloader.download(url)
-            # If nobody of the specific downloaders handles this, use generic
-            # one
-            return Downloader.download(url)
+            download(url)
         except forms.ValidationError as e:
             raise e
         except Exception as e:

boards/management/commands/sync_with_server.py

@@ -26,6 +26,7 b' class Command(BaseCommand):'
 
         pull_url = url + 'api/sync/pull/'
         get_url = url + 'api/sync/get/'
+        file_url = url[:-1]
 
         global_id_str = options.get('global_id')
         if global_id_str:
@@ -43,7 +44,7 b' class Command(BaseCommand):'
                 h = httplib2.Http()
                 response, content = h.request(get_url, method="POST", body=xml)
 
-                SyncManager.parse_response_get(content)
+                SyncManager.parse_response_get(content, file_url)
             else:
                 raise Exception('Invalid global ID')
         else:
@@ -72,7 +73,7 b' class Command(BaseCommand):'
                     h = httplib2.Http()
                     response, content = h.request(get_url, method="POST", body=xml)
 
-                    SyncManager.parse_response_get(content)
+                    SyncManager.parse_response_get(content, file_url)
                 else:
                     print('Nothing to get, everything synced')
             else:

boards/models/attachment/downloaders.py

@@ -54,6 +54,15 b' class Downloader:'
             return file
 
 
+def download(url):
+    for downloader in Downloader.__subclasses__():
+        if downloader.handles(url):
+            return downloader.download(url)
+    # If nobody of the specific downloaders handles this, use generic
+    # one
+    return Downloader.download(url)
+
+
 class YouTubeDownloader(Downloader):
     @staticmethod
     def download(url: str):

boards/models/post/__init__.py

@@ -1,17 +1,7 b''
 import logging
-import re
 import uuid
 
-from django.core.exceptions import ObjectDoesNotExist
-from django.core.urlresolvers import reverse
-from django.db import models
-from django.db.models import TextField, QuerySet
-from django.template.defaultfilters import truncatewords, striptags
-from django.template.loader import render_to_string
-from django.utils import timezone
-from django.dispatch import receiver
-from django.db.models.signals import pre_save, post_save
-
+import re
 from boards import settings
 from boards.abstracts.tripcode import Tripcode
 from boards.mdx_neboard import get_parser
@@ -20,6 +10,16 b' from boards.models.base import Viewable'
 from boards.models.post.export import get_exporter, DIFF_TYPE_JSON
 from boards.models.post.manager import PostManager
 from boards.models.user import Notification
+from django.core.exceptions import ObjectDoesNotExist
+from django.core.urlresolvers import reverse
+from django.db import models
+from django.db.models import TextField, QuerySet
+from django.db.models.signals import pre_save, post_save, pre_delete, \
+    post_delete
+from django.dispatch import receiver
+from django.template.defaultfilters import truncatewords, striptags
+from django.template.loader import render_to_string
+from django.utils import timezone
 
 CSS_CLS_HIDDEN_POST = 'hidden_post'
 CSS_CLS_DEAD_POST = 'dead_post'
@@ -104,7 +104,8 b' class Post(models.Model, Viewable):'
 
     # Global ID with author key. If the message was downloaded from another
     # server, this indicates the server.
-    global_id = models.OneToOneField('GlobalId', null=True, blank=True)
+    global_id = models.OneToOneField(GlobalId, null=True, blank=True,
+                                     on_delete=models.CASCADE)
 
     tripcode = models.CharField(max_length=50, blank=True, default='')
     opening = models.BooleanField(db_index=True)
@@ -174,6 +175,7 b' class Post(models.Model, Viewable):'
 
     def get_thread_id(self):
         return self.thread_id
+
     def get_threads(self) -> QuerySet:
         """
         Gets post's thread.
@@ -220,33 +222,6 b' class Post(models.Model, Viewable):'
     def get_first_image(self) -> PostImage:
         return self.images.earliest('id')
 
-    def delete(self, using=None):
-        """
-        Deletes all post images and the post itself.
-        """
-
-        for image in self.images.all():
-            image_refs_count = image.post_images.count()
-            if image_refs_count == 1:
-                image.delete()
-
-        for attachment in self.attachments.all():
-            attachment_refs_count = attachment.attachment_posts.count()
-            if attachment_refs_count == 1:
-                attachment.delete()
-
-        if self.global_id:
-            self.global_id.delete()
-
-        thread = self.get_thread()
-        thread.last_edit_time = timezone.now()
-        thread.save()
-
-        super(Post, self).delete(using)
-
-        logging.getLogger('boards.post.delete').info(
-            'Deleted post {}'.format(self))
-
     def set_global_id(self, key_pair=None):
         """
         Sets global id based on the given key pair. If no key pair is given,
@@ -373,7 +348,7 b' class Post(models.Model, Viewable):'
             absolute_post_id = str(Post.objects.get(id=post_id).global_id)
             replacements[post_id] = absolute_post_id
 
-        text = self.get_raw_text()
+        text = self.get_raw_text() or ''
         for key in replacements:
             text = text.replace('[post]{}[/post]'.format(key),
                                 '[post]{}[/post]'.format(replacements[key]))
@@ -453,3 +428,26 b' def connect_notifications(instance, **kw'
 @receiver(pre_save, sender=Post)
 def preparse_text(instance, **kwargs):
     instance._text_rendered = get_parser().parse(instance.get_raw_text())
+
+
+@receiver(pre_delete, sender=Post)
+def delete_images(instance, **kwargs):
+    for image in instance.images.all():
+        image_refs_count = image.post_images.count()
+        if image_refs_count == 1:
+            image.delete()
+
+
+@receiver(pre_delete, sender=Post)
+def delete_attachments(instance, **kwargs):
+    for attachment in instance.attachments.all():
+        attachment_refs_count = attachment.attachment_posts.count()
+        if attachment_refs_count == 1:
+            attachment.delete()
+
+
+@receiver(post_delete, sender=Post)
+def update_thread_on_delete(instance, **kwargs):
+    thread = instance.get_thread()
+    thread.last_edit_time = timezone.now()
+    thread.save()

boards/models/post/manager.py

@@ -80,13 +80,8 b' class PostManager(models.Manager):'
         logger.info('Created post [{}] with text [{}] by {}'.format(post,
             post.get_text(),post.poster_ip))
 
-        # TODO Move this to other place
         if file:
-            file_type = file.name.split('.')[-1].lower()
-            if file_type in IMAGE_TYPES:
-                post.images.add(PostImage.objects.create_with_hash(file))
-            else:
-                post.attachments.add(Attachment.objects.create_with_hash(file))
+            self._add_file_to_post(file, post)
         for image in images:
             post.images.add(image)
 
@@ -134,7 +129,7 b' class PostManager(models.Manager):'
 
     @transaction.atomic
     def import_post(self, title: str, text: str, pub_time: str, global_id,
-                    opening_post=None, tags=list()):
+                    opening_post=None, tags=list(), files=list()):
         is_opening = opening_post is None
         if is_opening:
             thread = boards.models.thread.Thread.objects.create(
@@ -151,4 +146,15 b' class PostManager(models.Manager):'
                            opening=is_opening,
                            thread=thread)
 
+        # TODO Add files
+        for file in files:
+            self._add_file_to_post(file, post)
+
         post.threads.add(thread)
+
+    def _add_file_to_post(self, file, post):
+        file_type = file.name.split('.')[-1].lower()
+        if file_type in IMAGE_TYPES:
+            post.images.add(PostImage.objects.create_with_hash(file))
+        else:
+            post.attachments.add(Attachment.objects.create_with_hash(file))

boards/models/post/sync.py

@@ -1,5 +1,6 b''
 import xml.etree.ElementTree as et
 
+from boards.models.attachment.downloaders import download
 from boards.utils import get_file_mimetype
 from django.db import transaction
 from boards.models import KeyPair, GlobalId, Signature, Post, Tag
@@ -40,6 +41,10 b" ATTR_URL = 'url'"
 STATUS_SUCCESS = 'success'
 
 
+class SyncException(Exception):
+    pass
+
+
 class SyncManager:
     @staticmethod
     def generate_response_get(model_list: list):
@@ -118,7 +123,7 b' class SyncManager:'
 
     @staticmethod
     @transaction.atomic
-    def parse_response_get(response_xml):
+    def parse_response_get(response_xml, hostname):
         tag_root = et.fromstring(response_xml)
         tag_status = tag_root.find(TAG_STATUS)
         if STATUS_SUCCESS == tag_status.text:
@@ -139,8 +144,8 b' class SyncManager:'
                         signature.global_id = global_id
                         signature.save()
 
-                    title = tag_content.find(TAG_TITLE).text
-                    text = tag_content.find(TAG_TEXT).text
+                    title = tag_content.find(TAG_TITLE).text or ''
+                    text = tag_content.find(TAG_TEXT).text or ''
                     pub_time = tag_content.find(TAG_PUB_TIME).text
 
                     thread = tag_content.find(TAG_THREAD)
@@ -151,7 +156,7 b' class SyncManager:'
                         if exists:
                             opening_post = Post.objects.get(global_id=op_global_id)
                         else:
-                            raise Exception('Load the OP first')
+                            raise SyncException('Load the OP first')
                     else:
                         opening_post = None
                         tag_tags = tag_content.find(TAG_TAGS)
@@ -163,12 +168,23 b' class SyncManager:'
                     # TODO Check that the replied posts are already present
                     # before adding new ones
 
-                    # TODO Get images
+                    files = []
+                    tag_attachments = tag_content.find(TAG_ATTACHMENTS) or list()
+                    tag_refs = tag_model.find(TAG_ATTACHMENT_REFS)
+                    for attachment in tag_attachments:
+                        tag_ref = tag_refs.find("{}[@ref='{}']".format(
+                            TAG_ATTACHMENT_REF, attachment.text))
+                        url = tag_ref.get(ATTR_URL)
+                        attached_file = download(hostname + url)
+                        if attached_file is None:
+                            raise SyncException('File was not dowloaded')
+                        files.append(attached_file)
+                        # TODO Check hash
 
-                    post = Post.objects.import_post(
+                    Post.objects.import_post(
                         title=title, text=text, pub_time=pub_time,
                         opening_post=opening_post, tags=tags,
-                        global_id=global_id)
+                        global_id=global_id, files=files)
         else:
             # TODO Throw an exception?
             pass
@@ -210,7 +226,7 b' class SyncManager:'
 
             if not KeyPair.objects.verify(
                     signature, content):
-                raise Exception('Invalid model signature for {}'.format(content))
+                raise SyncException('Invalid model signature for {}'.format(content))
 
             signatures.append(signature)
 

boards/tests/test_sync.py

@@ -59,7 +59,7 b' class SyncTest(TestCase):'
 
         KeyPair.objects.generate_key(primary=True)
 
-        SyncManager.parse_response_get(response)
+        SyncManager.parse_response_get(response, None)
         self.assertEqual(1, Post.objects.count(),
                          'Post was not created from XML response.')
 
@@ -68,7 +68,7 b' class SyncTest(TestCase):'
                          parsed_post.get_thread().get_tags().first().name,
                          'Invalid tag was parsed.')
 
-        SyncManager.parse_response_get(response)
+        SyncManager.parse_response_get(response, None)
         self.assertEqual(1, Post.objects.count(),
                          'The same post was imported twice.')
 

boards/views/sync.py

@@ -1,4 +1,6 b''
 import xml.etree.ElementTree as et
+import xml.dom.minidom
+
 from django.http import HttpResponse, Http404
 from boards.models import GlobalId, Post
 from boards.models.post.sync import SyncManager
@@ -46,9 +48,13 b' def get_post_sync_data(request, post_id)'
     except Post.DoesNotExist:
         raise Http404()
 
-    content = 'Global ID: %s\n\nXML: %s' \
-              % (post.global_id, SyncManager.generate_response_get([post]))
+    xml_str = SyncManager.generate_response_get([post])
 
+    xml_repr = xml.dom.minidom.parseString(xml_str)
+    xml_repr = xml_repr.toprettyxml()
+
+    content = '=Global ID=\n%s\n\n=XML=\n%s' \
+              % (post.global_id, xml_repr)
 
     return HttpResponse(
         content_type='text/plain',