Allow syncing URL attachments

boards/models/attachment/__init__.py

@@ -1,88 +1,91 b''
 import boards
 from boards.models import STATUS_ARCHIVE
 from django.core.files.images import get_image_dimensions
 from django.db import models
 
 from boards import utils
 from boards.models.attachment.viewers import get_viewers, AbstractViewer, \
     FILE_TYPES_IMAGE
 from boards.utils import get_upload_filename, get_extension, cached_result
 
 
 class AttachmentManager(models.Manager):
     def create_with_hash(self, file):
         file_hash = utils.get_file_hash(file)
         existing = self.filter(hash=file_hash)
         if len(existing) > 0:
             attachment = existing[0]
         else:
             # FIXME Use full mimetype here, need to modify viewers too
             file_type = get_extension(file.name)
             attachment = self.create(file=file, mimetype=file_type,
                                      hash=file_hash)
 
         return attachment
 
     def create_from_url(self, url):
         existing = self.filter(url=url)
         if len(existing) > 0:
             attachment = existing[0]
         else:
             attachment = self.create(url=url)
         return attachment
 
     def get_random_images(self, count, tags=None):
         images = self.filter(mimetype__in=FILE_TYPES_IMAGE).exclude(
             attachment_posts__thread__status=STATUS_ARCHIVE)
         if tags is not None:
             images = images.filter(attachment_posts__threads__tags__in=tags)
         return images.order_by('?')[:count]
 
 
 class Attachment(models.Model):
     objects = AttachmentManager()
 
     file = models.FileField(upload_to=get_upload_filename, null=True)
     mimetype = models.CharField(max_length=50, null=True)
     hash = models.CharField(max_length=36, null=True)
     alias = models.TextField(unique=True, null=True, blank=True)
     url = models.TextField(null=True, blank=True)
 
     def get_view(self):
         file_viewer = None
         for viewer in get_viewers():
             if viewer.supports(self.mimetype):
                 file_viewer = viewer
                 break
         if file_viewer is None:
             file_viewer = AbstractViewer
 
         return file_viewer(self.file, self.mimetype, self.hash, self.url).get_view()
 
     def __str__(self):
         return self.url or self.file.url
 
     def get_random_associated_post(self):
         posts = boards.models.Post.objects.filter(attachments__in=[self])
         return posts.order_by('?').first()
 
     @cached_result()
     def get_size(self):
         if self.file:
             if self.mimetype in FILE_TYPES_IMAGE:
                 return get_image_dimensions(self.file)
             else:
                 return 200, 150
 
     def get_thumb_url(self):
         split = self.file.url.rsplit('.', 1)
         w, h = 200, 150
         return '%s.%sx%s.%s' % (split[0], w, h, split[1])
 
     @cached_result()
     def get_preview_size(self):
         if self.mimetype in FILE_TYPES_IMAGE:
             preview_path = self.file.path.replace('.', '.200x150.')
             return get_image_dimensions(preview_path)
         else:
             return 200, 150
+
+    def is_internal(self):
+        return self.url is None or len(self.url) == 0

boards/models/post/sync.py

@@ -1,302 +1,323 b''
 import xml.etree.ElementTree as et
 import logging
 
 from boards.abstracts.exceptions import SyncException
 from boards.models import KeyPair, GlobalId, Signature, Post, Tag
 from boards.models.attachment.downloaders import download
 from boards.utils import get_file_mimetype, get_file_hash
 from django.db import transaction
 
 EXCEPTION_NODE = 'Sync node returned an error: {}.'
 EXCEPTION_OP = 'Load the OP first.'
 EXCEPTION_DOWNLOAD = 'File was not downloaded.'
 EXCEPTION_HASH = 'File hash does not match attachment hash.'
 EXCEPTION_SIGNATURE = 'Invalid model signature for {}.'
 EXCEPTION_AUTHOR_SIGNATURE = 'Model {} has no author signature.'
 ENCODING_UNICODE = 'unicode'
 
 TAG_MODEL = 'model'
 TAG_REQUEST = 'request'
 TAG_RESPONSE = 'response'
 TAG_ID = 'id'
 TAG_STATUS = 'status'
 TAG_MODELS = 'models'
 TAG_TITLE = 'title'
 TAG_TEXT = 'text'
 TAG_THREAD = 'thread'
 TAG_PUB_TIME = 'pub-time'
 TAG_SIGNATURES = 'signatures'
 TAG_SIGNATURE = 'signature'
 TAG_CONTENT = 'content'
 TAG_ATTACHMENTS = 'attachments'
 TAG_ATTACHMENT = 'attachment'
 TAG_TAGS = 'tags'
 TAG_TAG = 'tag'
 TAG_ATTACHMENT_REFS = 'attachment-refs'
 TAG_ATTACHMENT_REF = 'attachment-ref'
 TAG_TRIPCODE = 'tripcode'
 TAG_VERSION = 'version'
 
 TYPE_GET = 'get'
 
 ATTR_VERSION = 'version'
 ATTR_TYPE = 'type'
 ATTR_NAME = 'name'
 ATTR_VALUE = 'value'
 ATTR_MIMETYPE = 'mimetype'
 ATTR_KEY = 'key'
 ATTR_REF = 'ref'
 ATTR_URL = 'url'
 ATTR_ID_TYPE = 'id-type'
 
 ID_TYPE_MD5 = 'md5'
+ID_TYPE_URL = 'url'
 
 STATUS_SUCCESS = 'success'
 
 
 logger = logging.getLogger('boards.sync')
 
 
 class SyncManager:
     @staticmethod
     def generate_response_get(model_list: list):
         response = et.Element(TAG_RESPONSE)
 
         status = et.SubElement(response, TAG_STATUS)
         status.text = STATUS_SUCCESS
 
         models = et.SubElement(response, TAG_MODELS)
 
         for post in model_list:
             model = et.SubElement(models, TAG_MODEL)
             model.set(ATTR_NAME, 'post')
 
             global_id = post.global_id
 
             attachments = post.attachments.all()
             if global_id.content:
                 model.append(et.fromstring(global_id.content))
                 if len(attachments) > 0:
-                    attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS)
-                    for file in attachments:
-                        SyncManager._attachment_to_xml(
-                            None, attachment_refs, file.file.file,
-                            file.hash, file.file.url)
+                    internal_attachments = False
+                    for attachment in attachments:
+                        if attachment.is_internal():
+                            internal_attachments = True
+                            break
+
+                    if internal_attachments:
+                        attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS)
+                        for file in attachments:
+                            SyncManager._attachment_to_xml(
+                                None, attachment_refs, file)
             else:
                 content_tag = et.SubElement(model, TAG_CONTENT)
 
                 tag_id = et.SubElement(content_tag, TAG_ID)
                 global_id.to_xml_element(tag_id)
 
                 title = et.SubElement(content_tag, TAG_TITLE)
                 title.text = post.title
 
                 text = et.SubElement(content_tag, TAG_TEXT)
                 text.text = post.get_sync_text()
 
                 thread = post.get_thread()
                 if post.is_opening():
                     tag_tags = et.SubElement(content_tag, TAG_TAGS)
                     for tag in thread.get_tags():
                         tag_tag = et.SubElement(tag_tags, TAG_TAG)
                         tag_tag.text = tag.name
                 else:
                     tag_thread = et.SubElement(content_tag, TAG_THREAD)
                     thread_id = et.SubElement(tag_thread, TAG_ID)
                     thread.get_opening_post().global_id.to_xml_element(thread_id)
 
                 pub_time = et.SubElement(content_tag, TAG_PUB_TIME)
                 pub_time.text = str(post.get_pub_time_str())
 
                 if post.tripcode:
                     tripcode = et.SubElement(content_tag, TAG_TRIPCODE)
                     tripcode.text = post.tripcode
 
                 if len(attachments) > 0:
                     attachments_tag = et.SubElement(content_tag, TAG_ATTACHMENTS)
-                    attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS)
+                    
+                    internal_attachments = False
+                    for attachment in attachments:
+                        if attachment.is_internal():
+                            internal_attachments = True
+                            break
+
+                    if internal_attachments:
+                        attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS)
+                    else:
+                        attachment_refs = None
 
                     for file in attachments:
                         SyncManager._attachment_to_xml(
-                            attachments_tag, attachment_refs, file.file.file,
-                            file.hash, file.file.url)
+                            attachments_tag, attachment_refs, file)
                 version_tag = et.SubElement(content_tag, TAG_VERSION)
                 version_tag.text = str(post.version)
 
                 global_id.content = et.tostring(content_tag, ENCODING_UNICODE)
                 global_id.save()
 
             signatures_tag = et.SubElement(model, TAG_SIGNATURES)
             post_signatures = global_id.signature_set.all()
             if post_signatures:
                 signatures = post_signatures
             else:
                 key = KeyPair.objects.get(public_key=global_id.key)
                 signature = Signature(
                     key_type=key.key_type,
                     key=key.public_key,
                     signature=key.sign(global_id.content),
                     global_id=global_id,
                 )
                 signature.save()
                 signatures = [signature]
             for signature in signatures:
                 signature_tag = et.SubElement(signatures_tag, TAG_SIGNATURE)
                 signature_tag.set(ATTR_TYPE, signature.key_type)
                 signature_tag.set(ATTR_VALUE, signature.signature)
                 signature_tag.set(ATTR_KEY, signature.key)
 
         return et.tostring(response, ENCODING_UNICODE)
 
     @staticmethod
     @transaction.atomic
     def parse_response_get(response_xml, hostname):
         tag_root = et.fromstring(response_xml)
         tag_status = tag_root.find(TAG_STATUS)
         if STATUS_SUCCESS == tag_status.text:
             tag_models = tag_root.find(TAG_MODELS)
             for tag_model in tag_models:
                 tag_content = tag_model.find(TAG_CONTENT)
 
                 content_str = et.tostring(tag_content, ENCODING_UNICODE)
 
                 tag_id = tag_content.find(TAG_ID)
                 global_id, exists = GlobalId.from_xml_element(tag_id)
                 signatures = SyncManager._verify_model(global_id, content_str, tag_model)
 
                 version = int(tag_content.find(TAG_VERSION).text)
                 is_old = exists and global_id.post.version < version
                 if exists and not is_old:
                     print('Post with same ID exists and is up to date.')
                 else:
                     global_id.content = content_str
                     global_id.save()
                     for signature in signatures:
                         signature.global_id = global_id
                         signature.save()
 
                     title = tag_content.find(TAG_TITLE).text or ''
                     text = tag_content.find(TAG_TEXT).text or ''
                     pub_time = tag_content.find(TAG_PUB_TIME).text
                     tripcode_tag = tag_content.find(TAG_TRIPCODE)
                     if tripcode_tag is not None:
                         tripcode = tripcode_tag.text or ''
                     else:
                         tripcode = ''
 
                     thread = tag_content.find(TAG_THREAD)
                     tags = []
                     if thread:
                         thread_id = thread.find(TAG_ID)
                         op_global_id, exists = GlobalId.from_xml_element(thread_id)
                         if exists:
                             opening_post = Post.objects.get(global_id=op_global_id)
                         else:
                             raise SyncException(EXCEPTION_OP)
                     else:
                         opening_post = None
                         tag_tags = tag_content.find(TAG_TAGS)
                         for tag_tag in tag_tags:
                             tag, created = Tag.objects.get_or_create(
                                 name=tag_tag.text)
                             tags.append(tag)
 
                     # TODO Check that the replied posts are already present
                     # before adding new ones
 
                     files = []
                     tag_attachments = tag_content.find(TAG_ATTACHMENTS) or list()
                     tag_refs = tag_model.find(TAG_ATTACHMENT_REFS)
                     for attachment in tag_attachments:
                         tag_ref = tag_refs.find("{}[@ref='{}']".format(
                             TAG_ATTACHMENT_REF, attachment.text))
                         url = tag_ref.get(ATTR_URL)
                         attached_file = download(hostname + url)
                         if attached_file is None:
                             raise SyncException(EXCEPTION_DOWNLOAD)
 
                         hash = get_file_hash(attached_file)
                         if hash != attachment.text:
                             raise SyncException(EXCEPTION_HASH)
 
                         files.append(attached_file)
 
                     if is_old:
                         post = global_id.post
                         Post.objects.update_post(
                             post, title=title, text=text, pub_time=pub_time,
                             tags=tags, files=files, tripcode=tripcode,
                             version=version)
                         logger.debug('Parsed updated post {}'.format(global_id))
                     else:
                         Post.objects.import_post(
                             title=title, text=text, pub_time=pub_time,
                             opening_post=opening_post, tags=tags,
                             global_id=global_id, files=files, tripcode=tripcode,
                             version=version)
                         logger.debug('Parsed new post {}'.format(global_id))
         else:
             raise SyncException(EXCEPTION_NODE.format(tag_status.text))
 
     @staticmethod
     def generate_response_list():
         response = et.Element(TAG_RESPONSE)
 
         status = et.SubElement(response, TAG_STATUS)
         status.text = STATUS_SUCCESS
 
         models = et.SubElement(response, TAG_MODELS)
 
         for post in Post.objects.prefetch_related('global_id').all():
             tag_model = et.SubElement(models, TAG_MODEL)
             tag_id = et.SubElement(tag_model, TAG_ID)
             post.global_id.to_xml_element(tag_id)
             tag_version = et.SubElement(tag_model, TAG_VERSION)
             tag_version.text = str(post.version)
 
         return et.tostring(response, ENCODING_UNICODE)
 
     @staticmethod
     def _verify_model(global_id, content_str, tag_model):
         """
         Verifies all signatures for a single model.
         """
 
         signatures = []
 
         tag_signatures = tag_model.find(TAG_SIGNATURES)
         has_author_signature = False
         for tag_signature in tag_signatures:
             signature_type = tag_signature.get(ATTR_TYPE)
             signature_value = tag_signature.get(ATTR_VALUE)
             signature_key = tag_signature.get(ATTR_KEY)
 
             if global_id.key_type == signature_type and\
                     global_id.key == signature_key:
                 has_author_signature = True
 
             signature = Signature(key_type=signature_type,
                                   key=signature_key,
                                   signature=signature_value)
 
             if not KeyPair.objects.verify(signature, content_str):
                 raise SyncException(EXCEPTION_SIGNATURE.format(content_str))
 
             signatures.append(signature)
         if not has_author_signature:
             raise SyncException(EXCEPTION_AUTHOR_SIGNATURE.format(content_str))
 
         return signatures
 
     @staticmethod
-    def _attachment_to_xml(tag_attachments, tag_refs, file, hash, url):
+    def _attachment_to_xml(tag_attachments, tag_refs, attachment):
         if tag_attachments is not None:
-            mimetype = get_file_mimetype(file)
-            attachment = et.SubElement(tag_attachments, TAG_ATTACHMENT)
-            attachment.set(ATTR_MIMETYPE, mimetype)
-            attachment.set(ATTR_ID_TYPE, ID_TYPE_MD5)
-            attachment.text = hash
+            attachment_tag = et.SubElement(tag_attachments, TAG_ATTACHMENT)
+            if attachment.is_internal():
+                mimetype = get_file_mimetype(attachment.file.file)
+                attachment_tag.set(ATTR_MIMETYPE, mimetype)
+                attachment_tag.set(ATTR_ID_TYPE, ID_TYPE_MD5)
+                attachment_tag.text = attachment.hash
+            else:
+                attachment_tag.set(ATTR_ID_TYPE, ID_TYPE_URL)
+                attachment_tag.text = attachment.url
 
-        attachment_ref = et.SubElement(tag_refs, TAG_ATTACHMENT_REF)
-        attachment_ref.set(ATTR_REF, hash)
-        attachment_ref.set(ATTR_URL, url)
+        if tag_refs is not None:
+            attachment_ref = et.SubElement(tag_refs, TAG_ATTACHMENT_REF)
+            attachment_ref.set(ATTR_REF, attachment.hash)
+            attachment_ref.set(ATTR_URL, attachment.url)