Delete global ID when deleting post. Cache model's content XML tag into global ID

boards/models/post/sync.py

@@ -1,251 +1,258 b''
 import xml.etree.ElementTree as et
 
 from boards.models.attachment.downloaders import download
 from boards.utils import get_file_mimetype, get_file_hash
 from django.db import transaction
 from boards.models import KeyPair, GlobalId, Signature, Post, Tag
 
 ENCODING_UNICODE = 'unicode'
 
 TAG_MODEL = 'model'
 TAG_REQUEST = 'request'
 TAG_RESPONSE = 'response'
 TAG_ID = 'id'
 TAG_STATUS = 'status'
 TAG_MODELS = 'models'
 TAG_TITLE = 'title'
 TAG_TEXT = 'text'
 TAG_THREAD = 'thread'
 TAG_PUB_TIME = 'pub-time'
 TAG_SIGNATURES = 'signatures'
 TAG_SIGNATURE = 'signature'
 TAG_CONTENT = 'content'
 TAG_ATTACHMENTS = 'attachments'
 TAG_ATTACHMENT = 'attachment'
 TAG_TAGS = 'tags'
 TAG_TAG = 'tag'
 TAG_ATTACHMENT_REFS = 'attachment-refs'
 TAG_ATTACHMENT_REF = 'attachment-ref'
 
 TYPE_GET = 'get'
 
 ATTR_VERSION = 'version'
 ATTR_TYPE = 'type'
 ATTR_NAME = 'name'
 ATTR_VALUE = 'value'
 ATTR_MIMETYPE = 'mimetype'
 ATTR_KEY = 'key'
 ATTR_REF = 'ref'
 ATTR_URL = 'url'
 
 STATUS_SUCCESS = 'success'
 
 
 class SyncException(Exception):
     pass
 
 
 class SyncManager:
     @staticmethod
     def generate_response_get(model_list: list):
         response = et.Element(TAG_RESPONSE)
 
         status = et.SubElement(response, TAG_STATUS)
         status.text = STATUS_SUCCESS
 
         models = et.SubElement(response, TAG_MODELS)
 
-        # TODO Put global id's content into XML instad of manual serialization
         for post in model_list:
             model = et.SubElement(models, TAG_MODEL)
             model.set(ATTR_NAME, 'post')
 
+            global_id = post.global_id
+
+            if global_id.content:
+                model.append(et.fromstring(global_id.content))
+            else:
             content_tag = et.SubElement(model, TAG_CONTENT)
 
             tag_id = et.SubElement(content_tag, TAG_ID)
-            post.global_id.to_xml_element(tag_id)
+                global_id.to_xml_element(tag_id)
 
             title = et.SubElement(content_tag, TAG_TITLE)
             title.text = post.title
 
             text = et.SubElement(content_tag, TAG_TEXT)
             text.text = post.get_sync_text()
 
             thread = post.get_thread()
             if post.is_opening():
                 tag_tags = et.SubElement(content_tag, TAG_TAGS)
                 for tag in thread.get_tags():
                     tag_tag = et.SubElement(tag_tags, TAG_TAG)
                     tag_tag.text = tag.name
             else:
                 tag_thread = et.SubElement(content_tag, TAG_THREAD)
                 thread_id = et.SubElement(tag_thread, TAG_ID)
                 thread.get_opening_post().global_id.to_xml_element(thread_id)
 
             pub_time = et.SubElement(content_tag, TAG_PUB_TIME)
             pub_time.text = str(post.get_pub_time_str())
 
             images = post.images.all()
             attachments = post.attachments.all()
             if len(images) > 0 or len(attachments) > 0:
                 attachments_tag = et.SubElement(content_tag, TAG_ATTACHMENTS)
                 attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS)
 
                 for image in images:
                     SyncManager._attachment_to_xml(
                         attachments_tag, attachment_refs, image.image.file,
                         image.hash, image.image.url)
                 for file in attachments:
                     SyncManager._attachment_to_xml(
                         attachments_tag, attachment_refs, file.file.file,
                         file.hash, file.file.url)
 
+                global_id.content = et.tostring(content_tag, ENCODING_UNICODE)
+                global_id.save()
+
             signatures_tag = et.SubElement(model, TAG_SIGNATURES)
-            post_signatures = post.global_id.signature_set.all()
+            post_signatures = global_id.signature_set.all()
             if post_signatures:
                 signatures = post_signatures
             else:
-                key = KeyPair.objects.get(public_key=post.global_id.key)
+                key = KeyPair.objects.get(public_key=global_id.key)
                 signature = Signature(
                     key_type=key.key_type,
                     key=key.public_key,
-                    signature=key.sign(et.tostring(content_tag, encoding=ENCODING_UNICODE)),
-                    global_id=post.global_id,
+                    signature=key.sign(global_id.content),
+                    global_id=global_id,
                 )
                 signature.save()
                 signatures = [signature]
             for signature in signatures:
                 signature_tag = et.SubElement(signatures_tag, TAG_SIGNATURE)
                 signature_tag.set(ATTR_TYPE, signature.key_type)
                 signature_tag.set(ATTR_VALUE, signature.signature)
                 signature_tag.set(ATTR_KEY, signature.key)
 
         return et.tostring(response, ENCODING_UNICODE)
 
     @staticmethod
     @transaction.atomic
     def parse_response_get(response_xml, hostname):
         tag_root = et.fromstring(response_xml)
         tag_status = tag_root.find(TAG_STATUS)
         if STATUS_SUCCESS == tag_status.text:
             tag_models = tag_root.find(TAG_MODELS)
             for tag_model in tag_models:
                 tag_content = tag_model.find(TAG_CONTENT)
 
                 signatures = SyncManager._verify_model(tag_content, tag_model)
 
                 tag_id = tag_content.find(TAG_ID)
                 global_id, exists = GlobalId.from_xml_element(tag_id)
 
                 if exists:
                     print('Post with same ID already exists')
                 else:
-                    global_id.content = et.to_string(tag_content,
+                    global_id.content = et.tostring(tag_content,
                             ENCODING_UNICODE)
                     global_id.save()
                     for signature in signatures:
                         signature.global_id = global_id
                         signature.save()
 
                     title = tag_content.find(TAG_TITLE).text or ''
                     text = tag_content.find(TAG_TEXT).text or ''
                     pub_time = tag_content.find(TAG_PUB_TIME).text
 
                     thread = tag_content.find(TAG_THREAD)
                     tags = []
                     if thread:
                         thread_id = thread.find(TAG_ID)
                         op_global_id, exists = GlobalId.from_xml_element(thread_id)
                         if exists:
                             opening_post = Post.objects.get(global_id=op_global_id)
                         else:
                             raise SyncException('Load the OP first')
                     else:
                         opening_post = None
                         tag_tags = tag_content.find(TAG_TAGS)
                         for tag_tag in tag_tags:
                             tag, created = Tag.objects.get_or_create(
                                 name=tag_tag.text)
                             tags.append(tag)
 
                     # TODO Check that the replied posts are already present
                     # before adding new ones
 
                     files = []
                     tag_attachments = tag_content.find(TAG_ATTACHMENTS) or list()
                     tag_refs = tag_model.find(TAG_ATTACHMENT_REFS)
                     for attachment in tag_attachments:
                         tag_ref = tag_refs.find("{}[@ref='{}']".format(
                             TAG_ATTACHMENT_REF, attachment.text))
                         url = tag_ref.get(ATTR_URL)
                         attached_file = download(hostname + url)
                         if attached_file is None:
                             raise SyncException('File was not dowloaded')
 
                         hash = get_file_hash(file)
                         if hash != attachment.text:
                             raise SyncException('File hash does not match attachment hash')
 
                         files.append(attached_file)
 
                     Post.objects.import_post(
                         title=title, text=text, pub_time=pub_time,
                         opening_post=opening_post, tags=tags,
                         global_id=global_id, files=files)
         else:
             raise SyncException('Sync node returned an error: {}'.format(
                 tag_status.text))
 
     @staticmethod
     def generate_response_pull():
         response = et.Element(TAG_RESPONSE)
 
         status = et.SubElement(response, TAG_STATUS)
         status.text = STATUS_SUCCESS
 
         models = et.SubElement(response, TAG_MODELS)
 
         for post in Post.objects.all():
             tag_id = et.SubElement(models, TAG_ID)
             post.global_id.to_xml_element(tag_id)
 
         return et.tostring(response, ENCODING_UNICODE)
 
     @staticmethod
     def _verify_model(tag_content, tag_model):
         """
         Verifies all signatures for a single model.
         """
 
         signatures = []
 
         tag_signatures = tag_model.find(TAG_SIGNATURES)
         for tag_signature in tag_signatures:
             signature_type = tag_signature.get(ATTR_TYPE)
             signature_value = tag_signature.get(ATTR_VALUE)
             signature_key = tag_signature.get(ATTR_KEY)
 
             signature = Signature(key_type=signature_type,
                                   key=signature_key,
                                   signature=signature_value)
 
             content = et.tostring(tag_content, ENCODING_UNICODE)
 
             if not KeyPair.objects.verify(
                     signature, content):
                 raise SyncException('Invalid model signature for {}'.format(content))
 
             signatures.append(signature)
 
         return signatures
 
     @staticmethod
     def _attachment_to_xml(tag_attachments, tag_refs, file, hash, url):
         mimetype = get_file_mimetype(file)
         attachment = et.SubElement(tag_attachments, TAG_ATTACHMENT)
         attachment.set(ATTR_MIMETYPE, mimetype)
         attachment.text = hash
 
         attachment_ref = et.SubElement(tag_refs, TAG_ATTACHMENT_REF)
         attachment_ref.set(ATTR_REF, hash)
         attachment_ref.set(ATTR_URL, url)

boards/models/signature.py

@@ -1,138 +1,144 b''
 import xml.etree.ElementTree as et
 from django.db import models
 
 
 TAG_MODEL = 'model'
 TAG_REQUEST = 'request'
 TAG_ID = 'id'
 
 TYPE_GET = 'get'
 TYPE_PULL = 'pull'
 
 ATTR_VERSION = 'version'
 ATTR_TYPE = 'type'
 ATTR_NAME = 'name'
 
 ATTR_KEY = 'key'
 ATTR_KEY_TYPE = 'type'
 ATTR_LOCAL_ID = 'local-id'
 
 
 class GlobalIdManager(models.Manager):
     def generate_request_get(self, global_id_list: list):
         """
         Form a get request from a list of ModelId objects.
         """
 
         request = et.Element(TAG_REQUEST)
         request.set(ATTR_TYPE, TYPE_GET)
         request.set(ATTR_VERSION, '1.0')
 
         model = et.SubElement(request, TAG_MODEL)
         model.set(ATTR_VERSION, '1.0')
         model.set(ATTR_NAME, 'post')
 
         for global_id in global_id_list:
             tag_id = et.SubElement(model, TAG_ID)
             global_id.to_xml_element(tag_id)
 
         return et.tostring(request, 'unicode')
 
     def generate_request_pull(self):
         """
         Form a pull request from a list of ModelId objects.
         """
 
         request = et.Element(TAG_REQUEST)
         request.set(ATTR_TYPE, TYPE_PULL)
         request.set(ATTR_VERSION, '1.0')
 
         model = et.SubElement(request, TAG_MODEL)
         model.set(ATTR_VERSION, '1.0')
         model.set(ATTR_NAME, 'post')
 
         return et.tostring(request, 'unicode')
 
     def global_id_exists(self, global_id):
         """
         Checks if the same global id already exists in the system.
         """
 
         return self.filter(key=global_id.key,
                            key_type=global_id.key_type,
                            local_id=global_id.local_id).exists()
 
 
 class GlobalId(models.Model):
+    """
+    Global model ID and cache.
+    Key, key type and local ID make a single global identificator of the model.
+    Content is an XML cache of the model that can be passed along between nodes
+    without manual serialization each time.
+    """
     class Meta:
         app_label = 'boards'
 
     objects = GlobalIdManager()
 
     def __init__(self, *args, **kwargs):
         models.Model.__init__(self, *args, **kwargs)
 
         if 'key' in kwargs and 'key_type' in kwargs and 'local_id' in kwargs:
             self.key = kwargs['key']
             self.key_type = kwargs['key_type']
             self.local_id = kwargs['local_id']
 
     key = models.TextField()
     key_type = models.TextField()
     local_id = models.IntegerField()
     content = models.TextField(blank=True, null=True)
 
     def __str__(self):
         return '%s::%s::%d' % (self.key_type, self.key, self.local_id)
 
     def to_xml_element(self, element: et.Element):
         """
         Exports global id to an XML element.
         """
 
         element.set(ATTR_KEY, self.key)
         element.set(ATTR_KEY_TYPE, self.key_type)
         element.set(ATTR_LOCAL_ID, str(self.local_id))
 
     @staticmethod
     def from_xml_element(element: et.Element):
         """
         Parses XML id tag and gets global id from it.
 
         Arguments:
         element -- the XML 'id' element
 
         Returns:
         global_id -- id itself
         exists -- True if the global id was taken from database, False if it
         did not exist and was created.
         """
 
         try:
             return GlobalId.objects.get(key=element.get(ATTR_KEY),
                                         key_type=element.get(ATTR_KEY_TYPE),
                                         local_id=int(element.get(
                                             ATTR_LOCAL_ID))), True
         except GlobalId.DoesNotExist:
             return GlobalId(key=element.get(ATTR_KEY),
                             key_type=element.get(ATTR_KEY_TYPE),
                             local_id=int(element.get(ATTR_LOCAL_ID))), False
 
 
 class Signature(models.Model):
     class Meta:
         app_label = 'boards'
 
     def __init__(self, *args, **kwargs):
         models.Model.__init__(self, *args, **kwargs)
 
         if 'key' in kwargs and 'key_type' in kwargs and 'signature' in kwargs:
             self.key_type = kwargs['key_type']
             self.key = kwargs['key']
             self.signature = kwargs['signature']
 
     key_type = models.TextField()
     key = models.TextField()
     signature = models.TextField()
 
     global_id = models.ForeignKey('GlobalId')

boards/signals.py

@@ -1,81 +1,87 b''
 import re
 from boards.mdx_neboard import get_parser
 
 from boards.models import Post, GlobalId
 from boards.models.post import REGEX_NOTIFICATION
 from boards.models.post import REGEX_REPLY, REGEX_GLOBAL_REPLY
 from boards.models.user import Notification
 from django.db.models.signals import post_save, pre_save, pre_delete, \
     post_delete
 from django.dispatch import receiver
 from django.utils import timezone
 
 
 @receiver(post_save, sender=Post)
 def connect_replies(instance, **kwargs):
     for reply_number in re.finditer(REGEX_REPLY, instance.get_raw_text()):
         post_id = reply_number.group(1)
 
         try:
             referenced_post = Post.objects.get(id=post_id)
 
             referenced_post.referenced_posts.add(instance)
             referenced_post.last_edit_time = instance.pub_time
             referenced_post.build_refmap()
             referenced_post.save(update_fields=['refmap', 'last_edit_time'])
         except Post.ObjectDoesNotExist:
             pass
 
 
 @receiver(post_save, sender=Post)
 def connect_global_replies(instance, **kwargs):
     for reply_number in re.finditer(REGEX_GLOBAL_REPLY, instance.get_raw_text()):
         key_type = reply_number.group(1)
         key = reply_number.group(2)
         local_id = reply_number.group(3)
 
         try:
             global_id = GlobalId.objects.get(key_type=key_type, key=key,
                     local_id=local_id)
             referenced_post = Post.objects.get(global_id=global_id)
             referenced_post.referenced_posts.add(instance)
             referenced_post.last_edit_time = instance.pub_time
             referenced_post.build_refmap()
             referenced_post.save(update_fields=['refmap', 'last_edit_time'])
         except (GlobalId.ObjectDoesNotExist, Post.ObjectDoesNotExist):
             pass
 
 
 @receiver(post_save, sender=Post)
 def connect_notifications(instance, **kwargs):
     for reply_number in re.finditer(REGEX_NOTIFICATION, instance.get_raw_text()):
         user_name = reply_number.group(1).lower()
         Notification.objects.get_or_create(name=user_name, post=instance)
 
 
 @receiver(pre_save, sender=Post)
 def preparse_text(instance, **kwargs):
     instance._text_rendered = get_parser().parse(instance.get_raw_text())
 
 
 @receiver(pre_delete, sender=Post)
 def delete_images(instance, **kwargs):
     for image in instance.images.all():
         image_refs_count = image.post_images.count()
         if image_refs_count == 1:
             image.delete()
 
 
 @receiver(pre_delete, sender=Post)
 def delete_attachments(instance, **kwargs):
     for attachment in instance.attachments.all():
         attachment_refs_count = attachment.attachment_posts.count()
         if attachment_refs_count == 1:
             attachment.delete()
 
 
 @receiver(post_delete, sender=Post)
 def update_thread_on_delete(instance, **kwargs):
     thread = instance.get_thread()
     thread.last_edit_time = timezone.now()
     thread.save()
+
+
+@receiver(post_delete, sender=Post)
+def delete_global_id(instance, **kwargs):
+    if instance.global_id and instance.global_id.id:
+        instance.global_id.delete()