import xml.etree.ElementTree as et from boards.models.attachment.downloaders import download from boards.utils import get_file_mimetype, get_file_hash from django.db import transaction from boards.models import KeyPair, GlobalId, Signature, Post, Tag EXCEPTION_NODE = 'Sync node returned an error: {}' EXCEPTION_OP = 'Load the OP first' EXCEPTION_DOWNLOAD = 'File was not downloaded' EXCEPTION_HASH = 'File hash does not match attachment hash' EXCEPTION_SIGNATURE = 'Invalid model signature for {}' ENCODING_UNICODE = 'unicode' TAG_MODEL = 'model' TAG_REQUEST = 'request' TAG_RESPONSE = 'response' TAG_ID = 'id' TAG_STATUS = 'status' TAG_MODELS = 'models' TAG_TITLE = 'title' TAG_TEXT = 'text' TAG_THREAD = 'thread' TAG_PUB_TIME = 'pub-time' TAG_SIGNATURES = 'signatures' TAG_SIGNATURE = 'signature' TAG_CONTENT = 'content' TAG_ATTACHMENTS = 'attachments' TAG_ATTACHMENT = 'attachment' TAG_TAGS = 'tags' TAG_TAG = 'tag' TAG_ATTACHMENT_REFS = 'attachment-refs' TAG_ATTACHMENT_REF = 'attachment-ref' TAG_TRIPCODE = 'tripcode' TAG_VERSION = 'version' TYPE_GET = 'get' ATTR_VERSION = 'version' ATTR_TYPE = 'type' ATTR_NAME = 'name' ATTR_VALUE = 'value' ATTR_MIMETYPE = 'mimetype' ATTR_KEY = 'key' ATTR_REF = 'ref' ATTR_URL = 'url' ATTR_ID_TYPE = 'id-type' ID_TYPE_MD5 = 'md5' STATUS_SUCCESS = 'success' class SyncException(Exception): pass class SyncManager: @staticmethod def generate_response_get(model_list: list): response = et.Element(TAG_RESPONSE) status = et.SubElement(response, TAG_STATUS) status.text = STATUS_SUCCESS models = et.SubElement(response, TAG_MODELS) for post in model_list: model = et.SubElement(models, TAG_MODEL) model.set(ATTR_NAME, 'post') global_id = post.global_id images = post.images.all() attachments = post.attachments.all() if global_id.content: model.append(et.fromstring(global_id.content)) if len(images) > 0 or len(attachments) > 0: attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS) for image in images: SyncManager._attachment_to_xml( None, attachment_refs, image.image.file, image.hash, image.image.url) for file in attachments: SyncManager._attachment_to_xml( None, attachment_refs, file.file.file, file.hash, file.file.url) else: content_tag = et.SubElement(model, TAG_CONTENT) tag_id = et.SubElement(content_tag, TAG_ID) global_id.to_xml_element(tag_id) title = et.SubElement(content_tag, TAG_TITLE) title.text = post.title text = et.SubElement(content_tag, TAG_TEXT) text.text = post.get_sync_text() thread = post.get_thread() if post.is_opening(): tag_tags = et.SubElement(content_tag, TAG_TAGS) for tag in thread.get_tags(): tag_tag = et.SubElement(tag_tags, TAG_TAG) tag_tag.text = tag.name else: tag_thread = et.SubElement(content_tag, TAG_THREAD) thread_id = et.SubElement(tag_thread, TAG_ID) thread.get_opening_post().global_id.to_xml_element(thread_id) pub_time = et.SubElement(content_tag, TAG_PUB_TIME) pub_time.text = str(post.get_pub_time_str()) if post.tripcode: tripcode = et.SubElement(content_tag, TAG_TRIPCODE) tripcode.text = post.tripcode if len(images) > 0 or len(attachments) > 0: attachments_tag = et.SubElement(content_tag, TAG_ATTACHMENTS) attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS) for image in images: SyncManager._attachment_to_xml( attachments_tag, attachment_refs, image.image.file, image.hash, image.image.url) for file in attachments: SyncManager._attachment_to_xml( attachments_tag, attachment_refs, file.file.file, file.hash, file.file.url) version_tag = et.SubElement(content_tag, TAG_VERSION) version_tag.text = str(post.version) global_id.content = et.tostring(content_tag, ENCODING_UNICODE) global_id.save() signatures_tag = et.SubElement(model, TAG_SIGNATURES) post_signatures = global_id.signature_set.all() if post_signatures: signatures = post_signatures else: key = KeyPair.objects.get(public_key=global_id.key) signature = Signature( key_type=key.key_type, key=key.public_key, signature=key.sign(global_id.content), global_id=global_id, ) signature.save() signatures = [signature] for signature in signatures: signature_tag = et.SubElement(signatures_tag, TAG_SIGNATURE) signature_tag.set(ATTR_TYPE, signature.key_type) signature_tag.set(ATTR_VALUE, signature.signature) signature_tag.set(ATTR_KEY, signature.key) return et.tostring(response, ENCODING_UNICODE) @staticmethod @transaction.atomic def parse_response_get(response_xml, hostname): tag_root = et.fromstring(response_xml) tag_status = tag_root.find(TAG_STATUS) if STATUS_SUCCESS == tag_status.text: tag_models = tag_root.find(TAG_MODELS) for tag_model in tag_models: tag_content = tag_model.find(TAG_CONTENT) content_str = et.tostring(tag_content, ENCODING_UNICODE) signatures = SyncManager._verify_model(content_str, tag_model) tag_id = tag_content.find(TAG_ID) global_id, exists = GlobalId.from_xml_element(tag_id) if exists: print('Post with same ID already exists') else: global_id.content = content_str global_id.save() for signature in signatures: signature.global_id = global_id signature.save() title = tag_content.find(TAG_TITLE).text or '' text = tag_content.find(TAG_TEXT).text or '' pub_time = tag_content.find(TAG_PUB_TIME).text tripcode_tag = tag_content.find(TAG_TRIPCODE) if tripcode_tag is not None: tripcode = tripcode_tag.text or '' else: tripcode = '' thread = tag_content.find(TAG_THREAD) tags = [] if thread: thread_id = thread.find(TAG_ID) op_global_id, exists = GlobalId.from_xml_element(thread_id) if exists: opening_post = Post.objects.get(global_id=op_global_id) else: raise SyncException(EXCEPTION_OP) else: opening_post = None tag_tags = tag_content.find(TAG_TAGS) for tag_tag in tag_tags: tag, created = Tag.objects.get_or_create( name=tag_tag.text) tags.append(tag) # TODO Check that the replied posts are already present # before adding new ones files = [] tag_attachments = tag_content.find(TAG_ATTACHMENTS) or list() tag_refs = tag_model.find(TAG_ATTACHMENT_REFS) for attachment in tag_attachments: tag_ref = tag_refs.find("{}[@ref='{}']".format( TAG_ATTACHMENT_REF, attachment.text)) url = tag_ref.get(ATTR_URL) attached_file = download(hostname + url) if attached_file is None: raise SyncException(EXCEPTION_DOWNLOAD) hash = get_file_hash(attached_file) if hash != attachment.text: raise SyncException(EXCEPTION_HASH) files.append(attached_file) Post.objects.import_post( title=title, text=text, pub_time=pub_time, opening_post=opening_post, tags=tags, global_id=global_id, files=files, tripcode=tripcode) print('Parsed post {}'.format(global_id)) else: raise SyncException(EXCEPTION_NODE.format(tag_status.text)) @staticmethod def generate_response_list(): response = et.Element(TAG_RESPONSE) status = et.SubElement(response, TAG_STATUS) status.text = STATUS_SUCCESS models = et.SubElement(response, TAG_MODELS) for post in Post.objects.prefetch_related('global_id').all(): tag_model = et.SubElement(models, TAG_MODEL) tag_id = et.SubElement(tag_model, TAG_ID) post.global_id.to_xml_element(tag_id) tag_version = et.SubElement(tag_model, TAG_VERSION) tag_version.text = str(post.version) return et.tostring(response, ENCODING_UNICODE) @staticmethod def _verify_model(content_str, tag_model): """ Verifies all signatures for a single model. """ signatures = [] tag_signatures = tag_model.find(TAG_SIGNATURES) for tag_signature in tag_signatures: signature_type = tag_signature.get(ATTR_TYPE) signature_value = tag_signature.get(ATTR_VALUE) signature_key = tag_signature.get(ATTR_KEY) signature = Signature(key_type=signature_type, key=signature_key, signature=signature_value) if not KeyPair.objects.verify(signature, content_str): raise SyncException(EXCEPTION_SIGNATURE.format(content_str)) signatures.append(signature) return signatures @staticmethod def _attachment_to_xml(tag_attachments, tag_refs, file, hash, url): if tag_attachments is not None: mimetype = get_file_mimetype(file) attachment = et.SubElement(tag_attachments, TAG_ATTACHMENT) attachment.set(ATTR_MIMETYPE, mimetype) attachment.set(ATTR_ID_TYPE, ID_TYPE_MD5) attachment.text = hash attachment_ref = et.SubElement(tag_refs, TAG_ATTACHMENT_REF) attachment_ref.set(ATTR_REF, hash) attachment_ref.set(ATTR_URL, url)