##// END OF EJS Templates
Download attached filed to the post during sync
Download attached filed to the post during sync

File last commit:

r1511:ea51d39c decentral
r1511:ea51d39c decentral
Show More
sync.py
244 lines | 8.8 KiB | text/x-python | PythonLexer
neko259
Updated sync method for requesting and getting a post
r1177 import xml.etree.ElementTree as et
neko259
Added images and attachments to the XML post response
r1506
neko259
Download attached filed to the post during sync
r1511 from boards.models.attachment.downloaders import download
neko259
Added images and attachments to the XML post response
r1506 from boards.utils import get_file_mimetype
neko259
Sync-import of a single post is working
r1229 from django.db import transaction
neko259
Output and parse tags in OP
r1239 from boards.models import KeyPair, GlobalId, Signature, Post, Tag
neko259
Updated sync method for requesting and getting a post
r1177
ENCODING_UNICODE = 'unicode'
TAG_MODEL = 'model'
TAG_REQUEST = 'request'
TAG_RESPONSE = 'response'
TAG_ID = 'id'
TAG_STATUS = 'status'
TAG_MODELS = 'models'
TAG_TITLE = 'title'
TAG_TEXT = 'text'
TAG_THREAD = 'thread'
TAG_PUB_TIME = 'pub-time'
TAG_SIGNATURES = 'signatures'
TAG_SIGNATURE = 'signature'
TAG_CONTENT = 'content'
TAG_ATTACHMENTS = 'attachments'
TAG_ATTACHMENT = 'attachment'
neko259
Output and parse tags in OP
r1239 TAG_TAGS = 'tags'
TAG_TAG = 'tag'
neko259
Added local download links for attachments
r1508 TAG_ATTACHMENT_REFS = 'attachment-refs'
TAG_ATTACHMENT_REF = 'attachment-ref'
neko259
Updated sync method for requesting and getting a post
r1177
TYPE_GET = 'get'
ATTR_VERSION = 'version'
ATTR_TYPE = 'type'
ATTR_NAME = 'name'
ATTR_VALUE = 'value'
ATTR_MIMETYPE = 'mimetype'
neko259
Added signature verification for a post
r1237 ATTR_KEY = 'key'
neko259
Added local download links for attachments
r1508 ATTR_REF = 'ref'
ATTR_URL = 'url'
neko259
Updated sync method for requesting and getting a post
r1177
STATUS_SUCCESS = 'success'
neko259
Download attached filed to the post during sync
r1511 class SyncException(Exception):
pass
neko259
Updated sync method for requesting and getting a post
r1177 class SyncManager:
neko259
Made SyncManager's methods static
r1236 @staticmethod
def generate_response_get(model_list: list):
neko259
Updated sync method for requesting and getting a post
r1177 response = et.Element(TAG_RESPONSE)
status = et.SubElement(response, TAG_STATUS)
status.text = STATUS_SUCCESS
models = et.SubElement(response, TAG_MODELS)
for post in model_list:
model = et.SubElement(models, TAG_MODEL)
model.set(ATTR_NAME, 'post')
content_tag = et.SubElement(model, TAG_CONTENT)
tag_id = et.SubElement(content_tag, TAG_ID)
post.global_id.to_xml_element(tag_id)
title = et.SubElement(content_tag, TAG_TITLE)
title.text = post.title
text = et.SubElement(content_tag, TAG_TEXT)
neko259
Convert local post IDs to global when generating responses
r1228 text.text = post.get_sync_text()
neko259
Updated sync method for requesting and getting a post
r1177
neko259
Output and parse tags in OP
r1239 thread = post.get_thread()
if post.is_opening():
tag_tags = et.SubElement(content_tag, TAG_TAGS)
for tag in thread.get_tags():
tag_tag = et.SubElement(tag_tags, TAG_TAG)
tag_tag.text = tag.name
neko259
Updated sync method for requesting and getting a post
r1177 else:
neko259
Output and parse tags in OP
r1239 tag_thread = et.SubElement(content_tag, TAG_THREAD)
thread_id = et.SubElement(tag_thread, TAG_ID)
thread.get_opening_post().global_id.to_xml_element(thread_id)
neko259
Updated sync method for requesting and getting a post
r1177
pub_time = et.SubElement(content_tag, TAG_PUB_TIME)
neko259
Sync-import of a single post is working
r1229 pub_time.text = str(post.get_pub_time_str())
neko259
Updated sync method for requesting and getting a post
r1177
neko259
Added images and attachments to the XML post response
r1506 images = post.images.all()
attachments = post.attachments.all()
if len(images) > 0 or len(attachments) > 0:
attachments_tag = et.SubElement(content_tag, TAG_ATTACHMENTS)
neko259
Added local download links for attachments
r1508 attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS)
neko259
Added images and attachments to the XML post response
r1506 for image in images:
neko259
Added local download links for attachments
r1508 SyncManager._attachment_to_xml(
attachments_tag, attachment_refs, image.image.file,
image.hash, image.image.url)
neko259
Added images and attachments to the XML post response
r1506 for file in attachments:
neko259
Added local download links for attachments
r1508 SyncManager._attachment_to_xml(
attachments_tag, attachment_refs, file.file.file,
file.hash, file.file.url)
neko259
Added images and attachments to the XML post response
r1506
neko259
Updated sync method for requesting and getting a post
r1177 signatures_tag = et.SubElement(model, TAG_SIGNATURES)
neko259
Moved signature set to global id, not post
r1242 post_signatures = post.global_id.signature_set.all()
neko259
Updated sync method for requesting and getting a post
r1177 if post_signatures:
neko259
Added some minor info
r1238 signatures = post_signatures
neko259
Updated sync method for requesting and getting a post
r1177 else:
key = KeyPair.objects.get(public_key=post.global_id.key)
neko259
Sync fixes
r1386 signature = Signature(
neko259
Updated sync method for requesting and getting a post
r1177 key_type=key.key_type,
key=key.public_key,
neko259
Convert \r\n and \r to \n in the post text used in sync
r1504 signature=key.sign(et.tostring(content_tag, encoding=ENCODING_UNICODE)),
neko259
Sync fixes
r1386 global_id=post.global_id,
)
signature.save()
signatures = [signature]
neko259
Updated sync method for requesting and getting a post
r1177 for signature in signatures:
signature_tag = et.SubElement(signatures_tag, TAG_SIGNATURE)
signature_tag.set(ATTR_TYPE, signature.key_type)
signature_tag.set(ATTR_VALUE, signature.signature)
neko259
Added signature verification for a post
r1237 signature_tag.set(ATTR_KEY, signature.key)
neko259
Updated sync method for requesting and getting a post
r1177
return et.tostring(response, ENCODING_UNICODE)
neko259
Made SyncManager's methods static
r1236 @staticmethod
neko259
Sync-import of a single post is working
r1229 @transaction.atomic
neko259
Download attached filed to the post during sync
r1511 def parse_response_get(response_xml, hostname):
neko259
Updated sync method for requesting and getting a post
r1177 tag_root = et.fromstring(response_xml)
tag_status = tag_root.find(TAG_STATUS)
if STATUS_SUCCESS == tag_status.text:
tag_models = tag_root.find(TAG_MODELS)
for tag_model in tag_models:
tag_content = tag_model.find(TAG_CONTENT)
neko259
Added signature verification for a post
r1237
neko259
Save signatures when the post is parsed for the later use
r1244 signatures = SyncManager._verify_model(tag_content, tag_model)
neko259
Added signature verification for a post
r1237
neko259
Updated sync method for requesting and getting a post
r1177 tag_id = tag_content.find(TAG_ID)
neko259
Refactored code for getting existing and new global ids
r1233 global_id, exists = GlobalId.from_xml_element(tag_id)
neko259
Don't allow to import the same post twice
r1232
neko259
Refactored code for getting existing and new global ids
r1233 if exists:
print('Post with same ID already exists')
else:
global_id.save()
neko259
Save signatures when the post is parsed for the later use
r1244 for signature in signatures:
signature.global_id = global_id
signature.save()
neko259
Updated sync method for requesting and getting a post
r1177
neko259
Download attached filed to the post during sync
r1511 title = tag_content.find(TAG_TITLE).text or ''
text = tag_content.find(TAG_TEXT).text or ''
neko259
Refactored code for getting existing and new global ids
r1233 pub_time = tag_content.find(TAG_PUB_TIME).text
neko259
Don't allow to import the same post twice
r1232
neko259
Refactored code for getting existing and new global ids
r1233 thread = tag_content.find(TAG_THREAD)
neko259
Output and parse tags in OP
r1239 tags = []
neko259
Refactored code for getting existing and new global ids
r1233 if thread:
neko259
Sync fixes
r1386 thread_id = thread.find(TAG_ID)
op_global_id, exists = GlobalId.from_xml_element(thread_id)
if exists:
opening_post = Post.objects.get(global_id=op_global_id)
else:
neko259
Download attached filed to the post during sync
r1511 raise SyncException('Load the OP first')
neko259
Refactored code for getting existing and new global ids
r1233 else:
opening_post = None
neko259
Output and parse tags in OP
r1239 tag_tags = tag_content.find(TAG_TAGS)
for tag_tag in tag_tags:
neko259
Sync fixes
r1386 tag, created = Tag.objects.get_or_create(
name=tag_tag.text)
neko259
Output and parse tags in OP
r1239 tags.append(tag)
neko259
Updated sync method for requesting and getting a post
r1177
neko259
Refactored code for getting existing and new global ids
r1233 # TODO Check that the replied posts are already present
# before adding new ones
neko259
Sync-import of a single post is working
r1229
neko259
Download attached filed to the post during sync
r1511 files = []
tag_attachments = tag_content.find(TAG_ATTACHMENTS) or list()
tag_refs = tag_model.find(TAG_ATTACHMENT_REFS)
for attachment in tag_attachments:
tag_ref = tag_refs.find("{}[@ref='{}']".format(
TAG_ATTACHMENT_REF, attachment.text))
url = tag_ref.get(ATTR_URL)
attached_file = download(hostname + url)
if attached_file is None:
raise SyncException('File was not dowloaded')
files.append(attached_file)
# TODO Check hash
neko259
Added some minor info
r1238
neko259
Download attached filed to the post during sync
r1511 Post.objects.import_post(
neko259
Refactored code for getting existing and new global ids
r1233 title=title, text=text, pub_time=pub_time,
neko259
Save signatures when the post is parsed for the later use
r1244 opening_post=opening_post, tags=tags,
neko259
Download attached filed to the post during sync
r1511 global_id=global_id, files=files)
neko259
Updated sync method for requesting and getting a post
r1177 else:
# TODO Throw an exception?
pass
neko259
Added signature verification for a post
r1237
@staticmethod
neko259
Added PULL protocol method implementation without any filters
r1321 def generate_response_pull():
response = et.Element(TAG_RESPONSE)
status = et.SubElement(response, TAG_STATUS)
status.text = STATUS_SUCCESS
models = et.SubElement(response, TAG_MODELS)
for post in Post.objects.all():
tag_id = et.SubElement(models, TAG_ID)
post.global_id.to_xml_element(tag_id)
return et.tostring(response, ENCODING_UNICODE)
@staticmethod
neko259
Added some minor info
r1238 def _verify_model(tag_content, tag_model):
neko259
Added signature verification for a post
r1237 """
Verifies all signatures for a single model.
"""
neko259
Save signatures when the post is parsed for the later use
r1244 signatures = []
neko259
Added signature verification for a post
r1237
tag_signatures = tag_model.find(TAG_SIGNATURES)
for tag_signature in tag_signatures:
signature_type = tag_signature.get(ATTR_TYPE)
signature_value = tag_signature.get(ATTR_VALUE)
signature_key = tag_signature.get(ATTR_KEY)
neko259
Save signatures when the post is parsed for the later use
r1244 signature = Signature(key_type=signature_type,
key=signature_key,
signature=signature_value)
neko259
Sync fixes
r1386
content = et.tostring(tag_content, ENCODING_UNICODE)
neko259
Save signatures when the post is parsed for the later use
r1244
neko259
Added signature verification for a post
r1237 if not KeyPair.objects.verify(
neko259
Sync fixes
r1386 signature, content):
neko259
Download attached filed to the post during sync
r1511 raise SyncException('Invalid model signature for {}'.format(content))
neko259
Sync fixes
r1386
signatures.append(signature)
neko259
Added signature verification for a post
r1237
neko259
Save signatures when the post is parsed for the later use
r1244 return signatures
neko259
Added local download links for attachments
r1508
@staticmethod
def _attachment_to_xml(tag_attachments, tag_refs, file, hash, url):
mimetype = get_file_mimetype(file)
attachment = et.SubElement(tag_attachments, TAG_ATTACHMENT)
attachment.set(ATTR_MIMETYPE, mimetype)
attachment.text = hash
attachment_ref = et.SubElement(tag_refs, TAG_ATTACHMENT_REF)
attachment_ref.set(ATTR_REF, hash)
attachment_ref.set(ATTR_URL, url)