# HG changeset patch # User neko259 # Date 2015-05-17 12:45:19 # Node ID a55da940bcba38b30ebafcde47526a103e1ee91c # Parent 3d7a615a939d72f6c225c467d9126805223fb498 Updated sync method for requesting and getting a post diff --git a/boards/management/commands/sync_with_server.py b/boards/management/commands/sync_with_server.py --- a/boards/management/commands/sync_with_server.py +++ b/boards/management/commands/sync_with_server.py @@ -5,6 +5,7 @@ import xml.etree.ElementTree as ET from django.core.management import BaseCommand from boards.models import GlobalId +from boards.models.post.sync import SyncManager __author__ = 'neko259' @@ -18,7 +19,7 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument('url', type=str) - #parser.add_argument('global_id', type=str) # TODO Implement this + parser.add_argument('global_id', type=str) def handle(self, *args, **options): url = options.get('url') @@ -34,14 +35,11 @@ class Command(BaseCommand): local_id=local_id) xml = GlobalId.objects.generate_request_get([global_id]) - data = {'xml': xml} - body = urllib.parse.urlencode(data) + # body = urllib.parse.urlencode(data) h = httplib2.Http() - response, content = h.request(url, method="POST", body=body) + response, content = h.request(url, method="POST", body=xml) - # TODO Parse content and get the model list - - print(content) + SyncManager().parse_response_get(content) else: raise Exception('Invalid global ID') else: diff --git a/boards/models/post/__init__.py b/boards/models/post/__init__.py --- a/boards/models/post/__init__.py +++ b/boards/models/post/__init__.py @@ -3,7 +3,6 @@ from datetime import time as dtime import logging import re import uuid -import xml.etree.ElementTree as et from django.core.exceptions import ObjectDoesNotExist from django.core.urlresolvers import reverse @@ -13,7 +12,7 @@ from django.template.loader import rende from django.utils import timezone from boards.mdx_neboard import Parser -from boards.models import KeyPair, GlobalId, Signature +from boards.models import KeyPair, GlobalId from boards import settings from boards.models import PostImage from boards.models.base import Viewable @@ -22,9 +21,6 @@ from boards.models.post.export import ge from boards.models.user import Notification, Ban import boards.models.thread - -ENCODING_UNICODE = 'unicode' - WS_NOTIFICATION_TYPE_NEW_POST = 'new_post' WS_NOTIFICATION_TYPE = 'notification_type' @@ -48,31 +44,6 @@ REGEX_GLOBAL_REPLY = re.compile(r'\[post REGEX_URL = re.compile(r'https?\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(/\S*)?') REGEX_NOTIFICATION = re.compile(r'\[user\](\w+)\[/user\]') -TAG_MODEL = 'model' -TAG_REQUEST = 'request' -TAG_RESPONSE = 'response' -TAG_ID = 'id' -TAG_STATUS = 'status' -TAG_MODELS = 'models' -TAG_TITLE = 'title' -TAG_TEXT = 'text' -TAG_THREAD = 'thread' -TAG_PUB_TIME = 'pub-time' -TAG_SIGNATURES = 'signatures' -TAG_SIGNATURE = 'signature' -TAG_CONTENT = 'content' -TAG_ATTACHMENTS = 'attachments' -TAG_ATTACHMENT = 'attachment' - -TYPE_GET = 'get' - -ATTR_VERSION = 'version' -ATTR_TYPE = 'type' -ATTR_NAME = 'name' -ATTR_VALUE = 'value' -ATTR_MIMETYPE = 'mimetype' - -STATUS_SUCCESS = 'success' PARAMETER_TRUNCATED = 'truncated' PARAMETER_TAG = 'tag' @@ -89,7 +60,6 @@ PARAMETER_REPLY_LINK = 'reply_link' PARAMETER_NEED_OP_DATA = 'need_op_data' DIFF_TYPE_HTML = 'html' -DIFF_TYPE_JSON = 'json' REFMAP_STR = '>>{}' @@ -189,98 +159,6 @@ class PostManager(models.Manager): return ppd - # TODO Make a separate sync facade? - def generate_response_get(self, model_list: list): - response = et.Element(TAG_RESPONSE) - - status = et.SubElement(response, TAG_STATUS) - status.text = STATUS_SUCCESS - - models = et.SubElement(response, TAG_MODELS) - - for post in model_list: - model = et.SubElement(models, TAG_MODEL) - model.set(ATTR_NAME, 'post') - - content_tag = et.SubElement(model, TAG_CONTENT) - - tag_id = et.SubElement(content_tag, TAG_ID) - post.global_id.to_xml_element(tag_id) - - title = et.SubElement(content_tag, TAG_TITLE) - title.text = post.title - - text = et.SubElement(content_tag, TAG_TEXT) - # TODO Replace local links by global ones in the text - text.text = post.get_raw_text() - - if not post.is_opening(): - thread = et.SubElement(content_tag, TAG_THREAD) - thread_id = et.SubElement(thread, TAG_ID) - post.get_thread().get_opening_post().global_id.to_xml_element(thread_id) - else: - # TODO Output tags here - pass - - pub_time = et.SubElement(content_tag, TAG_PUB_TIME) - pub_time.text = str(post.get_pub_time_epoch()) - - signatures_tag = et.SubElement(model, TAG_SIGNATURES) - post_signatures = post.signature.all() - if post_signatures: - signatures = post.signatures - else: - # TODO Maybe the signature can be computed only once after - # the post is added? Need to add some on_save signal queue - # and add this there. - key = KeyPair.objects.get(public_key=post.global_id.key) - signatures = [Signature( - key_type=key.key_type, - key=key.public_key, - signature=key.sign(et.tostring(model, ENCODING_UNICODE)), - )] - for signature in signatures: - signature_tag = et.SubElement(signatures_tag, TAG_SIGNATURE) - signature_tag.set(ATTR_TYPE, signature.key_type) - signature_tag.set(ATTR_VALUE, signature.signature) - - return et.tostring(response, ENCODING_UNICODE) - - def parse_response_get(self, response_xml): - tag_root = et.fromstring(response_xml) - tag_status = tag_root[0] - if 'success' == tag_status.text: - tag_models = tag_root[1] - for tag_model in tag_models: - tag_content = tag_model[0] - tag_id = tag_content[1] - try: - GlobalId.from_xml_element(tag_id, existing=True) - # If this post already exists, just continue - # TODO Compare post content and update the post if necessary - pass - except GlobalId.DoesNotExist: - global_id = GlobalId.from_xml_element(tag_id) - - title = tag_content.find(TAG_TITLE).text - text = tag_content.find(TAG_TEXT).text - # TODO Check that the replied posts are already present - # before adding new ones - - # TODO Pub time, thread, tags - - post = Post.objects.create(title=title, text=text) - else: - # TODO Throw an exception? - pass - - # TODO Make a separate parser module and move preparser there - def _preparse_text(self, text: str) -> str: - """ - Preparses text to change patterns like '>>' to a proper bbcode - tags. - """ - class Post(models.Model, Viewable): """A post is a message.""" @@ -433,7 +311,6 @@ class Post(models.Model, Viewable): logging.getLogger('boards.post.delete').info( 'Deleted post {}'.format(self)) - # TODO Implement this with OOP, e.g. use the factory and HtmlPostData class def set_global_id(self, key_pair=None): """ Sets global id based on the given key pair. If no key pair is given, @@ -483,7 +360,6 @@ class Post(models.Model, Viewable): pass return local_replied + global_replied - def get_post_data(self, format_type=DIFF_TYPE_JSON, request=None, include_last_update=False) -> str: """ diff --git a/boards/models/post/sync.py b/boards/models/post/sync.py new file mode 100644 --- /dev/null +++ b/boards/models/post/sync.py @@ -0,0 +1,116 @@ +import xml.etree.ElementTree as et +from boards.models import KeyPair, GlobalId, Signature, Post + +ENCODING_UNICODE = 'unicode' + +TAG_MODEL = 'model' +TAG_REQUEST = 'request' +TAG_RESPONSE = 'response' +TAG_ID = 'id' +TAG_STATUS = 'status' +TAG_MODELS = 'models' +TAG_TITLE = 'title' +TAG_TEXT = 'text' +TAG_THREAD = 'thread' +TAG_PUB_TIME = 'pub-time' +TAG_SIGNATURES = 'signatures' +TAG_SIGNATURE = 'signature' +TAG_CONTENT = 'content' +TAG_ATTACHMENTS = 'attachments' +TAG_ATTACHMENT = 'attachment' + +TYPE_GET = 'get' + +ATTR_VERSION = 'version' +ATTR_TYPE = 'type' +ATTR_NAME = 'name' +ATTR_VALUE = 'value' +ATTR_MIMETYPE = 'mimetype' + +STATUS_SUCCESS = 'success' + + +class SyncManager: + def generate_response_get(self, model_list: list): + response = et.Element(TAG_RESPONSE) + + status = et.SubElement(response, TAG_STATUS) + status.text = STATUS_SUCCESS + + models = et.SubElement(response, TAG_MODELS) + + for post in model_list: + model = et.SubElement(models, TAG_MODEL) + model.set(ATTR_NAME, 'post') + + content_tag = et.SubElement(model, TAG_CONTENT) + + tag_id = et.SubElement(content_tag, TAG_ID) + post.global_id.to_xml_element(tag_id) + + title = et.SubElement(content_tag, TAG_TITLE) + title.text = post.title + + text = et.SubElement(content_tag, TAG_TEXT) + # TODO Replace local links by global ones in the text + text.text = post.get_raw_text() + + if not post.is_opening(): + thread = et.SubElement(content_tag, TAG_THREAD) + thread_id = et.SubElement(thread, TAG_ID) + post.get_thread().get_opening_post().global_id.to_xml_element(thread_id) + else: + # TODO Output tags here + pass + + pub_time = et.SubElement(content_tag, TAG_PUB_TIME) + pub_time.text = str(post.get_pub_time_epoch()) + + signatures_tag = et.SubElement(model, TAG_SIGNATURES) + post_signatures = post.signature.all() + if post_signatures: + signatures = post.signatures + else: + # TODO Maybe the signature can be computed only once after + # the post is added? Need to add some on_save signal queue + # and add this there. + key = KeyPair.objects.get(public_key=post.global_id.key) + signatures = [Signature( + key_type=key.key_type, + key=key.public_key, + signature=key.sign(et.tostring(model, ENCODING_UNICODE)), + )] + for signature in signatures: + signature_tag = et.SubElement(signatures_tag, TAG_SIGNATURE) + signature_tag.set(ATTR_TYPE, signature.key_type) + signature_tag.set(ATTR_VALUE, signature.signature) + + return et.tostring(response, ENCODING_UNICODE) + + def parse_response_get(self, response_xml): + tag_root = et.fromstring(response_xml) + tag_status = tag_root.find(TAG_STATUS) + if STATUS_SUCCESS == tag_status.text: + tag_models = tag_root.find(TAG_MODELS) + for tag_model in tag_models: + tag_content = tag_model.find(TAG_CONTENT) + tag_id = tag_content.find(TAG_ID) + try: + GlobalId.from_xml_element(tag_id, existing=True) + print('Post with same ID already exists') + except GlobalId.DoesNotExist: + global_id = GlobalId.from_xml_element(tag_id) + + title = tag_content.find(TAG_TITLE).text + text = tag_content.find(TAG_TEXT).text + # TODO Check that the replied posts are already present + # before adding new ones + + # TODO Pub time, thread, tags + + print(title) + print(text) + # post = Post.objects.create(title=title, text=text) + else: + # TODO Throw an exception? + pass diff --git a/boards/urls.py b/boards/urls.py --- a/boards/urls.py +++ b/boards/urls.py @@ -10,7 +10,7 @@ from boards.views.notifications import N from boards.views.search import BoardSearchView from boards.views.static import StaticPageView from boards.views.preview import PostPreviewView -from boards.views.sync import get_post_sync_data +from boards.views.sync import get_post_sync_data, response_get js_info_dict = { @@ -76,6 +76,7 @@ urlpatterns = patterns('', # Sync protocol API url(r'^api/sync/pull/$', api.sync_pull, name='api_sync_pull'), + url(r'^api/sync/get/$', response_get, name='api_sync_pull'), # TODO 'get' request # Search diff --git a/boards/views/api.py b/boards/views/api.py --- a/boards/views/api.py +++ b/boards/views/api.py @@ -1,13 +1,16 @@ import json import logging +import xml.etree.ElementTree as ET + from django.db import transaction from django.http import HttpResponse from django.shortcuts import get_object_or_404 from django.core import serializers from boards.forms import PostForm, PlainErrorList -from boards.models import Post, Thread, Tag +from boards.models import Post, Thread, Tag, GlobalId +from boards.models.post.sync import SyncManager from boards.utils import datetime_to_epoch from boards.views.thread import ThreadView from boards.models.user import Notification @@ -234,7 +237,7 @@ def get_post_data(post_id, format_type=D # TODO Make a separate module for sync API methods def sync_pull(request): """ - Return 'get' request response for all posts. + Return 'pull' request response for all posts. """ request_xml = request.get('xml') if request_xml is None: @@ -242,5 +245,5 @@ def sync_pull(request): else: pass # TODO Parse the XML and get filters from it - xml = Post.objects.generate_response_get(posts) + xml = SyncManager().generate_response_get(posts) return HttpResponse(content=xml) diff --git a/boards/views/sync.py b/boards/views/sync.py --- a/boards/views/sync.py +++ b/boards/views/sync.py @@ -1,19 +1,23 @@ import xml.etree.ElementTree as et from django.http import HttpResponse, Http404 from boards.models import GlobalId, Post +from boards.models.post.sync import SyncManager -def respond_pull(request): +def response_pull(request): pass -def respond_get(request): +def response_get(request): """ Processes a GET request with post ID list and returns the posts XML list. Request should contain an 'xml' post attribute with the actual request XML. """ - request_xml = request.POST['xml'] + request_xml = request.body + + if request_xml is None: + return HttpResponse(content='Use the API') posts = [] @@ -22,13 +26,13 @@ def respond_get(request): for id_tag in model_tag: try: global_id = GlobalId.from_xml_element(id_tag, existing=True) - posts += Post.objects.filter(global_id=global_id) + posts.append(Post.objects.get(global_id=global_id)) except GlobalId.DoesNotExist: # This is normal. If we don't have such GlobalId in the system, # just ignore this ID and proceed to the next one. pass - response_xml = Post.objects.generate_response_get(posts) + response_xml = SyncManager().generate_response_get(posts) return HttpResponse(content=response_xml) @@ -40,7 +44,7 @@ def get_post_sync_data(request, post_id) raise Http404() content = 'Global ID: %s\n\nXML: %s' \ - % (post.global_id, Post.objects.generate_response_get([post])) + % (post.global_id, SyncManager().generate_response_get([post])) return HttpResponse(