Sync fixes

boards/management/commands/sync_with_server.py

@@ -1,74 +1,79 b''
 import re
 import xml.etree.ElementTree as ET
 
 import httplib2
 from django.core.management import BaseCommand
 
 from boards.models import GlobalId
 from boards.models.post.sync import SyncManager
 
 __author__ = 'neko259'
 
 
 REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)')
 
 
 class Command(BaseCommand):
     help = 'Send a sync or get request to the server.'
 
     def add_arguments(self, parser):
-        parser.add_argument('url', type=str)
-        parser.add_argument('--global_id', type=str, default='',
+        parser.add_argument('url', type=str, help='Server root url')
+        parser.add_argument('--global-id', type=str, default='',
                             help='Post global ID')
 
     def handle(self, *args, **options):
         url = options.get('url')
+
+        pull_url = url + 'api/sync/pull/'
+        get_url = url + 'api/sync/get/'
+
         global_id_str = options.get('global_id')
         if global_id_str:
             match = REGEX_GLOBAL_ID.match(global_id_str)
             if match:
                 key_type = match.group(1)
                 key = match.group(2)
                 local_id = match.group(3)
 
                 global_id = GlobalId(key_type=key_type, key=key,
                                      local_id=local_id)
 
                 xml = GlobalId.objects.generate_request_get([global_id])
                 # body = urllib.parse.urlencode(data)
                 h = httplib2.Http()
-                response, content = h.request(url, method="POST", body=xml)
+                response, content = h.request(get_url, method="POST", body=xml)
 
                 SyncManager.parse_response_get(content)
             else:
                 raise Exception('Invalid global ID')
         else:
             h = httplib2.Http()
             xml = GlobalId.objects.generate_request_pull()
-            response, content = h.request(url, method="POST", body=xml)
+            response, content = h.request(pull_url, method="POST", body=xml)
 
-            print(content)
+            print(content.decode() + '\n')
 
             root = ET.fromstring(content)
             status = root.findall('status')[0].text
             if status == 'success':
                 ids_to_sync = list()
 
                 models = root.findall('models')[0]
                 for model in models:
                     global_id, exists = GlobalId.from_xml_element(model)
-                    print(global_id)
                     if not exists:
+                        print(global_id)
                         ids_to_sync.append(global_id)
+                print()
 
                 if len(ids_to_sync) > 0:
                     xml = GlobalId.objects.generate_request_get(ids_to_sync)
                     # body = urllib.parse.urlencode(data)
                     h = httplib2.Http()
-                    response, content = h.request(url, method="POST", body=xml)
+                    response, content = h.request(get_url, method="POST", body=xml)
 
                     SyncManager.parse_response_get(content)
                 else:
                     print('Nothing to get, everything synced')
             else:
                 raise Exception('Invalid response status')

boards/models/post/manager.py

@@ -1,151 +1,152 b''
 import logging
 
 from datetime import datetime, timedelta, date
 from datetime import time as dtime
 
 from django.db import models, transaction
 from django.utils import timezone
 
 import boards
 
 from boards.models.user import Ban
 from boards.mdx_neboard import Parser
 from boards.models import PostImage, Attachment
 from boards import utils
 
 __author__ = 'neko259'
 
 IMAGE_TYPES = (
     'jpeg',
     'jpg',
     'png',
     'bmp',
     'gif',
 )
 
 POSTS_PER_DAY_RANGE = 7
 NO_IP = '0.0.0.0'
 
 
 class PostManager(models.Manager):
     @transaction.atomic
     def create_post(self, title: str, text: str, file=None, thread=None,
                     ip=NO_IP, tags: list=None, opening_posts: list=None,
                     tripcode=''):
         """
         Creates new post
         """
 
         if not utils.is_anonymous_mode():
             is_banned = Ban.objects.filter(ip=ip).exists()
 
         # TODO Raise specific exception and catch it in the views
         if is_banned:
             raise Exception("This user is banned")
 
         if not tags:
             tags = []
         if not opening_posts:
             opening_posts = []
 
         posting_time = timezone.now()
         new_thread = False
         if not thread:
             thread = boards.models.thread.Thread.objects.create(
                 bump_time=posting_time, last_edit_time=posting_time)
             list(map(thread.tags.add, tags))
             boards.models.thread.Thread.objects.process_oldest_threads()
             new_thread = True
 
         pre_text = Parser().preparse(text)
 
         post = self.create(title=title,
                            text=pre_text,
                            pub_time=posting_time,
                            poster_ip=ip,
                            thread=thread,
                            last_edit_time=posting_time,
                            tripcode=tripcode,
                            opening=new_thread)
         post.threads.add(thread)
 
         logger = logging.getLogger('boards.post.create')
 
         logger.info('Created post [{}] with text [{}] by {}'.format(post,
                                                                     post.get_text(),post.poster_ip))
 
         # TODO Move this to other place
         if file:
             file_type = file.name.split('.')[-1].lower()
             if file_type in IMAGE_TYPES:
                 post.images.add(PostImage.objects.create_with_hash(file))
             else:
                 post.attachments.add(Attachment.objects.create_with_hash(file))
 
         post.build_url()
         post.connect_replies()
         post.connect_threads(opening_posts)
         post.connect_notifications()
         post.set_global_id()
 
         # Thread needs to be bumped only when the post is already created
         if not new_thread:
             thread.last_edit_time = posting_time
             thread.bump()
             thread.save()
 
         return post
 
     def delete_posts_by_ip(self, ip):
         """
         Deletes all posts of the author with same IP
         """
 
         posts = self.filter(poster_ip=ip)
         for post in posts:
             post.delete()
 
     @utils.cached_result()
     def get_posts_per_day(self) -> float:
         """
         Gets average count of posts per day for the last 7 days
         """
 
         day_end = date.today()
         day_start = day_end - timedelta(POSTS_PER_DAY_RANGE)
 
         day_time_start = timezone.make_aware(datetime.combine(
             day_start, dtime()), timezone.get_current_timezone())
         day_time_end = timezone.make_aware(datetime.combine(
             day_end, dtime()), timezone.get_current_timezone())
 
         posts_per_period = float(self.filter(
             pub_time__lte=day_time_end,
             pub_time__gte=day_time_start).count())
 
         ppd = posts_per_period / POSTS_PER_DAY_RANGE
 
         return ppd
 
     @transaction.atomic
     def import_post(self, title: str, text: str, pub_time: str, global_id,
                     opening_post=None, tags=list()):
         is_opening = opening_post is None
         if is_opening:
             thread = boards.models.thread.Thread.objects.create(
                 bump_time=pub_time, last_edit_time=pub_time)
             list(map(thread.tags.add, tags))
         else:
             thread = opening_post.get_thread()
 
         post = self.create(title=title, text=text,
                            pub_time=pub_time,
                            poster_ip=NO_IP,
                            last_edit_time=pub_time,
-                           thread_id=thread.id,
                            global_id=global_id,
-                           opening=is_opening)
+                           opening=is_opening,
+                           thread=thread)
 
+        post.threads.add(thread)
         post.build_url()
         post.connect_replies()
         post.connect_notifications()

boards/models/post/sync.py

@@ -1,190 +1,201 b''
 import xml.etree.ElementTree as et
 from django.db import transaction
 from boards.models import KeyPair, GlobalId, Signature, Post, Tag
 
 ENCODING_UNICODE = 'unicode'
 
 TAG_MODEL = 'model'
 TAG_REQUEST = 'request'
 TAG_RESPONSE = 'response'
 TAG_ID = 'id'
 TAG_STATUS = 'status'
 TAG_MODELS = 'models'
 TAG_TITLE = 'title'
 TAG_TEXT = 'text'
 TAG_THREAD = 'thread'
 TAG_PUB_TIME = 'pub-time'
 TAG_SIGNATURES = 'signatures'
 TAG_SIGNATURE = 'signature'
 TAG_CONTENT = 'content'
 TAG_ATTACHMENTS = 'attachments'
 TAG_ATTACHMENT = 'attachment'
 TAG_TAGS = 'tags'
 TAG_TAG = 'tag'
 
 TYPE_GET = 'get'
 
 ATTR_VERSION = 'version'
 ATTR_TYPE = 'type'
 ATTR_NAME = 'name'
 ATTR_VALUE = 'value'
 ATTR_MIMETYPE = 'mimetype'
 ATTR_KEY = 'key'
 
 STATUS_SUCCESS = 'success'
 
 
 class SyncManager:
     @staticmethod
     def generate_response_get(model_list: list):
         response = et.Element(TAG_RESPONSE)
 
         status = et.SubElement(response, TAG_STATUS)
         status.text = STATUS_SUCCESS
 
         models = et.SubElement(response, TAG_MODELS)
 
         for post in model_list:
             model = et.SubElement(models, TAG_MODEL)
             model.set(ATTR_NAME, 'post')
 
             content_tag = et.SubElement(model, TAG_CONTENT)
 
             tag_id = et.SubElement(content_tag, TAG_ID)
             post.global_id.to_xml_element(tag_id)
 
             title = et.SubElement(content_tag, TAG_TITLE)
             title.text = post.title
 
             text = et.SubElement(content_tag, TAG_TEXT)
             text.text = post.get_sync_text()
 
             thread = post.get_thread()
             if post.is_opening():
                 tag_tags = et.SubElement(content_tag, TAG_TAGS)
                 for tag in thread.get_tags():
                     tag_tag = et.SubElement(tag_tags, TAG_TAG)
                     tag_tag.text = tag.name
             else:
                 tag_thread = et.SubElement(content_tag, TAG_THREAD)
                 thread_id = et.SubElement(tag_thread, TAG_ID)
                 thread.get_opening_post().global_id.to_xml_element(thread_id)
 
             pub_time = et.SubElement(content_tag, TAG_PUB_TIME)
             pub_time.text = str(post.get_pub_time_str())
 
             signatures_tag = et.SubElement(model, TAG_SIGNATURES)
             post_signatures = post.global_id.signature_set.all()
             if post_signatures:
                 signatures = post_signatures
                 # TODO Adding signature to a post is not yet added. For now this
                 # block is useless
             else:
                 # TODO Maybe the signature can be computed only once after
                 # the post is added? Need to add some on_save signal queue
                 # and add this there.
                 key = KeyPair.objects.get(public_key=post.global_id.key)
-                signatures = [Signature(
+                signature = Signature(
                     key_type=key.key_type,
                     key=key.public_key,
                     signature=key.sign(et.tostring(content_tag, ENCODING_UNICODE)),
-                )]
+                    global_id=post.global_id,
+                )
+                signature.save()
+                signatures = [signature]
             for signature in signatures:
                 signature_tag = et.SubElement(signatures_tag, TAG_SIGNATURE)
                 signature_tag.set(ATTR_TYPE, signature.key_type)
                 signature_tag.set(ATTR_VALUE, signature.signature)
                 signature_tag.set(ATTR_KEY, signature.key)
 
         return et.tostring(response, ENCODING_UNICODE)
 
     @staticmethod
     @transaction.atomic
     def parse_response_get(response_xml):
         tag_root = et.fromstring(response_xml)
         tag_status = tag_root.find(TAG_STATUS)
         if STATUS_SUCCESS == tag_status.text:
             tag_models = tag_root.find(TAG_MODELS)
             for tag_model in tag_models:
                 tag_content = tag_model.find(TAG_CONTENT)
 
                 signatures = SyncManager._verify_model(tag_content, tag_model)
 
                 tag_id = tag_content.find(TAG_ID)
                 global_id, exists = GlobalId.from_xml_element(tag_id)
 
                 if exists:
                     print('Post with same ID already exists')
                 else:
                     global_id.save()
                     for signature in signatures:
                         signature.global_id = global_id
                         signature.save()
 
                     title = tag_content.find(TAG_TITLE).text
                     text = tag_content.find(TAG_TEXT).text
                     pub_time = tag_content.find(TAG_PUB_TIME).text
 
                     thread = tag_content.find(TAG_THREAD)
                     tags = []
                     if thread:
-                        opening_post = Post.objects.get(
-                            id=thread.find(TAG_ID).text)
+                        thread_id = thread.find(TAG_ID)
+                        op_global_id, exists = GlobalId.from_xml_element(thread_id)
+                        if exists:
+                            opening_post = Post.objects.get(global_id=op_global_id)
+                        else:
+                            raise Exception('Load the OP first')
                     else:
                         opening_post = None
                         tag_tags = tag_content.find(TAG_TAGS)
                         for tag_tag in tag_tags:
-                            tag, created = Tag.objects.get_or_create(name=tag_tag.text)
+                            tag, created = Tag.objects.get_or_create(
+                                name=tag_tag.text)
                             tags.append(tag)
 
                     # TODO Check that the replied posts are already present
                     # before adding new ones
 
                     # TODO Get images
 
                     post = Post.objects.import_post(
                         title=title, text=text, pub_time=pub_time,
                         opening_post=opening_post, tags=tags,
                         global_id=global_id)
         else:
             # TODO Throw an exception?
             pass
 
     @staticmethod
     def generate_response_pull():
         response = et.Element(TAG_RESPONSE)
 
         status = et.SubElement(response, TAG_STATUS)
         status.text = STATUS_SUCCESS
 
         models = et.SubElement(response, TAG_MODELS)
 
         for post in Post.objects.all():
             tag_id = et.SubElement(models, TAG_ID)
             post.global_id.to_xml_element(tag_id)
 
         return et.tostring(response, ENCODING_UNICODE)
 
     @staticmethod
     def _verify_model(tag_content, tag_model):
         """
         Verifies all signatures for a single model.
         """
 
         signatures = []
 
         tag_signatures = tag_model.find(TAG_SIGNATURES)
         for tag_signature in tag_signatures:
             signature_type = tag_signature.get(ATTR_TYPE)
             signature_value = tag_signature.get(ATTR_VALUE)
             signature_key = tag_signature.get(ATTR_KEY)
 
             signature = Signature(key_type=signature_type,
                                   key=signature_key,
                                   signature=signature_value)
-            signatures.append(signature)
+
+            content = et.tostring(tag_content, ENCODING_UNICODE)
 
             if not KeyPair.objects.verify(
-                    signature, et.tostring(tag_content, ENCODING_UNICODE)):
-                raise Exception('Invalid model signature')
+                    signature, content):
+                raise Exception('Invalid model signature for {}'.format(content))
+
+            signatures.append(signature)
 
         return signatures