Added test for reflinks. Added management command to get posts from other node

(incomplete). Added ability to parse global reflinks in the text

boards/management/commands/sync_with_server.py

@@ -0,0 +1,37 b''
+import re
+import urllib
+import httplib2
+from django.core.management import BaseCommand
+from boards.models import GlobalId
+
+__author__ = 'neko259'
+
+
+REGEX_GLOBAL_ID = r'\[(\w+)\]\[(\w+)\]\[(\d+)\]'
+
+
+class Command(BaseCommand):
+    help = 'Send a sync or get request to the server.' + \
+           'sync_with_server <server_url> [post_global_id]'
+
+    def handle(self, *args, **options):
+        url = args[0]
+        if len(args) > 1:
+            global_id_str = args[1]
+            match = re.match(REGEX_GLOBAL_ID, global_id_str)
+            key_type = match.group(1)
+            key = match.group(2)
+            local_id = match.group(3)
+
+            global_id = GlobalId(key_type=key_type, key=key,
+                                     local_id=local_id)
+
+            xml = GlobalId.objects.generate_request_get([global_id])
+            data = {'xml': xml}
+            body = urllib.urlencode(data)
+            h = httplib2.Http()
+            response, content = h.request(url, method="POST", body=body)
+
+            # TODO Parse content and get the model list
+        else:
+            raise Exception('Full sync is not supported yet.')

development/run.sh

@@ -0,0 +1,3 b''
+#! /usr/bin/env sh
+
+python3 manage.py runserver [::]:8000

development/test.sh

@@ -0,0 +1,3 b''
+#! /usr/bin/env sh
+
+python3 manage.py test

boards/mdx_neboard.py

@@ -10,6 +10,7 b' import boards'
 
 
 REFLINK_PATTERN = re.compile(r'^\d+$')
+GLOBAL_REFLINK_PATTERN = re.compile(r'^(\w+)::([^:]+)::(\d+)$')
 MULTI_NEWLINES_PATTERN = re.compile(r'(\r?\n){2,}')
 ONE_NEWLINE = '\n'
 
@@ -115,10 +116,28 b' class CodePattern(TextFormatter):'
 
 
 def render_reflink(tag_name, value, options, parent, context):
-    if not REFLINK_PATTERN.match(value):
-        return '>>%s' % value
+    post_id = None
 
-    post_id = int(value)
+    matches = REFLINK_PATTERN.findall(value)
+    if matches:
+        post_id = int(matches[0][0])
+    else:
+        match = GLOBAL_REFLINK_PATTERN.match(value)
+        if match:
+            key_type = match.group(1)
+            key = match.group(2)
+            local_id = match.group(3)
+
+            try:
+                global_id = boards.models.GlobalId.objects.get(key_type=key_type,
+                    key=key, local_id=local_id)
+                for post in boards.models.Post.objects.filter(global_id=global_id).only('id'):
+                    post_id = post.id
+            except boards.models.GlobalId.DoesNotExist:
+                pass
+
+    if not post_id:
+        return value
 
     posts = boards.models.Post.objects.filter(id=post_id)
     if posts.exists():

boards/models/post.py

@@ -17,6 +17,7 b' from boards.models.base import Viewable'
 from boards.models.thread import Thread
 from boards import utils
 
+ENCODING_UNICODE = 'unicode'
 
 APP_LABEL_BOARDS = 'boards'
 
@@ -40,6 +41,7 b" NO_IP = '0.0.0.0'"
 UNKNOWN_UA = ''
 
 REGEX_REPLY = re.compile(r'\[post\](\d+)\[/post\]')
+REGEX_GLOBAL_REPLY = re.compile(r'\[post\](\w+)::([^:]+)::(\d+)\[/post\]')
 
 TAG_MODEL = 'model'
 TAG_REQUEST = 'request'
@@ -51,9 +53,6 b" TAG_TITLE = 'title'"
 TAG_TEXT = 'text'
 TAG_THREAD = 'thread'
 TAG_PUB_TIME = 'pub-time'
-TAG_EDIT_TIME = 'edit-time'
-TAG_PREVIOUS = 'previous'
-TAG_NEXT = 'next'
 TAG_SIGNATURES = 'signatures'
 TAG_SIGNATURE = 'signature'
 TAG_CONTENT = 'content'
@@ -151,6 +150,7 b' class PostManager(models.Manager):'
         for post in posts:
             self.delete_post(post)
 
+    # TODO This can be moved into a post
     def connect_replies(self, post):
         """
         Connects replies to a post to show them as a reflink map
@@ -196,25 +196,7 b' class PostManager(models.Manager):'
         cache.set(cache_key, ppd)
         return ppd
 
-    def generate_request_get(self, model_list: list):
-        """
-        Form a get request from a list of ModelId objects.
-        """
-
-        request = et.Element(TAG_REQUEST)
-        request.set(ATTR_TYPE, TYPE_GET)
-        request.set(ATTR_VERSION, '1.0')
-
-        model = et.SubElement(request, TAG_MODEL)
-        model.set(ATTR_VERSION, '1.0')
-        model.set(ATTR_NAME, 'post')
-
-        for post in model_list:
-            tag_id = et.SubElement(model, TAG_ID)
-            post.global_id.to_xml_element(tag_id)
-
-        return et.tostring(request, 'unicode')
-
+    # TODO Make a separate sync facade?
     def generate_response_get(self, model_list: list):
         response = et.Element(TAG_RESPONSE)
 
@@ -236,6 +218,7 b' class PostManager(models.Manager):'
             title.text = post.title
 
             text = et.SubElement(content_tag, TAG_TEXT)
+            # TODO Replace local links by global ones in the text
             text.text = post.text.raw
 
             if not post.is_opening():
@@ -260,14 +243,42 b' class PostManager(models.Manager):'
                 signatures = [Signature(
                     key_type=key.key_type,
                     key=key.public_key,
-                    signature=key.sign(et.tostring(model, 'unicode')),
+                    signature=key.sign(et.tostring(model, ENCODING_UNICODE)),
                     )]
             for signature in signatures:
                 signature_tag = et.SubElement(signatures_tag, TAG_SIGNATURE)
                 signature_tag.set(ATTR_TYPE, signature.key_type)
                 signature_tag.set(ATTR_VALUE, signature.signature)
 
-        return et.tostring(response, 'unicode')
+        return et.tostring(response, ENCODING_UNICODE)
+
+    def parse_response_get(self, response_xml):
+        tag_root = et.fromstring(response_xml)
+        tag_status = tag_root[0]
+        if 'success' == tag_status.text:
+            tag_models = tag_root[1]
+            for tag_model in tag_models:
+                tag_content = tag_model[0]
+                tag_id = tag_content[1]
+                try:
+                    GlobalId.from_xml_element(tag_id, existing=True)
+                    # If this post already exists, just continue
+                    # TODO Compare post content and update the post if necessary
+                    pass
+                except GlobalId.DoesNotExist:
+                    global_id = GlobalId.from_xml_element(tag_id)
+
+                    title = tag_content.find(TAG_TITLE).text
+                    text = tag_content.find(TAG_TEXT).text
+                    # TODO Check that the replied posts are already present
+                    # before adding new ones
+
+                    # TODO Pub time, thread, tags
+
+                    post = Post.objects.create(title=title, text=text)
+        else:
+            # TODO Throw an exception?
+            pass
 
 
 class Post(models.Model, Viewable):
@@ -316,15 +327,7 b' class Post(models.Model, Viewable):'
                self.text.raw[:50] + ')'
 
     def get_title(self):
-        """
-        Gets original post title or part of its text.
-        """
-
-        title = self.title
-        if not title:
-            title = self.text.rendered
-
-        return title
+        return self.title
 
     def build_refmap(self):
         """
@@ -491,8 +494,25 b' class Post(models.Model, Viewable):'
     def get_pub_time_epoch(self):
         return utils.datetime_to_epoch(self.pub_time)
 
-    def get_edit_time_epoch(self):
-        return utils.datetime_to_epoch(self.last_edit_time)
+    def get_replied_ids(self):
+        """
+        Gets ID list of the posts that this post replies.
+        """
 
-    def get_replied_ids(self):
-        return re.findall(REGEX_REPLY, self.text.raw)
+        local_replied = REGEX_REPLY.findall(self.text.raw)
+        global_replied = []
+        # TODO Similar code is used in mdx_neboard, maybe it can be extracted
+        # into a method?
+        for match in REGEX_GLOBAL_REPLY.findall(self.text.raw):
+            key_type = match[0]
+            key = match[1]
+            local_id = match[2]
+
+            try:
+                global_id = GlobalId.objects.get(key_type=key_type,
+                    key=key, local_id=local_id)
+                for post in Post.objects.filter(global_id=global_id).only('id'):
+                    global_replied.append(post.id)
+            except GlobalId.DoesNotExist:
+                pass
+        return local_replied + global_replied

boards/models/signature.py

@@ -2,15 +2,48 b' import xml.etree.ElementTree as et'
 from django.db import models
 
 
+TAG_MODEL = 'model'
+TAG_REQUEST = 'request'
+TAG_ID = 'id'
+
+TYPE_GET = 'get'
+
+ATTR_VERSION = 'version'
+ATTR_TYPE = 'type'
+ATTR_NAME = 'name'
+
 ATTR_KEY = 'key'
 ATTR_KEY_TYPE = 'type'
 ATTR_LOCAL_ID = 'local-id'
 
 
+class GlobalIdManager(models.Manager):
+    def generate_request_get(self, global_id_list: list):
+        """
+        Form a get request from a list of ModelId objects.
+        """
+
+        request = et.Element(TAG_REQUEST)
+        request.set(ATTR_TYPE, TYPE_GET)
+        request.set(ATTR_VERSION, '1.0')
+
+        model = et.SubElement(request, TAG_MODEL)
+        model.set(ATTR_VERSION, '1.0')
+        model.set(ATTR_NAME, 'post')
+
+        for global_id in global_id_list:
+            tag_id = et.SubElement(model, TAG_ID)
+            global_id.to_xml_element(tag_id)
+
+        return et.tostring(request, 'unicode')
+
+
 class GlobalId(models.Model):
     class Meta:
         app_label = 'boards'
 
+    objects = GlobalIdManager()
+
     def __init__(self, *args, **kwargs):
         models.Model.__init__(self, *args, **kwargs)
 
@@ -24,7 +57,7 b' class GlobalId(models.Model):'
     local_id = models.IntegerField()
 
     def __str__(self):
-        return '[%s][%s][%d]' % (self.key_type, self.key, self.local_id)
+        return '%s::%s::%d' % (self.key_type, self.key, self.local_id)
 
     def to_xml_element(self, element: et.Element):
         """

boards/models/sync_key.py

@@ -52,7 +52,7 b' class KeyPair(models.Model):'
     primary = models.BooleanField(default=False)
 
     def __str__(self):
-        return '[%s][%s]' % (self.key_type, self.public_key)
+        return '%s::%s' % (self.key_type, self.public_key)
 
     def sign(self, string):
         private = SigningKey.from_string(base64.b64decode(

boards/tests/test_keys.py

@@ -36,7 +36,7 b' class KeyTest(TestCase):'
     def test_request_get(self):
         post = self._create_post_with_key()
 
-        request = Post.objects.generate_request_get([post])
+        request = GlobalId.objects.generate_request_get([post.global_id])
         logger.debug(request)
 
         key = KeyPair.objects.get(primary=True)

boards/tests/test_post.py

@@ -1,7 +1,7 b''
 from django.core.paginator import Paginator
 from django.test import TestCase
 from boards import settings
-from boards.models import Tag, Post, Thread
+from boards.models import Tag, Post, Thread, KeyPair
 
 
 class PostTests(TestCase):
@@ -109,4 +109,34 b' class PostTests(TestCase):'
         first_post = posts_in_second_page[0]
 
         self.assertEqual(all_threads[settings.THREADS_PER_PAGE].id,
-                         first_post.id)
  No newline at end of file
+                         first_post.id)
+
+    def test_reflinks(self):
+        """
+        Tests that reflinks are parsed within post and connecting replies
+        to the replied posts.
+
+        Local reflink example: [post]123[/post]
+        Global reflink example: [post]key_type::key::123[/post]
+        """
+
+        key = KeyPair.objects.generate_key(primary=True)
+
+        tag = Tag.objects.create(name='test_tag')
+
+        post = Post.objects.create_post(title='', text='', tags=[tag])
+        post_local_reflink = Post.objects.create_post(title='',
+                text='[post]%d[/post]' % post.id, thread=post.get_thread())
+
+        self.assertTrue(post_local_reflink in post.referenced_posts.all(),
+            'Local reflink not connecting posts.')
+
+        post_global_reflink = Post.objects.create_post(title='',
+                text='[post]%s::%s::%d[/post]' % (
+                    post.global_id.key_type, post.global_id.key, post.id),
+                thread=post.get_thread())
+
+        self.assertTrue(post_global_reflink in post.referenced_posts.all(),
+            'Global reflink not connecting posts.')
+
+        # TODO Check that links are parsed into the rendered text

requirements.txt

@@ -1,3 +1,4 b''
+httplib2
 simplejson
 south>=0.8.4
 haystack

todo.txt

@@ -15,6 +15,7 b''
 * Subscribing to tag via AJAX
 * Add buttons to insert a named link or a named quote to the markup panel
 * Add support for "attention posts" that are shown in the header"
+* Use absolute post reflinks in the raw text
 
 = Bugs =
 * Search sort order is confusing