Added test for reflinks. Added management command to get posts from other node

(incomplete). Added ability to parse global reflinks in the text

boards/management/commands/sync_with_server.py

@@ -0,0 +1,37 b''
+import re
+import urllib
+import httplib2
+from django.core.management import BaseCommand
+from boards.models import GlobalId
+
+__author__ = 'neko259'
+
+
+REGEX_GLOBAL_ID = r'\[(\w+)\]\[(\w+)\]\[(\d+)\]'
+
+
+class Command(BaseCommand):
+    help = 'Send a sync or get request to the server.' + \
+           'sync_with_server <server_url> [post_global_id]'
+
+    def handle(self, *args, **options):
+        url = args[0]
+        if len(args) > 1:
+            global_id_str = args[1]
+            match = re.match(REGEX_GLOBAL_ID, global_id_str)
+            key_type = match.group(1)
+            key = match.group(2)
+            local_id = match.group(3)
+
+            global_id = GlobalId(key_type=key_type, key=key,
+                                     local_id=local_id)
+
+            xml = GlobalId.objects.generate_request_get([global_id])
+            data = {'xml': xml}
+            body = urllib.urlencode(data)
+            h = httplib2.Http()
+            response, content = h.request(url, method="POST", body=body)
+
+            # TODO Parse content and get the model list
+        else:
+            raise Exception('Full sync is not supported yet.')

development/run.sh

@@ -0,0 +1,3 b''
+#! /usr/bin/env sh
+
+python3 manage.py runserver [::]:8000

development/test.sh

@@ -0,0 +1,3 b''
+#! /usr/bin/env sh
+
+python3 manage.py test

boards/mdx_neboard.py

@@ -1,182 +1,201 b''
 # coding=utf-8
 
 import re
 import bbcode
 
 import boards
 
 
 __author__ = 'neko259'
 
 
 REFLINK_PATTERN = re.compile(r'^\d+$')
+GLOBAL_REFLINK_PATTERN = re.compile(r'^(\w+)::([^:]+)::(\d+)$')
 MULTI_NEWLINES_PATTERN = re.compile(r'(\r?\n){2,}')
 ONE_NEWLINE = '\n'
 
 
 class TextFormatter():
     """
     An interface for formatter that can be used in the text format panel
     """
 
     def __init__(self):
         pass
 
     name = ''
 
     # Left and right tags for the button preview
     preview_left = ''
     preview_right = ''
 
     # Left and right characters for the textarea input
     format_left = ''
     format_right = ''
 
 
 class AutolinkPattern():
     def handleMatch(self, m):
         link_element = etree.Element('a')
         href = m.group(2)
         link_element.set('href', href)
         link_element.text = href
 
         return link_element
 
 
 class QuotePattern(TextFormatter):
     name = 'q'
     preview_left = '<span class="multiquote">'
     preview_right = '</span>'
 
     format_left = '[quote]'
     format_right = '[/quote]'
 
 
 class SpoilerPattern(TextFormatter):
     name = 'spoiler'
     preview_left = '<span class="spoiler">'
     preview_right = '</span>'
 
     format_left = '[spoiler]'
     format_right = '[/spoiler]'
 
     def handleMatch(self, m):
         quote_element = etree.Element('span')
         quote_element.set('class', 'spoiler')
         quote_element.text = m.group(2)
 
         return quote_element
 
 
 class CommentPattern(TextFormatter):
     name = ''
     preview_left = '<span class="comment">// '
     preview_right = '</span>'
 
     format_left = '[comment]'
     format_right = '[/comment]'
 
 
 # TODO Use <s> tag here
 class StrikeThroughPattern(TextFormatter):
     name = 's'
     preview_left = '<span class="strikethrough">'
     preview_right = '</span>'
 
     format_left = '[s]'
     format_right = '[/s]'
 
 
 class ItalicPattern(TextFormatter):
     name = 'i'
     preview_left = '<i>'
     preview_right = '</i>'
 
     format_left = '[i]'
     format_right = '[/i]'
 
 
 class BoldPattern(TextFormatter):
     name = 'b'
     preview_left = '<b>'
     preview_right = '</b>'
 
     format_left = '[b]'
     format_right = '[/b]'
 
 
 class CodePattern(TextFormatter):
     name = 'code'
     preview_left = '<code>'
     preview_right = '</code>'
 
     format_left = '[code]'
     format_right = '[/code]'
 
 
 def render_reflink(tag_name, value, options, parent, context):
-    if not REFLINK_PATTERN.match(value):
-        return '>>%s' % value
+    post_id = None
 
-    post_id = int(value)
+    matches = REFLINK_PATTERN.findall(value)
+    if matches:
+        post_id = int(matches[0][0])
+    else:
+        match = GLOBAL_REFLINK_PATTERN.match(value)
+        if match:
+            key_type = match.group(1)
+            key = match.group(2)
+            local_id = match.group(3)
+
+            try:
+                global_id = boards.models.GlobalId.objects.get(key_type=key_type,
+                    key=key, local_id=local_id)
+                for post in boards.models.Post.objects.filter(global_id=global_id).only('id'):
+                    post_id = post.id
+            except boards.models.GlobalId.DoesNotExist:
+                pass
+
+    if not post_id:
+        return value
 
     posts = boards.models.Post.objects.filter(id=post_id)
     if posts.exists():
         post = posts[0]
 
         return '<a href="%s">&gt;&gt;%s</a>' % (post.get_url(), post_id)
     else:
         return '>>%s' % value
 
 
 def render_quote(tag_name, value, options, parent, context):
     source = ''
     if 'source' in options:
         source = options['source']
 
     result = ''
     if source:
         result = '<div class="multiquote"><div class="quote-header">%s</div><div class="quote-text">%s</div></div>' % (source, value)
     else:
         result = '<div class="multiquote"><div class="quote-text">%s</div></div>' % value
 
     return result
 
 
 def preparse_text(text):
     """
     Performs manual parsing before the bbcode parser is used.
     """
 
     return MULTI_NEWLINES_PATTERN.sub(ONE_NEWLINE, text)
 
 
 def bbcode_extended(markup):
     # The newline hack is added because br's margin does not work in all
     # browsers except firefox, when the div's does.
     parser = bbcode.Parser(newline='<div class="br"></div>')
     parser.add_formatter('post', render_reflink, strip=True)
     parser.add_formatter('quote', render_quote, strip=True)
     parser.add_simple_formatter('comment',
             u'<span class="comment">//%(value)s</span>')
     parser.add_simple_formatter('spoiler',
             u'<span class="spoiler">%(value)s</span>')
     # TODO Use <s> here
     parser.add_simple_formatter('s',
             u'<span class="strikethrough">%(value)s</span>')
     # TODO Why not use built-in tag?
     parser.add_simple_formatter('code',
             u'<pre><code>%(value)s</pre></code>', render_embedded=False)
 
     text = preparse_text(markup)
     return parser.format(text)
 
 formatters = [
     QuotePattern,
     SpoilerPattern,
     ItalicPattern,
     BoldPattern,
     CommentPattern,
     StrikeThroughPattern,
     CodePattern,
 ]

boards/models/post.py

@@ -1,498 +1,518 b''
 from datetime import datetime, timedelta, date
 from datetime import time as dtime
 import logging
 import re
 import xml.etree.ElementTree as et
 
 from django.core.cache import cache
 from django.core.urlresolvers import reverse
 from django.db import models, transaction
 from django.template.loader import render_to_string
 from django.utils import timezone
 
 from markupfield.fields import MarkupField
 
 from boards.models import PostImage, KeyPair, GlobalId, Signature
 from boards.models.base import Viewable
 from boards.models.thread import Thread
 from boards import utils
 
+ENCODING_UNICODE = 'unicode'
 
 APP_LABEL_BOARDS = 'boards'
 
 CACHE_KEY_PPD = 'ppd'
 CACHE_KEY_POST_URL = 'post_url'
 
 POSTS_PER_DAY_RANGE = 7
 
 BAN_REASON_AUTO = 'Auto'
 
 IMAGE_THUMB_SIZE = (200, 150)
 
 TITLE_MAX_LENGTH = 200
 
 DEFAULT_MARKUP_TYPE = 'bbcode'
 
 # TODO This should be removed
 NO_IP = '0.0.0.0'
 
 # TODO Real user agent should be saved instead of this
 UNKNOWN_UA = ''
 
 REGEX_REPLY = re.compile(r'\[post\](\d+)\[/post\]')
+REGEX_GLOBAL_REPLY = re.compile(r'\[post\](\w+)::([^:]+)::(\d+)\[/post\]')
 
 TAG_MODEL = 'model'
 TAG_REQUEST = 'request'
 TAG_RESPONSE = 'response'
 TAG_ID = 'id'
 TAG_STATUS = 'status'
 TAG_MODELS = 'models'
 TAG_TITLE = 'title'
 TAG_TEXT = 'text'
 TAG_THREAD = 'thread'
 TAG_PUB_TIME = 'pub-time'
-TAG_EDIT_TIME = 'edit-time'
-TAG_PREVIOUS = 'previous'
-TAG_NEXT = 'next'
 TAG_SIGNATURES = 'signatures'
 TAG_SIGNATURE = 'signature'
 TAG_CONTENT = 'content'
 TAG_ATTACHMENTS = 'attachments'
 TAG_ATTACHMENT = 'attachment'
 
 TYPE_GET = 'get'
 
 ATTR_VERSION = 'version'
 ATTR_TYPE = 'type'
 ATTR_NAME = 'name'
 ATTR_VALUE = 'value'
 ATTR_MIMETYPE = 'mimetype'
 
 STATUS_SUCCESS = 'success'
 
 logger = logging.getLogger(__name__)
 
 
 class PostManager(models.Manager):
     def create_post(self, title, text, image=None, thread=None, ip=NO_IP,
                     tags=None):
         """
         Creates new post
         """
 
         if not tags:
             tags = []
 
         posting_time = timezone.now()
         if not thread:
             thread = Thread.objects.create(bump_time=posting_time,
                                            last_edit_time=posting_time)
             new_thread = True
         else:
             thread.bump()
             thread.last_edit_time = posting_time
             thread.save()
             new_thread = False
 
         post = self.create(title=title,
                            text=text,
                            pub_time=posting_time,
                            thread_new=thread,
                            poster_ip=ip,
                            poster_user_agent=UNKNOWN_UA,  # TODO Get UA at
                            # last!
                            last_edit_time=posting_time)
 
         post.set_global_id()
 
         if image:
             post_image = PostImage.objects.create(image=image)
             post.images.add(post_image)
             logger.info('Created image #%d for post #%d' % (post_image.id,
                                                             post.id))
 
         thread.replies.add(post)
         list(map(thread.add_tag, tags))
 
         if new_thread:
             Thread.objects.process_oldest_threads()
         self.connect_replies(post)
 
         logger.info('Created post #%d with title %s'
                     % (post.id, post.get_title()))
 
         return post
 
     def delete_post(self, post):
         """
         Deletes post and update or delete its thread
         """
 
         post_id = post.id
 
         thread = post.get_thread()
 
         if post.is_opening():
             thread.delete()
         else:
             thread.last_edit_time = timezone.now()
             thread.save()
 
             post.delete()
 
         logger.info('Deleted post #%d (%s)' % (post_id, post.get_title()))
 
     def delete_posts_by_ip(self, ip):
         """
         Deletes all posts of the author with same IP
         """
 
         posts = self.filter(poster_ip=ip)
         for post in posts:
             self.delete_post(post)
 
+    # TODO This can be moved into a post
     def connect_replies(self, post):
         """
         Connects replies to a post to show them as a reflink map
         """
 
         for reply_number in post.get_replied_ids():
             ref_post = self.filter(id=reply_number)
             if ref_post.count() > 0:
                 referenced_post = ref_post[0]
                 referenced_post.referenced_posts.add(post)
                 referenced_post.last_edit_time = post.pub_time
                 referenced_post.build_refmap()
                 referenced_post.save(update_fields=['refmap', 'last_edit_time'])
 
                 referenced_thread = referenced_post.get_thread()
                 referenced_thread.last_edit_time = post.pub_time
                 referenced_thread.save(update_fields=['last_edit_time'])
 
     def get_posts_per_day(self):
         """
         Gets average count of posts per day for the last 7 days
         """
 
         day_end = date.today()
         day_start = day_end - timedelta(POSTS_PER_DAY_RANGE)
 
         cache_key = CACHE_KEY_PPD + str(day_end)
         ppd = cache.get(cache_key)
         if ppd:
             return ppd
 
         day_time_start = timezone.make_aware(datetime.combine(
             day_start, dtime()), timezone.get_current_timezone())
         day_time_end = timezone.make_aware(datetime.combine(
             day_end, dtime()), timezone.get_current_timezone())
 
         posts_per_period = float(self.filter(
             pub_time__lte=day_time_end,
             pub_time__gte=day_time_start).count())
 
         ppd = posts_per_period / POSTS_PER_DAY_RANGE
 
         cache.set(cache_key, ppd)
         return ppd
 
-    def generate_request_get(self, model_list: list):
-        """
-        Form a get request from a list of ModelId objects.
-        """
-
-        request = et.Element(TAG_REQUEST)
-        request.set(ATTR_TYPE, TYPE_GET)
-        request.set(ATTR_VERSION, '1.0')
-
-        model = et.SubElement(request, TAG_MODEL)
-        model.set(ATTR_VERSION, '1.0')
-        model.set(ATTR_NAME, 'post')
-
-        for post in model_list:
-            tag_id = et.SubElement(model, TAG_ID)
-            post.global_id.to_xml_element(tag_id)
-
-        return et.tostring(request, 'unicode')
-
+    # TODO Make a separate sync facade?
     def generate_response_get(self, model_list: list):
         response = et.Element(TAG_RESPONSE)
 
         status = et.SubElement(response, TAG_STATUS)
         status.text = STATUS_SUCCESS
 
         models = et.SubElement(response, TAG_MODELS)
 
         for post in model_list:
             model = et.SubElement(models, TAG_MODEL)
             model.set(ATTR_NAME, 'post')
 
             content_tag = et.SubElement(model, TAG_CONTENT)
 
             tag_id = et.SubElement(content_tag, TAG_ID)
             post.global_id.to_xml_element(tag_id)
 
             title = et.SubElement(content_tag, TAG_TITLE)
             title.text = post.title
 
             text = et.SubElement(content_tag, TAG_TEXT)
+            # TODO Replace local links by global ones in the text
             text.text = post.text.raw
 
             if not post.is_opening():
                 thread = et.SubElement(content_tag, TAG_THREAD)
                 thread.text = str(post.get_thread().get_opening_post_id())
             else:
                 # TODO Output tags here
                 pass
 
             pub_time = et.SubElement(content_tag, TAG_PUB_TIME)
             pub_time.text = str(post.get_pub_time_epoch())
 
             signatures_tag = et.SubElement(model, TAG_SIGNATURES)
             post_signatures = post.signature.all()
             if post_signatures:
                 signatures = post.signatures
             else:
                 # TODO Maybe the signature can be computed only once after
                 # the post is added? Need to add some on_save signal queue
                 # and add this there.
                 key = KeyPair.objects.get(public_key=post.global_id.key)
                 signatures = [Signature(
                     key_type=key.key_type,
                     key=key.public_key,
-                    signature=key.sign(et.tostring(model, 'unicode')),
+                    signature=key.sign(et.tostring(model, ENCODING_UNICODE)),
                     )]
             for signature in signatures:
                 signature_tag = et.SubElement(signatures_tag, TAG_SIGNATURE)
                 signature_tag.set(ATTR_TYPE, signature.key_type)
                 signature_tag.set(ATTR_VALUE, signature.signature)
 
-        return et.tostring(response, 'unicode')
+        return et.tostring(response, ENCODING_UNICODE)
+
+    def parse_response_get(self, response_xml):
+        tag_root = et.fromstring(response_xml)
+        tag_status = tag_root[0]
+        if 'success' == tag_status.text:
+            tag_models = tag_root[1]
+            for tag_model in tag_models:
+                tag_content = tag_model[0]
+                tag_id = tag_content[1]
+                try:
+                    GlobalId.from_xml_element(tag_id, existing=True)
+                    # If this post already exists, just continue
+                    # TODO Compare post content and update the post if necessary
+                    pass
+                except GlobalId.DoesNotExist:
+                    global_id = GlobalId.from_xml_element(tag_id)
+
+                    title = tag_content.find(TAG_TITLE).text
+                    text = tag_content.find(TAG_TEXT).text
+                    # TODO Check that the replied posts are already present
+                    # before adding new ones
+
+                    # TODO Pub time, thread, tags
+
+                    post = Post.objects.create(title=title, text=text)
+        else:
+            # TODO Throw an exception?
+            pass
 
 
 class Post(models.Model, Viewable):
     """A post is a message."""
 
     objects = PostManager()
 
     class Meta:
         app_label = APP_LABEL_BOARDS
         ordering = ('id',)
 
     title = models.CharField(max_length=TITLE_MAX_LENGTH)
     pub_time = models.DateTimeField()
     text = MarkupField(default_markup_type=DEFAULT_MARKUP_TYPE,
                        escape_html=False)
 
     images = models.ManyToManyField(PostImage, null=True, blank=True,
                                     related_name='ip+', db_index=True)
 
     poster_ip = models.GenericIPAddressField()
     poster_user_agent = models.TextField()
 
     thread_new = models.ForeignKey('Thread', null=True, default=None,
                                    db_index=True)
     last_edit_time = models.DateTimeField()
 
     # Replies to the post
     referenced_posts = models.ManyToManyField('Post', symmetrical=False,
                                               null=True,
                                               blank=True, related_name='rfp+',
                                               db_index=True)
 
     # Replies map. This is built from the referenced posts list to speed up
     # page loading (no need to get all the referenced posts from the database).
     refmap = models.TextField(null=True, blank=True)
 
     # Global ID with author key. If the message was downloaded from another
     # server, this indicates the server.
     global_id = models.OneToOneField('GlobalId', null=True, blank=True)
 
     # One post can be signed by many nodes that give their trust to it
     signature = models.ManyToManyField('Signature', null=True, blank=True)
 
     def __unicode__(self):
         return '#' + str(self.id) + ' ' + self.title + ' (' + \
                self.text.raw[:50] + ')'
 
     def get_title(self):
-        """
-        Gets original post title or part of its text.
-        """
-
-        title = self.title
-        if not title:
-            title = self.text.rendered
-
-        return title
+        return self.title
 
     def build_refmap(self):
         """
         Builds a replies map string from replies list. This is a cache to stop
         the server from recalculating the map on every post show.
         """
         map_string = ''
 
         first = True
         for refpost in self.referenced_posts.all():
             if not first:
                 map_string += ', '
             map_string += '<a href="%s">&gt;&gt;%s</a>' % (refpost.get_url(),
                                                            refpost.id)
             first = False
 
         self.refmap = map_string
 
     def get_sorted_referenced_posts(self):
         return self.refmap
 
     def is_referenced(self):
         return len(self.refmap) > 0
 
     def is_opening(self):
         """
         Checks if this is an opening post or just a reply.
         """
 
         return self.get_thread().get_opening_post_id() == self.id
 
     @transaction.atomic
     def add_tag(self, tag):
         edit_time = timezone.now()
 
         thread = self.get_thread()
         thread.add_tag(tag)
         self.last_edit_time = edit_time
         self.save(update_fields=['last_edit_time'])
 
         thread.last_edit_time = edit_time
         thread.save(update_fields=['last_edit_time'])
 
     @transaction.atomic
     def remove_tag(self, tag):
         edit_time = timezone.now()
 
         thread = self.get_thread()
         thread.remove_tag(tag)
         self.last_edit_time = edit_time
         self.save(update_fields=['last_edit_time'])
 
         thread.last_edit_time = edit_time
         thread.save(update_fields=['last_edit_time'])
 
     def get_url(self, thread=None):
         """
         Gets full url to the post.
         """
 
         cache_key = CACHE_KEY_POST_URL + str(self.id)
         link = cache.get(cache_key)
 
         if not link:
             if not thread:
                 thread = self.get_thread()
 
             opening_id = thread.get_opening_post_id()
 
             if self.id != opening_id:
                 link = reverse('thread', kwargs={
                     'post_id': opening_id}) + '#' + str(self.id)
             else:
                 link = reverse('thread', kwargs={'post_id': self.id})
 
             cache.set(cache_key, link)
 
         return link
 
     def get_thread(self):
         """
         Gets post's thread.
         """
 
         return self.thread_new
 
     def get_referenced_posts(self):
         return self.referenced_posts.only('id', 'thread_new')
 
     def get_text(self):
         return self.text
 
     def get_view(self, moderator=False, need_open_link=False,
                  truncated=False, *args, **kwargs):
         if 'is_opening' in kwargs:
             is_opening = kwargs['is_opening']
         else:
             is_opening = self.is_opening()
 
         if 'thread' in kwargs:
             thread = kwargs['thread']
         else:
             thread = self.get_thread()
 
         if 'can_bump' in kwargs:
             can_bump = kwargs['can_bump']
         else:
             can_bump = thread.can_bump()
 
         if is_opening:
             opening_post_id = self.id
         else:
             opening_post_id = thread.get_opening_post_id()
 
         return render_to_string('boards/post.html', {
             'post': self,
             'moderator': moderator,
             'is_opening': is_opening,
             'thread': thread,
             'bumpable': can_bump,
             'need_open_link': need_open_link,
             'truncated': truncated,
             'opening_post_id': opening_post_id,
         })
 
     def get_first_image(self):
         return self.images.earliest('id')
 
     def delete(self, using=None):
         """
         Deletes all post images and the post itself.
         """
 
         self.images.all().delete()
         self.signature.all().delete()
         if self.global_id:
             self.global_id.delete()
 
         super(Post, self).delete(using)
 
     def set_global_id(self, key_pair=None):
         """
         Sets global id based on the given key pair. If no key pair is given,
         default one is used.
         """
 
         if key_pair:
             key = key_pair
         else:
             try:
                 key = KeyPair.objects.get(primary=True)
             except KeyPair.DoesNotExist:
                 # Do not update the global id because there is no key defined
                 return
         global_id = GlobalId(key_type=key.key_type,
                              key=key.public_key,
                              local_id = self.id)
         global_id.save()
 
         self.global_id = global_id
 
         self.save(update_fields=['global_id'])
 
     def get_pub_time_epoch(self):
         return utils.datetime_to_epoch(self.pub_time)
 
-    def get_edit_time_epoch(self):
-        return utils.datetime_to_epoch(self.last_edit_time)
+    def get_replied_ids(self):
+        """
+        Gets ID list of the posts that this post replies.
+        """
 
-    def get_replied_ids(self):
-        return re.findall(REGEX_REPLY, self.text.raw)
+        local_replied = REGEX_REPLY.findall(self.text.raw)
+        global_replied = []
+        # TODO Similar code is used in mdx_neboard, maybe it can be extracted
+        # into a method?
+        for match in REGEX_GLOBAL_REPLY.findall(self.text.raw):
+            key_type = match[0]
+            key = match[1]
+            local_id = match[2]
+
+            try:
+                global_id = GlobalId.objects.get(key_type=key_type,
+                    key=key, local_id=local_id)
+                for post in Post.objects.filter(global_id=global_id).only('id'):
+                    global_replied.append(post.id)
+            except GlobalId.DoesNotExist:
+                pass
+        return local_replied + global_replied

boards/models/signature.py

@@ -1,75 +1,108 b''
 import xml.etree.ElementTree as et
 from django.db import models
 
 
+TAG_MODEL = 'model'
+TAG_REQUEST = 'request'
+TAG_ID = 'id'
+
+TYPE_GET = 'get'
+
+ATTR_VERSION = 'version'
+ATTR_TYPE = 'type'
+ATTR_NAME = 'name'
+
 ATTR_KEY = 'key'
 ATTR_KEY_TYPE = 'type'
 ATTR_LOCAL_ID = 'local-id'
 
 
+class GlobalIdManager(models.Manager):
+    def generate_request_get(self, global_id_list: list):
+        """
+        Form a get request from a list of ModelId objects.
+        """
+
+        request = et.Element(TAG_REQUEST)
+        request.set(ATTR_TYPE, TYPE_GET)
+        request.set(ATTR_VERSION, '1.0')
+
+        model = et.SubElement(request, TAG_MODEL)
+        model.set(ATTR_VERSION, '1.0')
+        model.set(ATTR_NAME, 'post')
+
+        for global_id in global_id_list:
+            tag_id = et.SubElement(model, TAG_ID)
+            global_id.to_xml_element(tag_id)
+
+        return et.tostring(request, 'unicode')
+
+
 class GlobalId(models.Model):
     class Meta:
         app_label = 'boards'
 
+    objects = GlobalIdManager()
+
     def __init__(self, *args, **kwargs):
         models.Model.__init__(self, *args, **kwargs)
 
         if 'key' in kwargs and 'key_type' in kwargs and 'local_id' in kwargs:
             self.key = kwargs['key']
             self.key_type = kwargs['key_type']
             self.local_id = kwargs['local_id']
 
     key = models.TextField()
     key_type = models.TextField()
     local_id = models.IntegerField()
 
     def __str__(self):
-        return '[%s][%s][%d]' % (self.key_type, self.key, self.local_id)
+        return '%s::%s::%d' % (self.key_type, self.key, self.local_id)
 
     def to_xml_element(self, element: et.Element):
         """
         Exports global id to an XML element.
         """
 
         element.set(ATTR_KEY, self.key)
         element.set(ATTR_KEY_TYPE, self.key_type)
         element.set(ATTR_LOCAL_ID, str(self.local_id))
 
     @staticmethod
     def from_xml_element(element: et.Element, existing=False):
         """
         Parses XML id tag and gets global id from it.
 
         Arguments:
         element -- the XML 'id' element
         existing -- if this is False, a new instance of GlobalId will be
          created. Otherwise, we will search for an existing GlobalId instance
          and throw DoesNotExist if there isn't one.
         """
 
         if existing:
             return GlobalId.objects.get(key=element.get(ATTR_KEY),
                                         key_type=element.get(ATTR_KEY_TYPE),
                                         local_id=int(element.get(
                                             ATTR_LOCAL_ID)))
         else:
             return GlobalId(key=element.get(ATTR_KEY),
                             key_type=element.get(ATTR_KEY_TYPE),
                             local_id=int(element.get(ATTR_LOCAL_ID)))
 
 
 class Signature(models.Model):
     class Meta:
         app_label = 'boards'
 
     def __init__(self, *args, **kwargs):
         models.Model.__init__(self, *args, **kwargs)
 
         if 'key' in kwargs and 'key_type' in kwargs and 'signature' in kwargs:
             self.key_type = kwargs['key_type']
             self.key = kwargs['key']
             self.signature = kwargs['signature']
 
     key_type = models.TextField()
     key = models.TextField()
     signature = models.TextField()

boards/models/sync_key.py

@@ -1,61 +1,61 b''
 import base64
 from ecdsa import SigningKey, VerifyingKey, BadSignatureError
 from django.db import models
 
 TYPE_ECDSA = 'ecdsa'
 
 APP_LABEL_BOARDS = 'boards'
 
 
 class KeyPairManager(models.Manager):
     def generate_key(self, key_type=TYPE_ECDSA, primary=False):
         if primary and self.filter(primary=True).exists():
             raise Exception('There can be only one primary key')
 
         if key_type == TYPE_ECDSA:
             private = SigningKey.generate()
             public = private.get_verifying_key()
 
             private_key_str = base64.b64encode(private.to_string()).decode()
             public_key_str = base64.b64encode(public.to_string()).decode()
 
             return self.create(public_key=public_key_str,
                                private_key=private_key_str,
                                key_type=TYPE_ECDSA, primary=primary)
         else:
             raise Exception('Key type not supported')
 
     def verify(self, public_key_str, string, signature, key_type=TYPE_ECDSA):
         if key_type == TYPE_ECDSA:
             public = VerifyingKey.from_string(base64.b64decode(public_key_str))
             signature_byte = base64.b64decode(signature)
             try:
                 return public.verify(signature_byte, string.encode())
             except BadSignatureError:
                 return False
         else:
             raise Exception('Key type not supported')
 
     def has_primary(self):
         return self.filter(primary=True).exists()
 
 
 class KeyPair(models.Model):
     class Meta:
         app_label = APP_LABEL_BOARDS
 
     objects = KeyPairManager()
 
     public_key = models.TextField()
     private_key = models.TextField()
     key_type = models.TextField()
     primary = models.BooleanField(default=False)
 
     def __str__(self):
-        return '[%s][%s]' % (self.key_type, self.public_key)
+        return '%s::%s' % (self.key_type, self.public_key)
 
     def sign(self, string):
         private = SigningKey.from_string(base64.b64decode(
             self.private_key.encode()))
         signature_byte = private.sign_deterministic(string.encode())
         return base64.b64encode(signature_byte).decode()

boards/tests/test_keys.py

@@ -1,85 +1,85 b''
 from base64 import b64encode
 import logging
 
 from django.test import TestCase
 from boards.models import KeyPair, GlobalId, Post
 
 
 logger = logging.getLogger(__name__)
 
 
 class KeyTest(TestCase):
     def test_create_key(self):
         key = KeyPair.objects.generate_key('ecdsa')
 
         self.assertIsNotNone(key, 'The key was not created.')
 
     def test_validation(self):
         key = KeyPair.objects.generate_key(key_type='ecdsa')
         message = 'msg'
         signature = key.sign(message)
         valid = KeyPair.objects.verify(key.public_key, message, signature,
                                        key_type='ecdsa')
 
         self.assertTrue(valid, 'Message verification failed.')
 
     def test_primary_constraint(self):
         KeyPair.objects.generate_key(key_type='ecdsa', primary=True)
 
         with self.assertRaises(Exception):
             KeyPair.objects.generate_key(key_type='ecdsa', primary=True)
 
     def test_model_id_save(self):
         model_id = GlobalId(key_type='test', key='test key', local_id='1')
         model_id.save()
 
     def test_request_get(self):
         post = self._create_post_with_key()
 
-        request = Post.objects.generate_request_get([post])
+        request = GlobalId.objects.generate_request_get([post.global_id])
         logger.debug(request)
 
         key = KeyPair.objects.get(primary=True)
         self.assertTrue('<request type="get" version="1.0">'
                         '<model name="post" version="1.0">'
                         '<id key="%s" local-id="1" type="%s" />'
                         '</model>'
                         '</request>' % (
                             key.public_key,
                             key.key_type,
                         ) in request,
                         'Wrong XML generated for the GET request.')
 
     def test_response_get(self):
         post = self._create_post_with_key()
         reply_post = Post.objects.create_post(title='test_title',
                                               text='[post]%d[/post]' % post.id,
                                               thread=post.get_thread())
 
         response = Post.objects.generate_response_get([reply_post])
         logger.debug(response)
 
         key = KeyPair.objects.get(primary=True)
         self.assertTrue('<status>success</status>'
                         '<models>'
                         '<model name="post">'
                         '<content>'
                         '<id key="%s" local-id="%d" type="%s" />'
                         '<title>test_title</title>'
                         '<text>[post]%d[/post]</text>'
                         '<thread>%d</thread>'
                         '<pub-time>%s</pub-time>'
                         '</content>' % (
                             key.public_key,
                             reply_post.id,
                             key.key_type,
                             post.id,
                             post.id,
                             str(reply_post.get_pub_time_epoch()),
                         ) in response,
                         'Wrong XML generated for the GET response.')
 
     def _create_post_with_key(self):
         KeyPair.objects.generate_key(primary=True)
 
         return Post.objects.create_post(title='test_title', text='test_text')

boards/tests/test_post.py

@@ -1,112 +1,142 b''
 from django.core.paginator import Paginator
 from django.test import TestCase
 from boards import settings
-from boards.models import Tag, Post, Thread
+from boards.models import Tag, Post, Thread, KeyPair
 
 
 class PostTests(TestCase):
 
     def _create_post(self):
         tag = Tag.objects.create(name='test_tag')
         return Post.objects.create_post(title='title', text='text',
                                         tags=[tag])
 
     def test_post_add(self):
         """Test adding post"""
 
         post = self._create_post()
 
         self.assertIsNotNone(post, 'No post was created.')
         self.assertEqual('test_tag', post.get_thread().tags.all()[0].name,
                          'No tags were added to the post.')
 
     def test_delete_post(self):
         """Test post deletion"""
 
         post = self._create_post()
         post_id = post.id
 
         Post.objects.delete_post(post)
 
         self.assertFalse(Post.objects.filter(id=post_id).exists())
 
     def test_delete_thread(self):
         """Test thread deletion"""
 
         opening_post = self._create_post()
         thread = opening_post.get_thread()
         reply = Post.objects.create_post("", "", thread=thread)
 
         thread.delete()
 
         self.assertFalse(Post.objects.filter(id=reply.id).exists())
 
     def test_post_to_thread(self):
         """Test adding post to a thread"""
 
         op = self._create_post()
         post = Post.objects.create_post("", "", thread=op.get_thread())
 
         self.assertIsNotNone(post, 'Reply to thread wasn\'t created')
         self.assertEqual(op.get_thread().last_edit_time, post.pub_time,
                          'Post\'s create time doesn\'t match thread last edit'
                          ' time')
 
     def test_delete_posts_by_ip(self):
         """Test deleting posts with the given ip"""
 
         post = self._create_post()
         post_id = post.id
 
         Post.objects.delete_posts_by_ip('0.0.0.0')
 
         self.assertFalse(Post.objects.filter(id=post_id).exists())
 
     def test_get_thread(self):
         """Test getting all posts of a thread"""
 
         opening_post = self._create_post()
 
         for i in range(2):
             Post.objects.create_post('title', 'text',
                                      thread=opening_post.get_thread())
 
         thread = opening_post.get_thread()
 
         self.assertEqual(3, thread.replies.count())
 
     def test_create_post_with_tag(self):
         """Test adding tag to post"""
 
         tag = Tag.objects.create(name='test_tag')
         post = Post.objects.create_post(title='title', text='text', tags=[tag])
 
         thread = post.get_thread()
         self.assertIsNotNone(post, 'Post not created')
         self.assertTrue(tag in thread.tags.all(), 'Tag not added to thread')
         self.assertTrue(thread in tag.threads.all(), 'Thread not added to tag')
 
     def test_thread_max_count(self):
         """Test deletion of old posts when the max thread count is reached"""
 
         for i in range(settings.MAX_THREAD_COUNT + 1):
             self._create_post()
 
         self.assertEqual(settings.MAX_THREAD_COUNT,
                          len(Thread.objects.filter(archived=False)))
 
     def test_pages(self):
         """Test that the thread list is properly split into pages"""
 
         for i in range(settings.MAX_THREAD_COUNT):
             self._create_post()
 
         all_threads = Thread.objects.filter(archived=False)
 
         paginator = Paginator(Thread.objects.filter(archived=False),
                               settings.THREADS_PER_PAGE)
         posts_in_second_page = paginator.page(2).object_list
         first_post = posts_in_second_page[0]
 
         self.assertEqual(all_threads[settings.THREADS_PER_PAGE].id,
-                         first_post.id)
  No newline at end of file
+                         first_post.id)
+
+    def test_reflinks(self):
+        """
+        Tests that reflinks are parsed within post and connecting replies
+        to the replied posts.
+
+        Local reflink example: [post]123[/post]
+        Global reflink example: [post]key_type::key::123[/post]
+        """
+
+        key = KeyPair.objects.generate_key(primary=True)
+
+        tag = Tag.objects.create(name='test_tag')
+
+        post = Post.objects.create_post(title='', text='', tags=[tag])
+        post_local_reflink = Post.objects.create_post(title='',
+                text='[post]%d[/post]' % post.id, thread=post.get_thread())
+
+        self.assertTrue(post_local_reflink in post.referenced_posts.all(),
+            'Local reflink not connecting posts.')
+
+        post_global_reflink = Post.objects.create_post(title='',
+                text='[post]%s::%s::%d[/post]' % (
+                    post.global_id.key_type, post.global_id.key, post.id),
+                thread=post.get_thread())
+
+        self.assertTrue(post_global_reflink in post.referenced_posts.all(),
+            'Global reflink not connecting posts.')
+
+        # TODO Check that links are parsed into the rendered text

requirements.txt

@@ -1,9 +1,10 b''
+httplib2
 simplejson
 south>=0.8.4
 haystack
 pillow
 django>=1.6
 django_cleanup
 django-markupfield
 bbcode
 ecdsa

todo.txt

@@ -1,25 +1,26 b''
 = Features =
 * Tree view (JS)
 * Adding tags to images filename
 * Federative network for s2s communication
 * XMPP gate
 * Bitmessage gate
 * Notification engine
 * Group tags by first letter in all tags list
 * [JS] Character counter in the post field
 * Statistics module. Count views (optional, may result in bad
  performance), posts per day/week/month, IPs
 * Ban confirmation page (or alert) with reason
 * Post deletion confirmation page (or alert)
 * Get thread graph image using pygraphviz
 * Subscribing to tag via AJAX
 * Add buttons to insert a named link or a named quote to the markup panel
 * Add support for "attention posts" that are shown in the header"
+* Use absolute post reflinks in the raw text
 
 = Bugs =
 * Search sort order is confusing
 
 = Testing =
 * Make tests for every view
 * Make tests for every model
 * Make tests for every form