Merged the decentral branch into default branch

boards/apps.py

@@ -0,0 +1,11 b''
+from django.apps import AppConfig
+
+
+class BoardsAppConfig(AppConfig):
+    name = 'boards'
+    verbose_name = 'Boards'
+
+    def ready(self):
+        super().ready()
+
+        import boards.signals
  No newline at end of file

boards/management/commands/generate_keypair.py

@@ -0,0 +1,21 b''
+__author__ = 'neko259'
+
+
+from django.core.management import BaseCommand
+from django.db import transaction
+
+from boards.models import KeyPair, Post
+
+
+class Command(BaseCommand):
+    help = 'Generates the new keypair. The first one will be primary.'
+
+    @transaction.atomic
+    def handle(self, *args, **options):
+        first_key = not KeyPair.objects.has_primary()
+        key = KeyPair.objects.generate_key(
+            primary=first_key)
+        print(key)
+
+        for post in Post.objects.filter(global_id=None):
+            post.set_global_id()

boards/management/commands/sync_with_server.py

@@ -0,0 +1,80 b''
+import re
+import xml.etree.ElementTree as ET
+
+import httplib2
+from django.core.management import BaseCommand
+
+from boards.models import GlobalId
+from boards.models.post.sync import SyncManager
+
+__author__ = 'neko259'
+
+
+REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)')
+
+
+class Command(BaseCommand):
+    help = 'Send a sync or get request to the server.'
+
+    def add_arguments(self, parser):
+        parser.add_argument('url', type=str, help='Server root url')
+        parser.add_argument('--global-id', type=str, default='',
+                            help='Post global ID')
+
+    def handle(self, *args, **options):
+        url = options.get('url')
+
+        pull_url = url + 'api/sync/pull/'
+        get_url = url + 'api/sync/get/'
+        file_url = url[:-1]
+
+        global_id_str = options.get('global_id')
+        if global_id_str:
+            match = REGEX_GLOBAL_ID.match(global_id_str)
+            if match:
+                key_type = match.group(1)
+                key = match.group(2)
+                local_id = match.group(3)
+
+                global_id = GlobalId(key_type=key_type, key=key,
+                                     local_id=local_id)
+
+                xml = GlobalId.objects.generate_request_get([global_id])
+                # body = urllib.parse.urlencode(data)
+                h = httplib2.Http()
+                response, content = h.request(get_url, method="POST", body=xml)
+
+                SyncManager.parse_response_get(content, file_url)
+            else:
+                raise Exception('Invalid global ID')
+        else:
+            h = httplib2.Http()
+            xml = GlobalId.objects.generate_request_pull()
+            response, content = h.request(pull_url, method="POST", body=xml)
+
+            print(content.decode() + '\n')
+
+            root = ET.fromstring(content)
+            status = root.findall('status')[0].text
+            if status == 'success':
+                ids_to_sync = list()
+
+                models = root.findall('models')[0]
+                for model in models:
+                    global_id, exists = GlobalId.from_xml_element(model)
+                    if not exists:
+                        print(global_id)
+                        ids_to_sync.append(global_id)
+                print()
+
+                if len(ids_to_sync) > 0:
+                    xml = GlobalId.objects.generate_request_get(ids_to_sync)
+                    # body = urllib.parse.urlencode(data)
+                    h = httplib2.Http()
+                    response, content = h.request(get_url, method="POST", body=xml)
+
+                    SyncManager.parse_response_get(content, file_url)
+                else:
+                    print('Nothing to get, everything synced')
+            else:
+                raise Exception('Invalid response status')

boards/migrations/0026_auto_20150830_2006.py

@@ -0,0 +1,48 b''
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('boards', '0025_auto_20150825_2049'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='GlobalId',
+            fields=[
+                ('id', models.AutoField(serialize=False, verbose_name='ID', primary_key=True, auto_created=True)),
+                ('key', models.TextField()),
+                ('key_type', models.TextField()),
+                ('local_id', models.IntegerField()),
+            ],
+        ),
+        migrations.CreateModel(
+            name='KeyPair',
+            fields=[
+                ('id', models.AutoField(serialize=False, verbose_name='ID', primary_key=True, auto_created=True)),
+                ('public_key', models.TextField()),
+                ('private_key', models.TextField()),
+                ('key_type', models.TextField()),
+                ('primary', models.BooleanField(default=False)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Signature',
+            fields=[
+                ('id', models.AutoField(serialize=False, verbose_name='ID', primary_key=True, auto_created=True)),
+                ('key_type', models.TextField()),
+                ('key', models.TextField()),
+                ('signature', models.TextField()),
+                ('global_id', models.ForeignKey(to='boards.GlobalId')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='post',
+            name='global_id',
+            field=models.OneToOneField(to='boards.GlobalId', null=True, blank=True),
+        ),
+    ]

boards/migrations/0031_merge.py

@@ -0,0 +1,15 b''
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('boards', '0030_auto_20150929_1816'),
+        ('boards', '0026_auto_20150830_2006'),
+    ]
+
+    operations = [
+    ]

boards/migrations/0036_merge.py

@@ -0,0 +1,15 b''
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('boards', '0031_merge'),
+        ('boards', '0035_auto_20151021_1346'),
+    ]
+
+    operations = [
+    ]

boards/migrations/0041_merge.py

@@ -0,0 +1,15 b''
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('boards', '0040_thread_monochrome'),
+        ('boards', '0036_merge'),
+    ]
+
+    operations = [
+    ]

boards/migrations/0043_merge.py

@@ -0,0 +1,16 b''
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.5 on 2016-04-29 15:52
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('boards', '0041_merge'),
+        ('boards', '0042_auto_20160422_1053'),
+    ]
+
+    operations = [
+    ]

boards/migrations/0044_globalid_content.py

@@ -0,0 +1,20 b''
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.5 on 2016-05-04 15:36
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('boards', '0043_merge'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='globalid',
+            name='content',
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

boards/models/post/sync.py

@@ -0,0 +1,270 b''
+import xml.etree.ElementTree as et
+
+from boards.models.attachment.downloaders import download
+from boards.utils import get_file_mimetype, get_file_hash
+from django.db import transaction
+from boards.models import KeyPair, GlobalId, Signature, Post, Tag
+
+EXCEPTION_NODE = 'Sync node returned an error: {}'
+EXCEPTION_OP = 'Load the OP first'
+EXCEPTION_DOWNLOAD = 'File was not downloaded'
+EXCEPTION_HASH = 'File hash does not match attachment hash'
+EXCEPTION_SIGNATURE = 'Invalid model signature for {}'
+ENCODING_UNICODE = 'unicode'
+
+TAG_MODEL = 'model'
+TAG_REQUEST = 'request'
+TAG_RESPONSE = 'response'
+TAG_ID = 'id'
+TAG_STATUS = 'status'
+TAG_MODELS = 'models'
+TAG_TITLE = 'title'
+TAG_TEXT = 'text'
+TAG_THREAD = 'thread'
+TAG_PUB_TIME = 'pub-time'
+TAG_SIGNATURES = 'signatures'
+TAG_SIGNATURE = 'signature'
+TAG_CONTENT = 'content'
+TAG_ATTACHMENTS = 'attachments'
+TAG_ATTACHMENT = 'attachment'
+TAG_TAGS = 'tags'
+TAG_TAG = 'tag'
+TAG_ATTACHMENT_REFS = 'attachment-refs'
+TAG_ATTACHMENT_REF = 'attachment-ref'
+
+TYPE_GET = 'get'
+
+ATTR_VERSION = 'version'
+ATTR_TYPE = 'type'
+ATTR_NAME = 'name'
+ATTR_VALUE = 'value'
+ATTR_MIMETYPE = 'mimetype'
+ATTR_KEY = 'key'
+ATTR_REF = 'ref'
+ATTR_URL = 'url'
+
+STATUS_SUCCESS = 'success'
+
+
+class SyncException(Exception):
+    pass
+
+
+class SyncManager:
+    @staticmethod
+    def generate_response_get(model_list: list):
+        response = et.Element(TAG_RESPONSE)
+
+        status = et.SubElement(response, TAG_STATUS)
+        status.text = STATUS_SUCCESS
+
+        models = et.SubElement(response, TAG_MODELS)
+
+        for post in model_list:
+            model = et.SubElement(models, TAG_MODEL)
+            model.set(ATTR_NAME, 'post')
+
+            global_id = post.global_id
+
+            images = post.images.all()
+            attachments = post.attachments.all()
+            if global_id.content:
+                model.append(et.fromstring(global_id.content))
+                if len(images) > 0 or len(attachments) > 0:
+                    attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS)
+                    for image in images:
+                        SyncManager._attachment_to_xml(
+                            None, attachment_refs, image.image.file,
+                            image.hash, image.image.url)
+                    for file in attachments:
+                        SyncManager._attachment_to_xml(
+                            None, attachment_refs, file.file.file,
+                            file.hash, file.file.url)
+            else:
+                content_tag = et.SubElement(model, TAG_CONTENT)
+
+                tag_id = et.SubElement(content_tag, TAG_ID)
+                global_id.to_xml_element(tag_id)
+
+                title = et.SubElement(content_tag, TAG_TITLE)
+                title.text = post.title
+
+                text = et.SubElement(content_tag, TAG_TEXT)
+                text.text = post.get_sync_text()
+
+                thread = post.get_thread()
+                if post.is_opening():
+                    tag_tags = et.SubElement(content_tag, TAG_TAGS)
+                    for tag in thread.get_tags():
+                        tag_tag = et.SubElement(tag_tags, TAG_TAG)
+                        tag_tag.text = tag.name
+                else:
+                    tag_thread = et.SubElement(content_tag, TAG_THREAD)
+                    thread_id = et.SubElement(tag_thread, TAG_ID)
+                    thread.get_opening_post().global_id.to_xml_element(thread_id)
+
+                pub_time = et.SubElement(content_tag, TAG_PUB_TIME)
+                pub_time.text = str(post.get_pub_time_str())
+
+                if len(images) > 0 or len(attachments) > 0:
+                    attachments_tag = et.SubElement(content_tag, TAG_ATTACHMENTS)
+                    attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS)
+
+                    for image in images:
+                        SyncManager._attachment_to_xml(
+                            attachments_tag, attachment_refs, image.image.file,
+                            image.hash, image.image.url)
+                    for file in attachments:
+                        SyncManager._attachment_to_xml(
+                            attachments_tag, attachment_refs, file.file.file,
+                            file.hash, file.file.url)
+
+                global_id.content = et.tostring(content_tag, ENCODING_UNICODE)
+                global_id.save()
+
+            signatures_tag = et.SubElement(model, TAG_SIGNATURES)
+            post_signatures = global_id.signature_set.all()
+            if post_signatures:
+                signatures = post_signatures
+            else:
+                key = KeyPair.objects.get(public_key=global_id.key)
+                signature = Signature(
+                    key_type=key.key_type,
+                    key=key.public_key,
+                    signature=key.sign(global_id.content),
+                    global_id=global_id,
+                )
+                signature.save()
+                signatures = [signature]
+            for signature in signatures:
+                signature_tag = et.SubElement(signatures_tag, TAG_SIGNATURE)
+                signature_tag.set(ATTR_TYPE, signature.key_type)
+                signature_tag.set(ATTR_VALUE, signature.signature)
+                signature_tag.set(ATTR_KEY, signature.key)
+
+        return et.tostring(response, ENCODING_UNICODE)
+
+    @staticmethod
+    @transaction.atomic
+    def parse_response_get(response_xml, hostname):
+        tag_root = et.fromstring(response_xml)
+        tag_status = tag_root.find(TAG_STATUS)
+        if STATUS_SUCCESS == tag_status.text:
+            tag_models = tag_root.find(TAG_MODELS)
+            for tag_model in tag_models:
+                tag_content = tag_model.find(TAG_CONTENT)
+
+                content_str = et.tostring(tag_content, ENCODING_UNICODE)
+                signatures = SyncManager._verify_model(content_str, tag_model)
+
+                tag_id = tag_content.find(TAG_ID)
+                global_id, exists = GlobalId.from_xml_element(tag_id)
+
+                if exists:
+                    print('Post with same ID already exists')
+                else:
+                    global_id.content = content_str
+                    global_id.save()
+                    for signature in signatures:
+                        signature.global_id = global_id
+                        signature.save()
+
+                    title = tag_content.find(TAG_TITLE).text or ''
+                    text = tag_content.find(TAG_TEXT).text or ''
+                    pub_time = tag_content.find(TAG_PUB_TIME).text
+
+                    thread = tag_content.find(TAG_THREAD)
+                    tags = []
+                    if thread:
+                        thread_id = thread.find(TAG_ID)
+                        op_global_id, exists = GlobalId.from_xml_element(thread_id)
+                        if exists:
+                            opening_post = Post.objects.get(global_id=op_global_id)
+                        else:
+                            raise SyncException(EXCEPTION_OP)
+                    else:
+                        opening_post = None
+                        tag_tags = tag_content.find(TAG_TAGS)
+                        for tag_tag in tag_tags:
+                            tag, created = Tag.objects.get_or_create(
+                                name=tag_tag.text)
+                            tags.append(tag)
+
+                    # TODO Check that the replied posts are already present
+                    # before adding new ones
+
+                    files = []
+                    tag_attachments = tag_content.find(TAG_ATTACHMENTS) or list()
+                    tag_refs = tag_model.find(TAG_ATTACHMENT_REFS)
+                    for attachment in tag_attachments:
+                        tag_ref = tag_refs.find("{}[@ref='{}']".format(
+                            TAG_ATTACHMENT_REF, attachment.text))
+                        url = tag_ref.get(ATTR_URL)
+                        attached_file = download(hostname + url)
+                        if attached_file is None:
+                            raise SyncException(EXCEPTION_DOWNLOAD)
+
+                        hash = get_file_hash(attached_file)
+                        if hash != attachment.text:
+                            raise SyncException(EXCEPTION_HASH)
+
+                        files.append(attached_file)
+
+                    Post.objects.import_post(
+                        title=title, text=text, pub_time=pub_time,
+                        opening_post=opening_post, tags=tags,
+                        global_id=global_id, files=files)
+        else:
+            raise SyncException(EXCEPTION_NODE.format(tag_status.text))
+
+    @staticmethod
+    def generate_response_pull():
+        response = et.Element(TAG_RESPONSE)
+
+        status = et.SubElement(response, TAG_STATUS)
+        status.text = STATUS_SUCCESS
+
+        models = et.SubElement(response, TAG_MODELS)
+
+        for post in Post.objects.all():
+            tag_id = et.SubElement(models, TAG_ID)
+            post.global_id.to_xml_element(tag_id)
+
+        return et.tostring(response, ENCODING_UNICODE)
+
+    @staticmethod
+    def _verify_model(content_str, tag_model):
+        """
+        Verifies all signatures for a single model.
+        """
+
+        signatures = []
+
+        tag_signatures = tag_model.find(TAG_SIGNATURES)
+        for tag_signature in tag_signatures:
+            signature_type = tag_signature.get(ATTR_TYPE)
+            signature_value = tag_signature.get(ATTR_VALUE)
+            signature_key = tag_signature.get(ATTR_KEY)
+
+            signature = Signature(key_type=signature_type,
+                                  key=signature_key,
+                                  signature=signature_value)
+
+            if not KeyPair.objects.verify(signature, content_str):
+                raise SyncException(EXCEPTION_SIGNATURE.format(content_str))
+
+            signatures.append(signature)
+
+        return signatures
+
+    @staticmethod
+    def _attachment_to_xml(tag_attachments, tag_refs, file, hash, url):
+        if tag_attachments is not None:
+            mimetype = get_file_mimetype(file)
+            attachment = et.SubElement(tag_attachments, TAG_ATTACHMENT)
+            attachment.set(ATTR_MIMETYPE, mimetype)
+            attachment.text = hash
+
+        attachment_ref = et.SubElement(tag_refs, TAG_ATTACHMENT_REF)
+        attachment_ref.set(ATTR_REF, hash)
+        attachment_ref.set(ATTR_URL, url)

boards/models/signature.py

@@ -0,0 +1,144 b''
+import xml.etree.ElementTree as et
+from django.db import models
+
+
+TAG_MODEL = 'model'
+TAG_REQUEST = 'request'
+TAG_ID = 'id'
+
+TYPE_GET = 'get'
+TYPE_PULL = 'pull'
+
+ATTR_VERSION = 'version'
+ATTR_TYPE = 'type'
+ATTR_NAME = 'name'
+
+ATTR_KEY = 'key'
+ATTR_KEY_TYPE = 'type'
+ATTR_LOCAL_ID = 'local-id'
+
+
+class GlobalIdManager(models.Manager):
+    def generate_request_get(self, global_id_list: list):
+        """
+        Form a get request from a list of ModelId objects.
+        """
+
+        request = et.Element(TAG_REQUEST)
+        request.set(ATTR_TYPE, TYPE_GET)
+        request.set(ATTR_VERSION, '1.0')
+
+        model = et.SubElement(request, TAG_MODEL)
+        model.set(ATTR_VERSION, '1.0')
+        model.set(ATTR_NAME, 'post')
+
+        for global_id in global_id_list:
+            tag_id = et.SubElement(model, TAG_ID)
+            global_id.to_xml_element(tag_id)
+
+        return et.tostring(request, 'unicode')
+
+    def generate_request_pull(self):
+        """
+        Form a pull request from a list of ModelId objects.
+        """
+
+        request = et.Element(TAG_REQUEST)
+        request.set(ATTR_TYPE, TYPE_PULL)
+        request.set(ATTR_VERSION, '1.0')
+
+        model = et.SubElement(request, TAG_MODEL)
+        model.set(ATTR_VERSION, '1.0')
+        model.set(ATTR_NAME, 'post')
+
+        return et.tostring(request, 'unicode')
+
+    def global_id_exists(self, global_id):
+        """
+        Checks if the same global id already exists in the system.
+        """
+
+        return self.filter(key=global_id.key,
+                           key_type=global_id.key_type,
+                           local_id=global_id.local_id).exists()
+
+
+class GlobalId(models.Model):
+    """
+    Global model ID and cache.
+    Key, key type and local ID make a single global identificator of the model.
+    Content is an XML cache of the model that can be passed along between nodes
+    without manual serialization each time.
+    """
+    class Meta:
+        app_label = 'boards'
+
+    objects = GlobalIdManager()
+
+    def __init__(self, *args, **kwargs):
+        models.Model.__init__(self, *args, **kwargs)
+
+        if 'key' in kwargs and 'key_type' in kwargs and 'local_id' in kwargs:
+            self.key = kwargs['key']
+            self.key_type = kwargs['key_type']
+            self.local_id = kwargs['local_id']
+
+    key = models.TextField()
+    key_type = models.TextField()
+    local_id = models.IntegerField()
+    content = models.TextField(blank=True, null=True)
+
+    def __str__(self):
+        return '%s::%s::%d' % (self.key_type, self.key, self.local_id)
+
+    def to_xml_element(self, element: et.Element):
+        """
+        Exports global id to an XML element.
+        """
+
+        element.set(ATTR_KEY, self.key)
+        element.set(ATTR_KEY_TYPE, self.key_type)
+        element.set(ATTR_LOCAL_ID, str(self.local_id))
+
+    @staticmethod
+    def from_xml_element(element: et.Element):
+        """
+        Parses XML id tag and gets global id from it.
+
+        Arguments:
+        element -- the XML 'id' element
+
+        Returns:
+        global_id -- id itself
+        exists -- True if the global id was taken from database, False if it
+        did not exist and was created.
+        """
+
+        try:
+            return GlobalId.objects.get(key=element.get(ATTR_KEY),
+                                        key_type=element.get(ATTR_KEY_TYPE),
+                                        local_id=int(element.get(
+                                            ATTR_LOCAL_ID))), True
+        except GlobalId.DoesNotExist:
+            return GlobalId(key=element.get(ATTR_KEY),
+                            key_type=element.get(ATTR_KEY_TYPE),
+                            local_id=int(element.get(ATTR_LOCAL_ID))), False
+
+
+class Signature(models.Model):
+    class Meta:
+        app_label = 'boards'
+
+    def __init__(self, *args, **kwargs):
+        models.Model.__init__(self, *args, **kwargs)
+
+        if 'key' in kwargs and 'key_type' in kwargs and 'signature' in kwargs:
+            self.key_type = kwargs['key_type']
+            self.key = kwargs['key']
+            self.signature = kwargs['signature']
+
+    key_type = models.TextField()
+    key = models.TextField()
+    signature = models.TextField()
+
+    global_id = models.ForeignKey('GlobalId')

boards/models/sync_key.py

@@ -0,0 +1,61 b''
+import base64
+from ecdsa import SigningKey, VerifyingKey, BadSignatureError
+from django.db import models
+
+TYPE_ECDSA = 'ecdsa'
+
+APP_LABEL_BOARDS = 'boards'
+
+
+class KeyPairManager(models.Manager):
+    def generate_key(self, key_type=TYPE_ECDSA, primary=False):
+        if primary and self.filter(primary=True).exists():
+            raise Exception('There can be only one primary key')
+
+        if key_type == TYPE_ECDSA:
+            private = SigningKey.generate()
+            public = private.get_verifying_key()
+
+            private_key_str = base64.b64encode(private.to_string()).decode()
+            public_key_str = base64.b64encode(public.to_string()).decode()
+
+            return self.create(public_key=public_key_str,
+                               private_key=private_key_str,
+                               key_type=TYPE_ECDSA, primary=primary)
+        else:
+            raise Exception('Key type not supported')
+
+    def verify(self, signature, string):
+        if signature.key_type == TYPE_ECDSA:
+            public = VerifyingKey.from_string(base64.b64decode(signature.key))
+            signature_byte = base64.b64decode(signature.signature)
+            try:
+                return public.verify(signature_byte, string.encode())
+            except BadSignatureError:
+                return False
+        else:
+            raise Exception('Key type not supported')
+
+    def has_primary(self):
+        return self.filter(primary=True).exists()
+
+
+class KeyPair(models.Model):
+    class Meta:
+        app_label = APP_LABEL_BOARDS
+
+    objects = KeyPairManager()
+
+    public_key = models.TextField()
+    private_key = models.TextField()
+    key_type = models.TextField()
+    primary = models.BooleanField(default=False)
+
+    def __str__(self):
+        return '%s::%s' % (self.key_type, self.public_key)
+
+    def sign(self, string):
+        private = SigningKey.from_string(base64.b64decode(
+            self.private_key.encode()))
+        signature_byte = private.sign_deterministic(string.encode())
+        return base64.b64encode(signature_byte).decode()

boards/signals.py

@@ -0,0 +1,89 b''
+import re
+from boards.mdx_neboard import get_parser
+
+from boards.models import Post, GlobalId
+from boards.models.post import REGEX_NOTIFICATION
+from boards.models.post import REGEX_REPLY, REGEX_GLOBAL_REPLY
+from boards.models.user import Notification
+from django.db.models.signals import post_save, pre_save, pre_delete, \
+    post_delete
+from django.dispatch import receiver
+from django.utils import timezone
+
+
+@receiver(post_save, sender=Post)
+def connect_replies(instance, **kwargs):
+    for reply_number in re.finditer(REGEX_REPLY, instance.get_raw_text()):
+        post_id = reply_number.group(1)
+
+        try:
+            referenced_post = Post.objects.get(id=post_id)
+
+            if not referenced_post.referenced_posts.filter(
+                    id=instance.id).exists():
+                referenced_post.referenced_posts.add(instance)
+                referenced_post.last_edit_time = instance.pub_time
+                referenced_post.build_refmap()
+                referenced_post.save(update_fields=['refmap', 'last_edit_time'])
+        except Post.DoesNotExist:
+            pass
+
+
+@receiver(post_save, sender=Post)
+def connect_global_replies(instance, **kwargs):
+    for reply_number in re.finditer(REGEX_GLOBAL_REPLY, instance.get_raw_text()):
+        key_type = reply_number.group(1)
+        key = reply_number.group(2)
+        local_id = reply_number.group(3)
+
+        try:
+            global_id = GlobalId.objects.get(key_type=key_type, key=key,
+                    local_id=local_id)
+            referenced_post = Post.objects.get(global_id=global_id)
+            referenced_post.referenced_posts.add(instance)
+            referenced_post.last_edit_time = instance.pub_time
+            referenced_post.build_refmap()
+            referenced_post.save(update_fields=['refmap', 'last_edit_time'])
+        except (GlobalId.DoesNotExist, Post.DoesNotExist):
+            pass
+
+
+@receiver(post_save, sender=Post)
+def connect_notifications(instance, **kwargs):
+    for reply_number in re.finditer(REGEX_NOTIFICATION, instance.get_raw_text()):
+        user_name = reply_number.group(1).lower()
+        Notification.objects.get_or_create(name=user_name, post=instance)
+
+
+@receiver(pre_save, sender=Post)
+def preparse_text(instance, **kwargs):
+    instance._text_rendered = get_parser().parse(instance.get_raw_text())
+
+
+@receiver(pre_delete, sender=Post)
+def delete_images(instance, **kwargs):
+    for image in instance.images.all():
+        image_refs_count = image.post_images.count()
+        if image_refs_count == 1:
+            image.delete()
+
+
+@receiver(pre_delete, sender=Post)
+def delete_attachments(instance, **kwargs):
+    for attachment in instance.attachments.all():
+        attachment_refs_count = attachment.attachment_posts.count()
+        if attachment_refs_count == 1:
+            attachment.delete()
+
+
+@receiver(post_delete, sender=Post)
+def update_thread_on_delete(instance, **kwargs):
+    thread = instance.get_thread()
+    thread.last_edit_time = timezone.now()
+    thread.save()
+
+
+@receiver(post_delete, sender=Post)
+def delete_global_id(instance, **kwargs):
+    if instance.global_id and instance.global_id.id:
+        instance.global_id.delete()

boards/tests/test_keys.py

@@ -0,0 +1,89 b''
+from base64 import b64encode
+import logging
+
+from django.test import TestCase
+from boards.models import KeyPair, GlobalId, Post, Signature
+from boards.models.post.sync import SyncManager
+
+logger = logging.getLogger(__name__)
+
+
+class KeyTest(TestCase):
+    def test_create_key(self):
+        key = KeyPair.objects.generate_key('ecdsa')
+
+        self.assertIsNotNone(key, 'The key was not created.')
+
+    def test_validation(self):
+        key = KeyPair.objects.generate_key(key_type='ecdsa')
+        message = 'msg'
+        signature_value = key.sign(message)
+
+        signature = Signature(key_type='ecdsa', key=key.public_key,
+                              signature=signature_value)
+        valid = KeyPair.objects.verify(signature, message)
+
+        self.assertTrue(valid, 'Message verification failed.')
+
+    def test_primary_constraint(self):
+        KeyPair.objects.generate_key(key_type='ecdsa', primary=True)
+
+        with self.assertRaises(Exception):
+            KeyPair.objects.generate_key(key_type='ecdsa', primary=True)
+
+    def test_model_id_save(self):
+        model_id = GlobalId(key_type='test', key='test key', local_id='1')
+        model_id.save()
+
+    def test_request_get(self):
+        post = self._create_post_with_key()
+
+        request = GlobalId.objects.generate_request_get([post.global_id])
+        logger.debug(request)
+
+        key = KeyPair.objects.get(primary=True)
+        self.assertTrue('<request type="get" version="1.0">'
+                        '<model name="post" version="1.0">'
+                        '<id key="%s" local-id="1" type="%s" />'
+                        '</model>'
+                        '</request>' % (
+                            key.public_key,
+                            key.key_type,
+                        ) in request,
+                        'Wrong XML generated for the GET request.')
+
+    def test_response_get(self):
+        post = self._create_post_with_key()
+        reply_post = Post.objects.create_post(title='test_title',
+                                              text='[post]%d[/post]' % post.id,
+                                              thread=post.get_thread())
+
+        response = SyncManager.generate_response_get([reply_post])
+        logger.debug(response)
+
+        key = KeyPair.objects.get(primary=True)
+        self.assertTrue('<status>success</status>'
+                        '<models>'
+                        '<model name="post">'
+                        '<content>'
+                        '<id key="%s" local-id="%d" type="%s" />'
+                        '<title>test_title</title>'
+                        '<text>[post]%s[/post]</text>'
+                        '<thread><id key="%s" local-id="%d" type="%s" /></thread>'
+                        '<pub-time>%s</pub-time>'
+                        '</content>' % (
+                            key.public_key,
+                            reply_post.id,
+                            key.key_type,
+                            str(post.global_id),
+                            key.public_key,
+                            post.id,
+                            key.key_type,
+                            str(reply_post.get_pub_time_str()),
+                        ) in response,
+                        'Wrong XML generated for the GET response.')
+
+    def _create_post_with_key(self):
+        KeyPair.objects.generate_key(primary=True)
+
+        return Post.objects.create_post(title='test_title', text='test_text')

boards/tests/test_sync.py

@@ -0,0 +1,102 b''
+from boards.models import KeyPair, Post, Tag
+from boards.models.post.sync import SyncManager
+from boards.tests.mocks import MockRequest
+from boards.views.sync import response_get
+
+__author__ = 'neko259'
+
+
+from django.test import TestCase
+
+
+class SyncTest(TestCase):
+    def test_get(self):
+        """
+        Forms a GET request of a post and checks the response.
+        """
+
+        key = KeyPair.objects.generate_key(primary=True)
+        tag = Tag.objects.create(name='tag1')
+        post = Post.objects.create_post(title='test_title',
+                                        text='test_text\rline two',
+                                        tags=[tag])
+
+        request = MockRequest()
+        request.body = (
+            '<request type="get" version="1.0">'
+            '<model name="post" version="1.0">'
+            '<id key="%s" local-id="%d" type="%s" />'
+            '</model>'
+            '</request>' % (post.global_id.key,
+                            post.id,
+                            post.global_id.key_type)
+        )
+
+        response = response_get(request).content.decode()
+        self.assertTrue(
+            '<status>success</status>'
+            '<models>'
+            '<model name="post">'
+            '<content>'
+            '<id key="%s" local-id="%d" type="%s" />'
+            '<title>%s</title>'
+            '<text>%s</text>'
+            '<tags><tag>%s</tag></tags>'
+            '<pub-time>%s</pub-time>'
+            '</content>' % (
+                post.global_id.key,
+                post.global_id.local_id,
+                post.global_id.key_type,
+                post.title,
+                post.get_sync_text(),
+                post.get_thread().get_tags().first().name,
+                post.get_pub_time_str(),
+            ) in response,
+            'Wrong response generated for the GET request.')
+
+        post.delete()
+        key.delete()
+
+        KeyPair.objects.generate_key(primary=True)
+
+        SyncManager.parse_response_get(response, None)
+        self.assertEqual(1, Post.objects.count(),
+                         'Post was not created from XML response.')
+
+        parsed_post = Post.objects.first()
+        self.assertEqual('tag1',
+                         parsed_post.get_thread().get_tags().first().name,
+                         'Invalid tag was parsed.')
+
+        SyncManager.parse_response_get(response, None)
+        self.assertEqual(1, Post.objects.count(),
+                         'The same post was imported twice.')
+
+        self.assertEqual(1, parsed_post.global_id.signature_set.count(),
+                         'Signature was not saved.')
+
+        post = parsed_post
+
+        # Trying to sync the same once more
+        response = response_get(request).content.decode()
+
+        self.assertTrue(
+            '<status>success</status>'
+            '<models>'
+            '<model name="post">'
+            '<content>'
+            '<id key="%s" local-id="%d" type="%s" />'
+            '<title>%s</title>'
+            '<text>%s</text>'
+            '<tags><tag>%s</tag></tags>'
+            '<pub-time>%s</pub-time>'
+            '</content>' % (
+                post.global_id.key,
+                post.global_id.local_id,
+                post.global_id.key_type,
+                post.title,
+                post.get_sync_text(),
+                post.get_thread().get_tags().first().name,
+                post.get_pub_time_str(),
+            ) in response,
+            'Wrong response generated for the GET request.')

boards/views/sync.py

@@ -0,0 +1,62 b''
+import xml.etree.ElementTree as et
+import xml.dom.minidom
+
+from django.http import HttpResponse, Http404
+from boards.models import GlobalId, Post
+from boards.models.post.sync import SyncManager
+
+
+def response_pull(request):
+    request_xml = request.body
+
+    if request_xml is None:
+        return HttpResponse(content='Use the API')
+
+    response_xml = SyncManager.generate_response_pull()
+
+    return HttpResponse(content=response_xml)
+
+
+def response_get(request):
+    """
+    Processes a GET request with post ID list and returns the posts XML list.
+    Request should contain an 'xml' post attribute with the actual request XML.
+    """
+
+    request_xml = request.body
+
+    if request_xml is None:
+        return HttpResponse(content='Use the API')
+
+    posts = []
+
+    root_tag = et.fromstring(request_xml)
+    model_tag = root_tag[0]
+    for id_tag in model_tag:
+        global_id, exists = GlobalId.from_xml_element(id_tag)
+        if exists:
+            posts.append(Post.objects.get(global_id=global_id))
+
+    response_xml = SyncManager.generate_response_get(posts)
+
+    return HttpResponse(content=response_xml)
+
+
+def get_post_sync_data(request, post_id):
+    try:
+        post = Post.objects.get(id=post_id)
+    except Post.DoesNotExist:
+        raise Http404()
+
+    xml_str = SyncManager.generate_response_get([post])
+
+    xml_repr = xml.dom.minidom.parseString(xml_str)
+    xml_repr = xml_repr.toprettyxml()
+
+    content = '=Global ID=\n%s\n\n=XML=\n%s' \
+              % (post.global_id, xml_repr)
+
+    return HttpResponse(
+        content_type='text/plain',
+        content=content,
+    )
  No newline at end of file

development/run.sh

@@ -0,0 +1,3 b''
+#! /usr/bin/env sh
+
+python3 manage.py runserver [::]:8000

development/test.sh

@@ -0,0 +1,3 b''
+#! /usr/bin/env sh
+
+python3 manage.py test

docs/dip-1.markdown

@@ -0,0 +1,203 b''
+# 0 Title #
+
+DIP-1 Common protocol description
+
+# 1 Intro #
+
+This document describes the Data Interchange Protocol (DIP), designed to
+exchange filtered data that can be stored as a graph structure between
+network nodes.
+
+# 2 Purpose #
+
+This protocol will be used to share the models (originally imageboard posts)
+across multiple servers. The main differnce of this protocol is that the node
+can specify what models it wants to get and from whom. The nodes can get
+models from a specific server, or from all except some specific servers. Also
+the models can be filtered by timestamps or tags.
+
+# 3 Protocol description #
+
+The node requests other node's changes list since some time (since epoch if
+this is the start). The other node sends a list of post ids or posts in the
+XML format.
+
+Protocol version is the version of the sync api. Model version is the version
+of data models. If at least one of them is different, the sync cannot be
+performed.
+
+The node signs the data with its keys. The receiving node saves the key at the
+first sync and checks it every time. If the key has changed, the info won't be
+saved from the node (or the node id must be changed). A model can be signed 
+with several keys but at least one of them must be the same as in the global 
+ID to verify the sender.
+
+Each node can have several keys. Nodes can have shared keys to serve as a pool
+(several nodes with the same key).
+
+Each post has an ID in the unique format: key-type::key::local-id
+
+All requests pass a request type, protocol and model versions, and a list of
+optional arguments used for filtering.
+
+Each request has its own version. Version consists of 2 numbers: first is
+incompatible version (1.3 and 2.0 are not compatible and must not be in sync)
+and the second one is minor and compatible (for example, new optional field
+is added which will be igroned by those who don't support it yet).
+
+Post edits and reflinks are not saved to the sync model. The replied post ID 
+can be got from the post text, and reflinks can be computed when loading 
+posts. The edit time is not saved because a foreign post can be 'edited' (new
+replies are added) but the signature must not change (so we can't update the
+content). The inner posts can be edited, and the signature will change then
+but the local-id won't, so the other node can detect that and replace the post
+instead of adding a new one.
+
+## 3.1 Requests ##
+
+There is no constraint on how the server should calculate the request. The
+server can return any information by any filter and the requesting node is
+responsible for validating it.
+
+The server is required to return the status of request. See 3.2 for details.
+
+### 3.1.1 pull ###
+
+"pull" request gets the desired model id list by the given filter (e.g. thread, tags,
+author)
+
+Sample request is as follows:
+
+    <?xml version="1.1" encoding="UTF-8" ?>
+    <request version="1.0" type="pull">
+        <model version="1.0" name="post">
+            <timestamp_from>0</timestamp_from>
+            <timestamp_to>0</timestamp_to>
+            <tags>
+                <tag>tag1</tag>
+            </tags>
+            <sender>
+                <allow>
+                    <key>abcehy3h9t</key>
+                    <key>ehoehyoe</key>
+                </allow>
+                <!-- There can be only allow block (all other are denied) or deny block (all other are allowed) -->
+            </sender>
+        </model>
+    </request>
+
+Under the <model> tag there are filters. Filters for the "post" model can
+be found in DIP-2.
+
+Sample response:
+
+    <?xml version="1.1" encoding="UTF-8" ?>
+    <response>
+        <status>success</status>
+        <models>
+            <id key="id1" type="ecdsa" local-id="1" />
+            <id key="id1" type="ecdsa" local-id="2" />
+            <id key="id2" type="ecdsa" local-id="1" />
+            <id key="id2" type="ecdsa" local-id="5" />
+        </models>
+    </response>
+
+### 3.1.2 get ###
+
+"get" gets models by id list.
+
+Sample request:
+
+    <?xml version="1.1" encoding="UTF-8" ?>
+    <request version="1.0" type="get">
+        <model version="1.0" name="post">
+            <id key="id1" type="ecdsa" local-id="1" />
+            <id key="id1" type="ecdsa" local-id="2" />
+        </model>
+    </request>
+
+Id consists of a key, key type and local id. This key is used for signing and
+validating of data in the model content.
+
+Sample response:
+
+    <?xml version="1.1" encoding="UTF-8" ?>
+    <response>
+        <!--
+             Valid statuses are 'success' and 'error'.
+        -->
+        <status>success</status>
+        <models>
+            <model name="post">
+                <!--
+                    Content tag is the data that is signed by signatures and must
+                    not be changed for the post from other node.
+                -->
+                <content>
+                    <id key="id1" type="ecdsa" local-id="1" />
+                    <title>13</title>
+                    <text>Thirteen</text>
+                    <thread><id key="id1" type="ecdsa" local-id="2" /></thread>
+                    <pub-time>12</pub-time>
+                    <!--
+                        Images are saved as attachments and included in the
+                        signature.
+                    -->
+                    <attachments>
+                        <attachment mimetype="image/png">TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5I</attachment>
+                    </attachments>
+                </content>
+                <!--
+                    There can be several signatures for one model. At least one
+                    signature must be made with the key used in global ID.
+                 -->
+                <signatures>
+                    <signature key="id1" type="ecdsa" value="dhefhtreh" />
+                    <signature key="id45" type="ecdsa" value="dsgfgdhefhtreh" />
+                </signatures>
+                <attachment-refs>
+                    <attachment-ref ref="TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5IGhpcyByZWFzb24sIGJ1dCBieSB0"
+                        url="/media/images/12345.png" />
+                </attachment-refs>
+            </model>
+            <model name="post">
+                <content>
+                    <id key="id1" type="ecdsa" local-id="id2" />
+                    <title>13</title>
+                    <text>Thirteen</text>
+                    <pub-time>12</pub-time>
+                    <edit-time>13</edit-time>
+                    <tags>
+                        <tag>tag1</tag>
+                    </tags>
+                </content>
+                <signatures>
+                    <signature key="id2" type="ecdsa" value="dehdfh" />
+                </signatures>
+            </model>
+        </models>
+    </response>
+
+### 3.1.3 put ###
+
+"put" gives a model to the given node (you have no guarantee the node takes
+it, consider you are just advising the node to take your post). This request
+type is useful in pool where all the nodes try to duplicate all of their data
+across the pool.
+
+## 3.2 Responses ##
+
+### 3.2.1 "not supported" ###
+
+If the request if completely not supported, a "not supported" status will be
+returned.
+
+### 3.2.2 "success" ###
+
+"success" status means the request was processed and the result is returned.
+
+### 3.2.3 "error" ###
+
+If the server knows for sure that the operation cannot be processed, it sends
+the "error" status. Additional tags describing the error may be <description>
+and <stack>.

docs/dip-2.markdown

@@ -0,0 +1,29 b''
+# 0 Title #
+
+"post" model reference
+
+# 1 Description #
+
+"post" is a model that defines an imageboard message, or post.
+
+# 2 Fields #
+
+# 2.1 Mandatory fields #
+
+* title -- text field.
+* text -- text field.
+* pub-time -- timestamp (TBD: Define format).
+
+# 2.2 Optional fields #
+
+* attachments -- defines attachments such as images.
+* attachment -- contains and attachment or link to attachment to be downloaded
+manually. Required attributes are mimetype and name.
+
+This field is used for the opening post (thread):
+
+* tags -- text tag name list.
+
+This field is used for non-opening post:
+
+* thread -- ID of a post model that this post is related to.

boards/__init__.py

@@ -0,0 +1,1 b''
+default_app_config = 'boards.apps.BoardsAppConfig'
  No newline at end of file

boards/admin.py

@@ -1,7 +1,7 b''
 from django.contrib import admin
 from django.utils.translation import ugettext_lazy as _
 from django.core.urlresolvers import reverse
-from boards.models import Post, Tag, Ban, Thread, Banner, PostImage
+from boards.models import Post, Tag, Ban, Thread, Banner, PostImage, KeyPair, GlobalId
 
 
 @admin.register(Post)
@@ -62,7 +62,6 b' class TagAdmin(admin.ModelAdmin):'
         super().save_model(request, obj, form, change)
         for thread in obj.get_threads().all():
             thread.refresh_tags()
-
     list_display = ('name', 'thread_count', 'display_children')
     search_fields = ('name',)
 
@@ -89,7 +88,6 b' class ThreadAdmin(admin.ModelAdmin):'
     def save_related(self, request, form, formsets, change):
         super().save_related(request, form, formsets, change)
         form.instance.refresh_tags()
-
     list_display = ('id', 'op', 'title', 'reply_count', 'status', 'ip',
             'display_tags')
     list_filter = ('bump_time', 'status')
@@ -97,6 +95,13 b' class ThreadAdmin(admin.ModelAdmin):'
     filter_horizontal = ('tags',)
 
 
[e-mail](KeyPair)
+class KeyPairAdmin(admin.ModelAdmin):
+    list_display = ('public_key', 'primary')
+    list_filter = ('primary',)
+    search_fields = ('public_key',)
+
+
 @admin.register(Ban)
 class BanAdmin(admin.ModelAdmin):
     list_display = ('ip', 'can_read')
@@ -112,3 +117,11 b' class BannerAdmin(admin.ModelAdmin):'
 @admin.register(PostImage)
 class PostImageAdmin(admin.ModelAdmin):
     search_fields = ('alias',)
+
+
[e-mail](GlobalId)
+class GlobalIdAdmin(admin.ModelAdmin):
+    def is_linked(self, obj):
+        return Post.objects.filter(global_id=obj).exists()
+
+    list_display = ('__str__', 'is_linked',)
  No newline at end of file

boards/forms.py

@@ -15,7 +15,7 b' from django.utils import timezone'
 from boards.abstracts.settingsmanager import get_settings_manager
 from boards.abstracts.attachment_alias import get_image_by_alias
 from boards.mdx_neboard import formatters
-from boards.models.attachment.downloaders import Downloader
+from boards.models.attachment.downloaders import download
 from boards.models.post import TITLE_MAX_LENGTH
 from boards.models import Tag, Post
 from boards.utils import validate_file_size, get_file_mimetype, \
@@ -375,12 +375,7 b' class PostForm(NeboardForm):'
         img_temp = None
 
         try:
-            for downloader in Downloader.__subclasses__():
-                if downloader.handles(url):
-                    return downloader.download(url)
-            # If nobody of the specific downloaders handles this, use generic
-            # one
-            return Downloader.download(url)
+            download(url)
         except forms.ValidationError as e:
             raise e
         except Exception as e:

boards/management/commands/enforce_privacy.py

@@ -2,8 +2,7 b' from django.core.management import BaseC'
 from django.db import transaction
 
 from boards.models import Post
-from boards.models.post import NO_IP
-
+from boards.models.post.manager import NO_IP
 
 __author__ = 'neko259'
 

boards/mdx_neboard.py

@@ -16,6 +16,7 b' import boards'
 
 
 REFLINK_PATTERN = re.compile(r'^\d+$')
+GLOBAL_REFLINK_PATTERN = re.compile(r'(\w+)::([^:]+)::(\d+)')
 MULTI_NEWLINES_PATTERN = re.compile(r'(\r?\n){2,}')
 ONE_NEWLINE = '\n'
 REGEX_URL = re.compile(r'https?\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(/\S*)?')
@@ -121,15 +122,27 b' class CodePattern(TextFormatter):'
 def render_reflink(tag_name, value, options, parent, context):
     result = '>>%s' % value
 
+    post = None
     if REFLINK_PATTERN.match(value):
         post_id = int(value)
 
         try:
             post = boards.models.Post.objects.get(id=post_id)
 
-            result = post.get_link_view()
         except ObjectDoesNotExist:
             pass
+    elif GLOBAL_REFLINK_PATTERN.match(value):
+        match = GLOBAL_REFLINK_PATTERN.search(value)
+        try:
+            global_id = boards.models.GlobalId.objects.get(
+                key_type=match.group(1), key=match.group(2),
+                local_id=match.group(3))
+            post = global_id.post
+        except ObjectDoesNotExist:
+            pass
+
+    if post is not None:
+        result = post.get_link_view()
 
     return result
 
@@ -177,9 +190,6 b' def render_spoiler(tag_name, value, opti'
     return '<span class="spoiler">{}{}{}</span>'.format(side_spaces, value,
             side_spaces)
 
-    return quote_element
-
-
 
 formatters = [
     QuotePattern,

boards/models/__init__.py

@@ -3,6 +3,8 b" STATUS_BUMPLIMIT = 'bumplimit'"
 STATUS_ARCHIVE = 'archived'
 
 
+from boards.models.signature import GlobalId, Signature
+from boards.models.sync_key import KeyPair
 from boards.models.image import PostImage
 from boards.models.attachment import Attachment
 from boards.models.thread import Thread

boards/models/attachment/downloaders.py

@@ -54,6 +54,15 b' class Downloader:'
             return file
 
 
+def download(url):
+    for downloader in Downloader.__subclasses__():
+        if downloader.handles(url):
+            return downloader.download(url)
+    # If nobody of the specific downloaders handles this, use generic
+    # one
+    return Downloader.download(url)
+
+
 class YouTubeDownloader(Downloader):
     @staticmethod
     def download(url: str):

boards/models/post/__init__.py

@@ -1,26 +1,19 b''
-import logging
-import re
 import uuid
 
+import re
+from boards import settings
+from boards.abstracts.tripcode import Tripcode
+from boards.models import PostImage, Attachment, KeyPair, GlobalId
+from boards.models.base import Viewable
+from boards.models.post.export import get_exporter, DIFF_TYPE_JSON
+from boards.models.post.manager import PostManager
 from boards.utils import datetime_to_epoch
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.urlresolvers import reverse
 from django.db import models
 from django.db.models import TextField, QuerySet
-from django.template.defaultfilters import striptags, truncatewords
+from django.template.defaultfilters import truncatewords, striptags
 from django.template.loader import render_to_string
-from django.utils import timezone
-from django.db.models.signals import post_save, pre_save
-from django.dispatch import receiver
-
-from boards import settings
-from boards.abstracts.tripcode import Tripcode
-from boards.mdx_neboard import get_parser
-from boards.models import PostImage, Attachment
-from boards.models.base import Viewable
-from boards.models.post.export import get_exporter, DIFF_TYPE_JSON
-from boards.models.post.manager import PostManager
-from boards.models.user import Notification
 
 CSS_CLS_HIDDEN_POST = 'hidden_post'
 CSS_CLS_DEAD_POST = 'dead_post'
@@ -39,6 +32,8 b' IMAGE_THUMB_SIZE = (200, 150)'
 TITLE_MAX_LENGTH = 200
 
 REGEX_REPLY = re.compile(r'\[post\](\d+)\[/post\]')
+REGEX_GLOBAL_REPLY = re.compile(r'\[post\](\w+)::([^:]+)::(\d+)\[/post\]')
+REGEX_URL = re.compile(r'https?\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(/\S*)?')
 REGEX_NOTIFICATION = re.compile(r'\[user\](\w+)\[/user\]')
 
 PARAMETER_TRUNCATED = 'truncated'
@@ -101,6 +96,11 b' class Post(models.Model, Viewable):'
     url = models.TextField()
     uid = models.TextField(db_index=True)
 
+    # Global ID with author key. If the message was downloaded from another
+    # server, this indicates the server.
+    global_id = models.OneToOneField(GlobalId, null=True, blank=True,
+                                     on_delete=models.CASCADE)
+
     tripcode = models.CharField(max_length=50, blank=True, default='')
     opening = models.BooleanField(db_index=True)
     hidden = models.BooleanField(default=False)
@@ -214,29 +214,54 b' class Post(models.Model, Viewable):'
     def get_first_image(self) -> PostImage:
         return self.images.earliest('id')
 
-    def delete(self, using=None):
+    def set_global_id(self, key_pair=None):
         """
-        Deletes all post images and the post itself.
+        Sets global id based on the given key pair. If no key pair is given,
+        default one is used.
         """
 
-        for image in self.images.all():
-            image_refs_count = image.post_images.count()
-            if image_refs_count == 1:
-                image.delete()
+        if key_pair:
+            key = key_pair
+        else:
+            try:
+                key = KeyPair.objects.get(primary=True)
+            except KeyPair.DoesNotExist:
+                # Do not update the global id because there is no key defined
+                return
+        global_id = GlobalId(key_type=key.key_type,
+                             key=key.public_key,
+                             local_id=self.id)
+        global_id.save()
+
+        self.global_id = global_id
+
+        self.save(update_fields=['global_id'])
+
+    def get_pub_time_str(self):
+        return str(self.pub_time)
 
-        for attachment in self.attachments.all():
-            attachment_refs_count = attachment.attachment_posts.count()
-            if attachment_refs_count == 1:
-                attachment.delete()
+    def get_replied_ids(self):
+        """
+        Gets ID list of the posts that this post replies.
+        """
+
+        raw_text = self.get_raw_text()
 
-        thread = self.get_thread()
-        thread.last_edit_time = timezone.now()
-        thread.save()
+        local_replied = REGEX_REPLY.findall(raw_text)
+        global_replied = []
+        for match in REGEX_GLOBAL_REPLY.findall(raw_text):
+            key_type = match[0]
+            key = match[1]
+            local_id = match[2]
 
-        super(Post, self).delete(using)
-
-        logging.getLogger('boards.post.delete').info(
-            'Deleted post {}'.format(self))
+            try:
+                global_id = GlobalId.objects.get(key_type=key_type,
+                                                 key=key, local_id=local_id)
+                for post in Post.objects.filter(global_id=global_id).only('id'):
+                    global_replied.append(post.id)
+            except GlobalId.DoesNotExist:
+                pass
+        return local_replied + global_replied
 
     def get_post_data(self, format_type=DIFF_TYPE_JSON, request=None,
                       include_last_update=False) -> str:
@@ -305,6 +330,24 b' class Post(models.Model, Viewable):'
     def get_raw_text(self) -> str:
         return self.text
 
+    def get_sync_text(self) -> str:
+        """
+        Returns text applicable for sync. It has absolute post reflinks.
+        """
+
+        replacements = dict()
+        for post_id in REGEX_REPLY.findall(self.get_raw_text()):
+            absolute_post_id = str(Post.objects.get(id=post_id).global_id)
+            replacements[post_id] = absolute_post_id
+
+        text = self.get_raw_text() or ''
+        for key in replacements:
+            text = text.replace('[post]{}[/post]'.format(key),
+                                '[post]{}[/post]'.format(replacements[key]))
+        text = text.replace('\r\n', '\n').replace('\r', '\n')
+
+        return text
+
     def connect_threads(self, opening_posts):
         for opening_post in opening_posts:
             threads = opening_post.get_threads().all()
@@ -336,34 +379,3 b' class Post(models.Model, Viewable):'
 
     def set_hidden(self, hidden):
         self.hidden = hidden
-
-
-# SIGNALS (Maybe move to other module?)
-@receiver(post_save, sender=Post)
-def connect_replies(instance, **kwargs):
-    for reply_number in re.finditer(REGEX_REPLY, instance.get_raw_text()):
-        post_id = reply_number.group(1)
-
-        try:
-            referenced_post = Post.objects.get(id=post_id)
-
-            # Connect only to posts that are not connected to already
-            if not referenced_post.referenced_posts.filter(id=instance.id).exists():
-                referenced_post.referenced_posts.add(instance)
-                referenced_post.last_edit_time = instance.pub_time
-                referenced_post.build_refmap()
-                referenced_post.save(update_fields=['refmap', 'last_edit_time'])
-        except ObjectDoesNotExist:
-            pass
-
-
-@receiver(post_save, sender=Post)
-def connect_notifications(instance, **kwargs):
-    for reply_number in re.finditer(REGEX_NOTIFICATION, instance.get_raw_text()):
-        user_name = reply_number.group(1).lower()
-        Notification.objects.get_or_create(name=user_name, post=instance)
-
-
-@receiver(pre_save, sender=Post)
-def preparse_text(instance, **kwargs):
-    instance._text_rendered = get_parser().parse(instance.get_raw_text())

boards/models/post/manager.py

@@ -80,17 +80,13 b' class PostManager(models.Manager):'
         logger.info('Created post [{}] with text [{}] by {}'.format(post,
             post.get_text(),post.poster_ip))
 
-        # TODO Move this to other place
         if file:
-            file_type = file.name.split('.')[-1].lower()
-            if file_type in IMAGE_TYPES:
-                post.images.add(PostImage.objects.create_with_hash(file))
-            else:
-                post.attachments.add(Attachment.objects.create_with_hash(file))
+            self._add_file_to_post(file, post)
         for image in images:
             post.images.add(image)
 
         post.connect_threads(opening_posts)
+        post.set_global_id()
 
         # Thread needs to be bumped only when the post is already created
         if not new_thread:
@@ -131,3 +127,34 b' class PostManager(models.Manager):'
 
         return ppd
 
+    @transaction.atomic
+    def import_post(self, title: str, text: str, pub_time: str, global_id,
+                    opening_post=None, tags=list(), files=list()):
+        is_opening = opening_post is None
+        if is_opening:
+            thread = boards.models.thread.Thread.objects.create(
+                bump_time=pub_time, last_edit_time=pub_time)
+            list(map(thread.tags.add, tags))
+        else:
+            thread = opening_post.get_thread()
+
+        post = self.create(title=title, text=text,
+                           pub_time=pub_time,
+                           poster_ip=NO_IP,
+                           last_edit_time=pub_time,
+                           global_id=global_id,
+                           opening=is_opening,
+                           thread=thread)
+
+        # TODO Add files
+        for file in files:
+            self._add_file_to_post(file, post)
+
+        post.threads.add(thread)
+
+    def _add_file_to_post(self, file, post):
+        file_type = file.name.split('.')[-1].lower()
+        if file_type in IMAGE_TYPES:
+            post.images.add(PostImage.objects.create_with_hash(file))
+        else:
+            post.attachments.add(Attachment.objects.create_with_hash(file))

boards/models/user.py

@@ -1,6 +1,5 b''
 from django.db import models
-
-import boards.models.post
+import boards
 
 __author__ = 'neko259'
 

boards/static/css/md/base_page.css

@@ -492,6 +492,11 b' ul {'
     font-size: 1.2em;
 }
 
+.global-id {
+    font-weight: bolder;
+    opacity: .5;
+}
+
 /* Reflink preview */
 .post_preview {
     border-left: 1px solid #777;

boards/templates/boards/post.html

@@ -55,7 +55,9 b''
                        | <a href="{% url 'admin:boards_thread_change' thread.id %}">{% trans 'Edit thread' %}</a>
                     {% endif %}
                 {% endif %}
-            </form>
+                {% if post.global_id_id %}
+                    | <a href="{% url 'post_sync_data' post.id %}">RAW</a>
+                {% endif %}
             </span>
         {% endif %}
     </div>

boards/tests/test_forms.py

@@ -42,7 +42,7 b' class FormTest(TestCase):'
         # Change posting delay so we don't have to wait for 30 seconds or more
         old_posting_delay = neboard.settings.POSTING_DELAY
         # Wait fot the posting delay or we won't be able to post
-        settings.POSTING_DELAY = 1
+        neboard.settings.POSTING_DELAY = 1
         time.sleep(neboard.settings.POSTING_DELAY + 1)
         response = client.post(THREAD_PAGE_ONE, {'text': TEST_TEXT,
                                                  'tags': valid_tags})

boards/tests/test_post.py

@@ -2,7 +2,7 b' from django.core.paginator import Pagina'
 from django.test import TestCase
 
 from boards import settings
-from boards.models import Tag, Post, Thread
+from boards.models import Tag, Post, Thread, KeyPair
 from boards.models.thread import STATUS_ARCHIVE
 
 
@@ -115,6 +115,27 b' class PostTests(TestCase):'
         self.assertEqual(all_threads[settings.get_int('View', 'ThreadsPerPage')].id,
                          first_post.id)
 
+    def test_reflinks(self):
+        """
+        Tests that reflinks are parsed within post and connecting replies
+        to the replied posts.
+
+        Local reflink example: [post]123[/post]
+        Global reflink example: [post]key_type::key::123[/post]
+        """
+
+        key = KeyPair.objects.generate_key(primary=True)
+
+        tag = Tag.objects.create(name='test_tag')
+
+        post = Post.objects.create_post(title='', text='', tags=[tag])
+        post_local_reflink = Post.objects.create_post(title='',
+                text='[post]%d[/post]' % post.id, thread=post.get_thread())
+
+        self.assertTrue(post_local_reflink in post.referenced_posts.all(),
+            'Local reflink not connecting posts.')
+
+
     def test_thread_replies(self):
         """
         Tests that the replies can be queried from a thread in all possible

boards/tests/test_views.py

@@ -9,8 +9,9 b' logger = logging.getLogger(__name__)'
 HTTP_CODE_OK = 200
 
 EXCLUDED_VIEWS = {
-        'banned',
-        'get_thread_diff',
+    'banned',
+    'get_thread_diff',
+    'api_sync_pull',
 }
 
 

boards/urls.py

@@ -2,14 +2,17 b' from django.conf.urls import url'
 #from django.views.i18n import javascript_catalog
 
 import neboard
+
 from boards import views
 from boards.rss import AllThreadsFeed, TagThreadsFeed, ThreadPostsFeed
 from boards.views import api, tag_threads, all_threads, \
     settings, all_tags, feed
 from boards.views.authors import AuthorsView
 from boards.views.notifications import NotificationView
+from boards.views.search import BoardSearchView
 from boards.views.static import StaticPageView
 from boards.views.preview import PostPreviewView
+from boards.views.sync import get_post_sync_data, response_get, response_pull
 from boards.views.random import RandomImageView
 from boards.views.tag_gallery import TagGalleryView
 from boards.views.translation import cached_javascript_catalog
@@ -74,12 +77,19 b' urlpatterns = ['
     url(r'^api/preview/$', api.api_get_preview, name='preview'),
     url(r'^api/new_posts/$', api.api_get_new_posts, name='new_posts'),
 
+    # Sync protocol API
+    url(r'^api/sync/pull/$', response_pull, name='api_sync_pull'),
+    url(r'^api/sync/get/$', response_get, name='api_sync_pull'),
+    # TODO 'get' request
+
     # Notifications
     url(r'^notifications/(?P<username>\w+)/$', NotificationView.as_view(), name='notifications'),
     url(r'^notifications/$', NotificationView.as_view(), name='notifications'),
 
     # Post preview
     url(r'^preview/$', PostPreviewView.as_view(), name='preview'),
+    url(r'^post_xml/(?P<post_id>\d+)$', get_post_sync_data,
+        name='post_sync_data'),
 ]
 
 # Search

boards/views/api.py

@@ -1,26 +1,20 b''
-from collections import OrderedDict
 import json
 import logging
 
+from django.core import serializers
 from django.db import transaction
-from django.db.models import Count
 from django.http import HttpResponse
 from django.shortcuts import get_object_or_404
-from django.core import serializers
-from django.template.context_processors import csrf
 from django.views.decorators.csrf import csrf_protect
 
-from boards.abstracts.settingsmanager import get_settings_manager,\
-        FAV_THREAD_NO_UPDATES
-
+from boards.abstracts.settingsmanager import get_settings_manager
 from boards.forms import PostForm, PlainErrorList
+from boards.mdx_neboard import Parser
 from boards.models import Post, Thread, Tag
 from boards.models.thread import STATUS_ARCHIVE
+from boards.models.user import Notification
 from boards.utils import datetime_to_epoch
 from boards.views.thread import ThreadView
-from boards.models.user import Notification
-from boards.mdx_neboard import Parser
-
 
 __author__ = 'neko259'
 

neboard/settings.py

@@ -2,7 +2,6 b''
 import os
 
 DEBUG = True
-TEMPLATE_DEBUG = DEBUG
 
 ADMINS = (
     # ('Your Name', '[e-mail]'),
@@ -67,16 +66,7 b" STATIC_ROOT = ''"
 # Example: "http://media.lawrence.com/static/"
 STATIC_URL = '/static/'
 
-# Additional locations of static files
-# It is really a hack, put real paths, not related
-STATICFILES_DIRS = (
-    os.path.dirname(__file__) + '/boards/static',
-
-#    '/d/work/python/django/neboard/neboard/boards/static',
-    # Put strings here, like "/home/html/static" or "C:/www/django/static".
-    # Always use forward slashes, even on Windows.
-    # Don't forget to use absolute paths, not relative paths.
-)
+STATICFILES_DIRS = []
 
 # List of finder classes that know how to find static files in
 # various locations.
@@ -227,6 +217,7 b' MIDDLEWARE_CLASSES += ['
     'debug_toolbar.middleware.DebugToolbarMiddleware',
 ]
 
+
 def custom_show_toolbar(request):
     return request.user.has_perm('admin.debug')
 

readme.markdown

@@ -9,21 +9,54 b' Site: http://neboard.me/'
 
 # INSTALLATION #
 
-1. Install all dependencies over pip or system-wide
-2. Setup a database in `neboard/settings.py`
-3. Run `./manage.py migrate` to apply all south migrations
-4. Apply config changes to `boards/config/config.ini`. You can see the default settings in `boards/config/default_config.ini`
+1. Download application and move inside it:
+
+`hg clone https://bitbucket.org/neko259/neboard`
+
+`cd neboard`
+
+If you wish to use *decentral* version, change branch to *decentral*:
+
+`hg up decentral`
+
+2. Install all application dependencies:
+
+Some minimal system-wide depenencies:
+
+* python3
+* pip/pip3
+* jpeg
+
+Python dependencies:
+
+`pip3 install -r requirements.txt`
+
+You can use virtualenv to speed up the process or avoid conflicts.
+
+3. Setup a database in `neboard/settings.py`. You can also change other settings like search engine.
+
+Depending on configured database and search engine, you need to install corresponding dependencies manually.
+
+Default database is *sqlite*, default search engine is *whoosh*.
+
+4. Setup SECRET_KEY to a secret value in `neboard/settings.py
+5. Run `./manage.py migrate` to apply all migrations
+6. Apply config changes to `boards/config/config.ini`. You can see the default settings in `boards/config/default_config.ini`
+7. If you want to use decetral engine, run `./manage.py generate_keypair` to generate keys
 
 # RUNNING #
 
-You can run the server using django default embedded webserver by running
+You can run the server using django default embedded webserver by running:
 
     ./manage.py runserver <address>:<port>
 
-See django-admin command help for details
+See django-admin command help for details.
 
 Also consider using wsgi or fcgi interfaces on production servers.
 
+When running for the first time, you need to setup at least one section tag.
+Go to the admin page and manually create one tag with "required" property set.
+
 # UPGRADE #
 
 1. Backup your project data.
@@ -34,4 +67,4 b' You can also just clone the mercurial pr'
 
 # CONCLUSION #
 
-Enjoy our software and thank you!
+Enjoy our software and thank you!
  No newline at end of file

requirements.txt

@@ -1,4 +1,6 b''
 python-magic
+httplib2
+simplejson
 pytube
 requests
 adjacent
@@ -8,3 +10,4 b' django>=1.8'
 bbcode
 django-debug-toolbar
 pytz
+ecdsa
  No newline at end of file

todo.txt

@@ -15,6 +15,7 b''
 * Subscribing to tag via AJAX
 * Add buttons to insert a named link or a named quote to the markup panel
 * Add support for "attention posts" that are shown in the header"
+* Use absolute post reflinks in the raw text
 * Use default boards/settings when it is not defined. Maybe default_settings.py module?
 
 = Bugs =