Merged the decentral branch into default branch

boards/apps.py

@@ -0,0 +1,11 b''
+from django.apps import AppConfig
+
+
+class BoardsAppConfig(AppConfig):
+    name = 'boards'
+    verbose_name = 'Boards'
+
+    def ready(self):
+        super().ready()
+
+        import boards.signals
  No newline at end of file

boards/management/commands/generate_keypair.py

@@ -0,0 +1,21 b''
+__author__ = 'neko259'
+
+
+from django.core.management import BaseCommand
+from django.db import transaction
+
+from boards.models import KeyPair, Post
+
+
+class Command(BaseCommand):
+    help = 'Generates the new keypair. The first one will be primary.'
+
+    @transaction.atomic
+    def handle(self, *args, **options):
+        first_key = not KeyPair.objects.has_primary()
+        key = KeyPair.objects.generate_key(
+            primary=first_key)
+        print(key)
+
+        for post in Post.objects.filter(global_id=None):
+            post.set_global_id()

boards/management/commands/sync_with_server.py

@@ -0,0 +1,80 b''
+import re
+import xml.etree.ElementTree as ET
+
+import httplib2
+from django.core.management import BaseCommand
+
+from boards.models import GlobalId
+from boards.models.post.sync import SyncManager
+
+__author__ = 'neko259'
+
+
+REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)')
+
+
+class Command(BaseCommand):
+    help = 'Send a sync or get request to the server.'
+
+    def add_arguments(self, parser):
+        parser.add_argument('url', type=str, help='Server root url')
+        parser.add_argument('--global-id', type=str, default='',
+                            help='Post global ID')
+
+    def handle(self, *args, **options):
+        url = options.get('url')
+
+        pull_url = url + 'api/sync/pull/'
+        get_url = url + 'api/sync/get/'
+        file_url = url[:-1]
+
+        global_id_str = options.get('global_id')
+        if global_id_str:
+            match = REGEX_GLOBAL_ID.match(global_id_str)
+            if match:
+                key_type = match.group(1)
+                key = match.group(2)
+                local_id = match.group(3)
+
+                global_id = GlobalId(key_type=key_type, key=key,
+                                     local_id=local_id)
+
+                xml = GlobalId.objects.generate_request_get([global_id])
+                # body = urllib.parse.urlencode(data)
+                h = httplib2.Http()
+                response, content = h.request(get_url, method="POST", body=xml)
+
+                SyncManager.parse_response_get(content, file_url)
+            else:
+                raise Exception('Invalid global ID')
+        else:
+            h = httplib2.Http()
+            xml = GlobalId.objects.generate_request_pull()
+            response, content = h.request(pull_url, method="POST", body=xml)
+
+            print(content.decode() + '\n')
+
+            root = ET.fromstring(content)
+            status = root.findall('status')[0].text
+            if status == 'success':
+                ids_to_sync = list()
+
+                models = root.findall('models')[0]
+                for model in models:
+                    global_id, exists = GlobalId.from_xml_element(model)
+                    if not exists:
+                        print(global_id)
+                        ids_to_sync.append(global_id)
+                print()
+
+                if len(ids_to_sync) > 0:
+                    xml = GlobalId.objects.generate_request_get(ids_to_sync)
+                    # body = urllib.parse.urlencode(data)
+                    h = httplib2.Http()
+                    response, content = h.request(get_url, method="POST", body=xml)
+
+                    SyncManager.parse_response_get(content, file_url)
+                else:
+                    print('Nothing to get, everything synced')
+            else:
+                raise Exception('Invalid response status')

boards/migrations/0026_auto_20150830_2006.py

@@ -0,0 +1,48 b''
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('boards', '0025_auto_20150825_2049'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='GlobalId',
+            fields=[
+                ('id', models.AutoField(serialize=False, verbose_name='ID', primary_key=True, auto_created=True)),
+                ('key', models.TextField()),
+                ('key_type', models.TextField()),
+                ('local_id', models.IntegerField()),
+            ],
+        ),
+        migrations.CreateModel(
+            name='KeyPair',
+            fields=[
+                ('id', models.AutoField(serialize=False, verbose_name='ID', primary_key=True, auto_created=True)),
+                ('public_key', models.TextField()),
+                ('private_key', models.TextField()),
+                ('key_type', models.TextField()),
+                ('primary', models.BooleanField(default=False)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Signature',
+            fields=[
+                ('id', models.AutoField(serialize=False, verbose_name='ID', primary_key=True, auto_created=True)),
+                ('key_type', models.TextField()),
+                ('key', models.TextField()),
+                ('signature', models.TextField()),
+                ('global_id', models.ForeignKey(to='boards.GlobalId')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='post',
+            name='global_id',
+            field=models.OneToOneField(to='boards.GlobalId', null=True, blank=True),
+        ),
+    ]

boards/migrations/0031_merge.py

@@ -0,0 +1,15 b''
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('boards', '0030_auto_20150929_1816'),
+        ('boards', '0026_auto_20150830_2006'),
+    ]
+
+    operations = [
+    ]

boards/migrations/0036_merge.py

@@ -0,0 +1,15 b''
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('boards', '0031_merge'),
+        ('boards', '0035_auto_20151021_1346'),
+    ]
+
+    operations = [
+    ]

boards/migrations/0041_merge.py

@@ -0,0 +1,15 b''
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('boards', '0040_thread_monochrome'),
+        ('boards', '0036_merge'),
+    ]
+
+    operations = [
+    ]

boards/migrations/0043_merge.py

@@ -0,0 +1,16 b''
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.5 on 2016-04-29 15:52
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('boards', '0041_merge'),
+        ('boards', '0042_auto_20160422_1053'),
+    ]
+
+    operations = [
+    ]

boards/migrations/0044_globalid_content.py

@@ -0,0 +1,20 b''
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.5 on 2016-05-04 15:36
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('boards', '0043_merge'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='globalid',
+            name='content',
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

boards/models/post/sync.py

@@ -0,0 +1,270 b''
+import xml.etree.ElementTree as et
+
+from boards.models.attachment.downloaders import download
+from boards.utils import get_file_mimetype, get_file_hash
+from django.db import transaction
+from boards.models import KeyPair, GlobalId, Signature, Post, Tag
+
+EXCEPTION_NODE = 'Sync node returned an error: {}'
+EXCEPTION_OP = 'Load the OP first'
+EXCEPTION_DOWNLOAD = 'File was not downloaded'
+EXCEPTION_HASH = 'File hash does not match attachment hash'
+EXCEPTION_SIGNATURE = 'Invalid model signature for {}'
+ENCODING_UNICODE = 'unicode'
+
+TAG_MODEL = 'model'
+TAG_REQUEST = 'request'
+TAG_RESPONSE = 'response'
+TAG_ID = 'id'
+TAG_STATUS = 'status'
+TAG_MODELS = 'models'
+TAG_TITLE = 'title'
+TAG_TEXT = 'text'
+TAG_THREAD = 'thread'
+TAG_PUB_TIME = 'pub-time'
+TAG_SIGNATURES = 'signatures'
+TAG_SIGNATURE = 'signature'
+TAG_CONTENT = 'content'
+TAG_ATTACHMENTS = 'attachments'
+TAG_ATTACHMENT = 'attachment'
+TAG_TAGS = 'tags'
+TAG_TAG = 'tag'
+TAG_ATTACHMENT_REFS = 'attachment-refs'
+TAG_ATTACHMENT_REF = 'attachment-ref'
+
+TYPE_GET = 'get'
+
+ATTR_VERSION = 'version'
+ATTR_TYPE = 'type'
+ATTR_NAME = 'name'
+ATTR_VALUE = 'value'
+ATTR_MIMETYPE = 'mimetype'
+ATTR_KEY = 'key'
+ATTR_REF = 'ref'
+ATTR_URL = 'url'
+
+STATUS_SUCCESS = 'success'
+
+
+class SyncException(Exception):
+    pass
+
+
+class SyncManager:
+    @staticmethod
+    def generate_response_get(model_list: list):
+        response = et.Element(TAG_RESPONSE)
+
+        status = et.SubElement(response, TAG_STATUS)
+        status.text = STATUS_SUCCESS
+
+        models = et.SubElement(response, TAG_MODELS)
+
+        for post in model_list:
+            model = et.SubElement(models, TAG_MODEL)
+            model.set(ATTR_NAME, 'post')
+
+            global_id = post.global_id
+
+            images = post.images.all()
+            attachments = post.attachments.all()
+            if global_id.content:
+                model.append(et.fromstring(global_id.content))
+                if len(images) > 0 or len(attachments) > 0:
+                    attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS)
+                    for image in images:
+                        SyncManager._attachment_to_xml(
+                            None, attachment_refs, image.image.file,
+                            image.hash, image.image.url)
+                    for file in attachments:
+                        SyncManager._attachment_to_xml(
+                            None, attachment_refs, file.file.file,
+                            file.hash, file.file.url)
+            else:
+                content_tag = et.SubElement(model, TAG_CONTENT)
+
+                tag_id = et.SubElement(content_tag, TAG_ID)
+                global_id.to_xml_element(tag_id)
+
+                title = et.SubElement(content_tag, TAG_TITLE)
+                title.text = post.title
+
+                text = et.SubElement(content_tag, TAG_TEXT)
+                text.text = post.get_sync_text()
+
+                thread = post.get_thread()
+                if post.is_opening():
+                    tag_tags = et.SubElement(content_tag, TAG_TAGS)
+                    for tag in thread.get_tags():
+                        tag_tag = et.SubElement(tag_tags, TAG_TAG)
+                        tag_tag.text = tag.name
+                else:
+                    tag_thread = et.SubElement(content_tag, TAG_THREAD)
+                    thread_id = et.SubElement(tag_thread, TAG_ID)
+                    thread.get_opening_post().global_id.to_xml_element(thread_id)
+
+                pub_time = et.SubElement(content_tag, TAG_PUB_TIME)
+                pub_time.text = str(post.get_pub_time_str())
+
+                if len(images) > 0 or len(attachments) > 0:
+                    attachments_tag = et.SubElement(content_tag, TAG_ATTACHMENTS)
+                    attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS)
+
+                    for image in images:
+                        SyncManager._attachment_to_xml(
+                            attachments_tag, attachment_refs, image.image.file,
+                            image.hash, image.image.url)
+                    for file in attachments:
+                        SyncManager._attachment_to_xml(
+                            attachments_tag, attachment_refs, file.file.file,
+                            file.hash, file.file.url)
+
+                global_id.content = et.tostring(content_tag, ENCODING_UNICODE)
+                global_id.save()
+
+            signatures_tag = et.SubElement(model, TAG_SIGNATURES)
+            post_signatures = global_id.signature_set.all()
+            if post_signatures:
+                signatures = post_signatures
+            else:
+                key = KeyPair.objects.get(public_key=global_id.key)
+                signature = Signature(
+                    key_type=key.key_type,
+                    key=key.public_key,
+                    signature=key.sign(global_id.content),
+                    global_id=global_id,
+                )
+                signature.save()
+                signatures = [signature]
+            for signature in signatures:
+                signature_tag = et.SubElement(signatures_tag, TAG_SIGNATURE)
+                signature_tag.set(ATTR_TYPE, signature.key_type)
+                signature_tag.set(ATTR_VALUE, signature.signature)
+                signature_tag.set(ATTR_KEY, signature.key)
+
+        return et.tostring(response, ENCODING_UNICODE)
+
+    @staticmethod
+    @transaction.atomic
+    def parse_response_get(response_xml, hostname):
+        tag_root = et.fromstring(response_xml)
+        tag_status = tag_root.find(TAG_STATUS)
+        if STATUS_SUCCESS == tag_status.text:
+            tag_models = tag_root.find(TAG_MODELS)
+            for tag_model in tag_models:
+                tag_content = tag_model.find(TAG_CONTENT)
+
+                content_str = et.tostring(tag_content, ENCODING_UNICODE)
+                signatures = SyncManager._verify_model(content_str, tag_model)
+
+                tag_id = tag_content.find(TAG_ID)
+                global_id, exists = GlobalId.from_xml_element(tag_id)
+
+                if exists:
+                    print('Post with same ID already exists')
+                else:
+                    global_id.content = content_str
+                    global_id.save()
+                    for signature in signatures:
+                        signature.global_id = global_id
+                        signature.save()
+
+                    title = tag_content.find(TAG_TITLE).text or ''
+                    text = tag_content.find(TAG_TEXT).text or ''
+                    pub_time = tag_content.find(TAG_PUB_TIME).text
+
+                    thread = tag_content.find(TAG_THREAD)
+                    tags = []
+                    if thread:
+                        thread_id = thread.find(TAG_ID)
+                        op_global_id, exists = GlobalId.from_xml_element(thread_id)
+                        if exists:
+                            opening_post = Post.objects.get(global_id=op_global_id)
+                        else:
+                            raise SyncException(EXCEPTION_OP)
+                    else:
+                        opening_post = None
+                        tag_tags = tag_content.find(TAG_TAGS)
+                        for tag_tag in tag_tags:
+                            tag, created = Tag.objects.get_or_create(
+                                name=tag_tag.text)
+                            tags.append(tag)
+
+                    # TODO Check that the replied posts are already present
+                    # before adding new ones
+
+                    files = []
+                    tag_attachments = tag_content.find(TAG_ATTACHMENTS) or list()
+                    tag_refs = tag_model.find(TAG_ATTACHMENT_REFS)
+                    for attachment in tag_attachments:
+                        tag_ref = tag_refs.find("{}[@ref='{}']".format(
+                            TAG_ATTACHMENT_REF, attachment.text))
+                        url = tag_ref.get(ATTR_URL)
+                        attached_file = download(hostname + url)
+                        if attached_file is None:
+                            raise SyncException(EXCEPTION_DOWNLOAD)
+
+                        hash = get_file_hash(attached_file)
+                        if hash != attachment.text:
+                            raise SyncException(EXCEPTION_HASH)
+
+                        files.append(attached_file)
+
+                    Post.objects.import_post(
+                        title=title, text=text, pub_time=pub_time,
+                        opening_post=opening_post, tags=tags,
+                        global_id=global_id, files=files)
+        else:
+            raise SyncException(EXCEPTION_NODE.format(tag_status.text))
+
+    @staticmethod
+    def generate_response_pull():
+        response = et.Element(TAG_RESPONSE)
+
+        status = et.SubElement(response, TAG_STATUS)
+        status.text = STATUS_SUCCESS
+
+        models = et.SubElement(response, TAG_MODELS)
+
+        for post in Post.objects.all():
+            tag_id = et.SubElement(models, TAG_ID)
+            post.global_id.to_xml_element(tag_id)
+
+        return et.tostring(response, ENCODING_UNICODE)
+
+    @staticmethod
+    def _verify_model(content_str, tag_model):
+        """
+        Verifies all signatures for a single model.
+        """
+
+        signatures = []
+
+        tag_signatures = tag_model.find(TAG_SIGNATURES)
+        for tag_signature in tag_signatures:
+            signature_type = tag_signature.get(ATTR_TYPE)
+            signature_value = tag_signature.get(ATTR_VALUE)
+            signature_key = tag_signature.get(ATTR_KEY)
+
+            signature = Signature(key_type=signature_type,
+                                  key=signature_key,
+                                  signature=signature_value)
+
+            if not KeyPair.objects.verify(signature, content_str):
+                raise SyncException(EXCEPTION_SIGNATURE.format(content_str))
+
+            signatures.append(signature)
+
+        return signatures
+
+    @staticmethod
+    def _attachment_to_xml(tag_attachments, tag_refs, file, hash, url):
+        if tag_attachments is not None:
+            mimetype = get_file_mimetype(file)
+            attachment = et.SubElement(tag_attachments, TAG_ATTACHMENT)
+            attachment.set(ATTR_MIMETYPE, mimetype)
+            attachment.text = hash
+
+        attachment_ref = et.SubElement(tag_refs, TAG_ATTACHMENT_REF)
+        attachment_ref.set(ATTR_REF, hash)
+        attachment_ref.set(ATTR_URL, url)

boards/models/signature.py

@@ -0,0 +1,144 b''
+import xml.etree.ElementTree as et
+from django.db import models
+
+
+TAG_MODEL = 'model'
+TAG_REQUEST = 'request'
+TAG_ID = 'id'
+
+TYPE_GET = 'get'
+TYPE_PULL = 'pull'
+
+ATTR_VERSION = 'version'
+ATTR_TYPE = 'type'
+ATTR_NAME = 'name'
+
+ATTR_KEY = 'key'
+ATTR_KEY_TYPE = 'type'
+ATTR_LOCAL_ID = 'local-id'
+
+
+class GlobalIdManager(models.Manager):
+    def generate_request_get(self, global_id_list: list):
+        """
+        Form a get request from a list of ModelId objects.
+        """
+
+        request = et.Element(TAG_REQUEST)
+        request.set(ATTR_TYPE, TYPE_GET)
+        request.set(ATTR_VERSION, '1.0')
+
+        model = et.SubElement(request, TAG_MODEL)
+        model.set(ATTR_VERSION, '1.0')
+        model.set(ATTR_NAME, 'post')
+
+        for global_id in global_id_list:
+            tag_id = et.SubElement(model, TAG_ID)
+            global_id.to_xml_element(tag_id)
+
+        return et.tostring(request, 'unicode')
+
+    def generate_request_pull(self):
+        """
+        Form a pull request from a list of ModelId objects.
+        """
+
+        request = et.Element(TAG_REQUEST)
+        request.set(ATTR_TYPE, TYPE_PULL)
+        request.set(ATTR_VERSION, '1.0')
+
+        model = et.SubElement(request, TAG_MODEL)
+        model.set(ATTR_VERSION, '1.0')
+        model.set(ATTR_NAME, 'post')
+
+        return et.tostring(request, 'unicode')
+
+    def global_id_exists(self, global_id):
+        """
+        Checks if the same global id already exists in the system.
+        """
+
+        return self.filter(key=global_id.key,
+                           key_type=global_id.key_type,
+                           local_id=global_id.local_id).exists()
+
+
+class GlobalId(models.Model):
+    """
+    Global model ID and cache.
+    Key, key type and local ID make a single global identificator of the model.
+    Content is an XML cache of the model that can be passed along between nodes
+    without manual serialization each time.
+    """
+    class Meta:
+        app_label = 'boards'
+
+    objects = GlobalIdManager()
+
+    def __init__(self, *args, **kwargs):
+        models.Model.__init__(self, *args, **kwargs)
+
+        if 'key' in kwargs and 'key_type' in kwargs and 'local_id' in kwargs:
+            self.key = kwargs['key']
+            self.key_type = kwargs['key_type']
+            self.local_id = kwargs['local_id']
+
+    key = models.TextField()
+    key_type = models.TextField()
+    local_id = models.IntegerField()
+    content = models.TextField(blank=True, null=True)
+
+    def __str__(self):
+        return '%s::%s::%d' % (self.key_type, self.key, self.local_id)
+
+    def to_xml_element(self, element: et.Element):
+        """
+        Exports global id to an XML element.
+        """
+
+        element.set(ATTR_KEY, self.key)
+        element.set(ATTR_KEY_TYPE, self.key_type)
+        element.set(ATTR_LOCAL_ID, str(self.local_id))
+
+    @staticmethod
+    def from_xml_element(element: et.Element):
+        """
+        Parses XML id tag and gets global id from it.
+
+        Arguments:
+        element -- the XML 'id' element
+
+        Returns:
+        global_id -- id itself
+        exists -- True if the global id was taken from database, False if it
+        did not exist and was created.
+        """
+
+        try:
+            return GlobalId.objects.get(key=element.get(ATTR_KEY),
+                                        key_type=element.get(ATTR_KEY_TYPE),
+                                        local_id=int(element.get(
+                                            ATTR_LOCAL_ID))), True
+        except GlobalId.DoesNotExist:
+            return GlobalId(key=element.get(ATTR_KEY),
+                            key_type=element.get(ATTR_KEY_TYPE),
+                            local_id=int(element.get(ATTR_LOCAL_ID))), False
+
+
+class Signature(models.Model):
+    class Meta:
+        app_label = 'boards'
+
+    def __init__(self, *args, **kwargs):
+        models.Model.__init__(self, *args, **kwargs)
+
+        if 'key' in kwargs and 'key_type' in kwargs and 'signature' in kwargs:
+            self.key_type = kwargs['key_type']
+            self.key = kwargs['key']
+            self.signature = kwargs['signature']
+
+    key_type = models.TextField()
+    key = models.TextField()
+    signature = models.TextField()
+
+    global_id = models.ForeignKey('GlobalId')

boards/models/sync_key.py

@@ -0,0 +1,61 b''
+import base64
+from ecdsa import SigningKey, VerifyingKey, BadSignatureError
+from django.db import models
+
+TYPE_ECDSA = 'ecdsa'
+
+APP_LABEL_BOARDS = 'boards'
+
+
+class KeyPairManager(models.Manager):
+    def generate_key(self, key_type=TYPE_ECDSA, primary=False):
+        if primary and self.filter(primary=True).exists():
+            raise Exception('There can be only one primary key')
+
+        if key_type == TYPE_ECDSA:
+            private = SigningKey.generate()
+            public = private.get_verifying_key()
+
+            private_key_str = base64.b64encode(private.to_string()).decode()
+            public_key_str = base64.b64encode(public.to_string()).decode()
+
+            return self.create(public_key=public_key_str,
+                               private_key=private_key_str,
+                               key_type=TYPE_ECDSA, primary=primary)
+        else:
+            raise Exception('Key type not supported')
+
+    def verify(self, signature, string):
+        if signature.key_type == TYPE_ECDSA:
+            public = VerifyingKey.from_string(base64.b64decode(signature.key))
+            signature_byte = base64.b64decode(signature.signature)
+            try:
+                return public.verify(signature_byte, string.encode())
+            except BadSignatureError:
+                return False
+        else:
+            raise Exception('Key type not supported')
+
+    def has_primary(self):
+        return self.filter(primary=True).exists()
+
+
+class KeyPair(models.Model):
+    class Meta:
+        app_label = APP_LABEL_BOARDS
+
+    objects = KeyPairManager()
+
+    public_key = models.TextField()
+    private_key = models.TextField()
+    key_type = models.TextField()
+    primary = models.BooleanField(default=False)
+
+    def __str__(self):
+        return '%s::%s' % (self.key_type, self.public_key)
+
+    def sign(self, string):
+        private = SigningKey.from_string(base64.b64decode(
+            self.private_key.encode()))
+        signature_byte = private.sign_deterministic(string.encode())
+        return base64.b64encode(signature_byte).decode()

boards/signals.py

@@ -0,0 +1,89 b''
+import re
+from boards.mdx_neboard import get_parser
+
+from boards.models import Post, GlobalId
+from boards.models.post import REGEX_NOTIFICATION
+from boards.models.post import REGEX_REPLY, REGEX_GLOBAL_REPLY
+from boards.models.user import Notification
+from django.db.models.signals import post_save, pre_save, pre_delete, \
+    post_delete
+from django.dispatch import receiver
+from django.utils import timezone
+
+
+@receiver(post_save, sender=Post)
+def connect_replies(instance, **kwargs):
+    for reply_number in re.finditer(REGEX_REPLY, instance.get_raw_text()):
+        post_id = reply_number.group(1)
+
+        try:
+            referenced_post = Post.objects.get(id=post_id)
+
+            if not referenced_post.referenced_posts.filter(
+                    id=instance.id).exists():
+                referenced_post.referenced_posts.add(instance)
+                referenced_post.last_edit_time = instance.pub_time
+                referenced_post.build_refmap()
+                referenced_post.save(update_fields=['refmap', 'last_edit_time'])
+        except Post.DoesNotExist:
+            pass
+
+
+@receiver(post_save, sender=Post)
+def connect_global_replies(instance, **kwargs):
+    for reply_number in re.finditer(REGEX_GLOBAL_REPLY, instance.get_raw_text()):
+        key_type = reply_number.group(1)
+        key = reply_number.group(2)
+        local_id = reply_number.group(3)
+
+        try:
+            global_id = GlobalId.objects.get(key_type=key_type, key=key,
+                    local_id=local_id)
+            referenced_post = Post.objects.get(global_id=global_id)
+            referenced_post.referenced_posts.add(instance)
+            referenced_post.last_edit_time = instance.pub_time
+            referenced_post.build_refmap()
+            referenced_post.save(update_fields=['refmap', 'last_edit_time'])
+        except (GlobalId.DoesNotExist, Post.DoesNotExist):
+            pass
+
+
+@receiver(post_save, sender=Post)
+def connect_notifications(instance, **kwargs):
+    for reply_number in re.finditer(REGEX_NOTIFICATION, instance.get_raw_text()):
+        user_name = reply_number.group(1).lower()
+        Notification.objects.get_or_create(name=user_name, post=instance)
+
+
+@receiver(pre_save, sender=Post)
+def preparse_text(instance, **kwargs):
+    instance._text_rendered = get_parser().parse(instance.get_raw_text())
+
+
+@receiver(pre_delete, sender=Post)
+def delete_images(instance, **kwargs):
+    for image in instance.images.all():
+        image_refs_count = image.post_images.count()
+        if image_refs_count == 1:
+            image.delete()
+
+
+@receiver(pre_delete, sender=Post)
+def delete_attachments(instance, **kwargs):
+    for attachment in instance.attachments.all():
+        attachment_refs_count = attachment.attachment_posts.count()
+        if attachment_refs_count == 1:
+            attachment.delete()
+
+
+@receiver(post_delete, sender=Post)
+def update_thread_on_delete(instance, **kwargs):
+    thread = instance.get_thread()
+    thread.last_edit_time = timezone.now()
+    thread.save()
+
+
+@receiver(post_delete, sender=Post)
+def delete_global_id(instance, **kwargs):
+    if instance.global_id and instance.global_id.id:
+        instance.global_id.delete()

boards/tests/test_keys.py

@@ -0,0 +1,89 b''
+from base64 import b64encode
+import logging
+
+from django.test import TestCase
+from boards.models import KeyPair, GlobalId, Post, Signature
+from boards.models.post.sync import SyncManager
+
+logger = logging.getLogger(__name__)
+
+
+class KeyTest(TestCase):
+    def test_create_key(self):
+        key = KeyPair.objects.generate_key('ecdsa')
+
+        self.assertIsNotNone(key, 'The key was not created.')
+
+    def test_validation(self):
+        key = KeyPair.objects.generate_key(key_type='ecdsa')
+        message = 'msg'
+        signature_value = key.sign(message)
+
+        signature = Signature(key_type='ecdsa', key=key.public_key,
+                              signature=signature_value)
+        valid = KeyPair.objects.verify(signature, message)
+
+        self.assertTrue(valid, 'Message verification failed.')
+
+    def test_primary_constraint(self):
+        KeyPair.objects.generate_key(key_type='ecdsa', primary=True)
+
+        with self.assertRaises(Exception):
+            KeyPair.objects.generate_key(key_type='ecdsa', primary=True)
+
+    def test_model_id_save(self):
+        model_id = GlobalId(key_type='test', key='test key', local_id='1')
+        model_id.save()
+
+    def test_request_get(self):
+        post = self._create_post_with_key()
+
+        request = GlobalId.objects.generate_request_get([post.global_id])
+        logger.debug(request)
+
+        key = KeyPair.objects.get(primary=True)
+        self.assertTrue('<request type="get" version="1.0">'
+                        '<model name="post" version="1.0">'
+                        '<id key="%s" local-id="1" type="%s" />'
+                        '</model>'
+                        '</request>' % (
+                            key.public_key,
+                            key.key_type,
+                        ) in request,
+                        'Wrong XML generated for the GET request.')
+
+    def test_response_get(self):
+        post = self._create_post_with_key()
+        reply_post = Post.objects.create_post(title='test_title',
+                                              text='[post]%d[/post]' % post.id,
+                                              thread=post.get_thread())
+
+        response = SyncManager.generate_response_get([reply_post])
+        logger.debug(response)
+
+        key = KeyPair.objects.get(primary=True)
+        self.assertTrue('<status>success</status>'
+                        '<models>'
+                        '<model name="post">'
+                        '<content>'
+                        '<id key="%s" local-id="%d" type="%s" />'
+                        '<title>test_title</title>'
+                        '<text>[post]%s[/post]</text>'
+                        '<thread><id key="%s" local-id="%d" type="%s" /></thread>'
+                        '<pub-time>%s</pub-time>'
+                        '</content>' % (
+                            key.public_key,
+                            reply_post.id,
+                            key.key_type,
+                            str(post.global_id),
+                            key.public_key,
+                            post.id,
+                            key.key_type,
+                            str(reply_post.get_pub_time_str()),
+                        ) in response,
+                        'Wrong XML generated for the GET response.')
+
+    def _create_post_with_key(self):
+        KeyPair.objects.generate_key(primary=True)
+
+        return Post.objects.create_post(title='test_title', text='test_text')

boards/tests/test_sync.py

@@ -0,0 +1,102 b''
+from boards.models import KeyPair, Post, Tag
+from boards.models.post.sync import SyncManager
+from boards.tests.mocks import MockRequest
+from boards.views.sync import response_get
+
+__author__ = 'neko259'
+
+
+from django.test import TestCase
+
+
+class SyncTest(TestCase):
+    def test_get(self):
+        """
+        Forms a GET request of a post and checks the response.
+        """
+
+        key = KeyPair.objects.generate_key(primary=True)
+        tag = Tag.objects.create(name='tag1')
+        post = Post.objects.create_post(title='test_title',
+                                        text='test_text\rline two',
+                                        tags=[tag])
+
+        request = MockRequest()
+        request.body = (
+            '<request type="get" version="1.0">'
+            '<model name="post" version="1.0">'
+            '<id key="%s" local-id="%d" type="%s" />'
+            '</model>'
+            '</request>' % (post.global_id.key,
+                            post.id,
+                            post.global_id.key_type)
+        )
+
+        response = response_get(request).content.decode()
+        self.assertTrue(
+            '<status>success</status>'
+            '<models>'
+            '<model name="post">'
+            '<content>'
+            '<id key="%s" local-id="%d" type="%s" />'
+            '<title>%s</title>'
+            '<text>%s</text>'
+            '<tags><tag>%s</tag></tags>'
+            '<pub-time>%s</pub-time>'
+            '</content>' % (
+                post.global_id.key,
+                post.global_id.local_id,
+                post.global_id.key_type,
+                post.title,
+                post.get_sync_text(),
+                post.get_thread().get_tags().first().name,
+                post.get_pub_time_str(),
+            ) in response,
+            'Wrong response generated for the GET request.')
+
+        post.delete()
+        key.delete()
+
+        KeyPair.objects.generate_key(primary=True)
+
+        SyncManager.parse_response_get(response, None)
+        self.assertEqual(1, Post.objects.count(),
+                         'Post was not created from XML response.')
+
+        parsed_post = Post.objects.first()
+        self.assertEqual('tag1',
+                         parsed_post.get_thread().get_tags().first().name,
+                         'Invalid tag was parsed.')
+
+        SyncManager.parse_response_get(response, None)
+        self.assertEqual(1, Post.objects.count(),
+                         'The same post was imported twice.')
+
+        self.assertEqual(1, parsed_post.global_id.signature_set.count(),
+                         'Signature was not saved.')
+
+        post = parsed_post
+
+        # Trying to sync the same once more
+        response = response_get(request).content.decode()
+
+        self.assertTrue(
+            '<status>success</status>'
+            '<models>'
+            '<model name="post">'
+            '<content>'
+            '<id key="%s" local-id="%d" type="%s" />'
+            '<title>%s</title>'
+            '<text>%s</text>'
+            '<tags><tag>%s</tag></tags>'
+            '<pub-time>%s</pub-time>'
+            '</content>' % (
+                post.global_id.key,
+                post.global_id.local_id,
+                post.global_id.key_type,
+                post.title,
+                post.get_sync_text(),
+                post.get_thread().get_tags().first().name,
+                post.get_pub_time_str(),
+            ) in response,
+            'Wrong response generated for the GET request.')

boards/views/sync.py

@@ -0,0 +1,62 b''
+import xml.etree.ElementTree as et
+import xml.dom.minidom
+
+from django.http import HttpResponse, Http404
+from boards.models import GlobalId, Post
+from boards.models.post.sync import SyncManager
+
+
+def response_pull(request):
+    request_xml = request.body
+
+    if request_xml is None:
+        return HttpResponse(content='Use the API')
+
+    response_xml = SyncManager.generate_response_pull()
+
+    return HttpResponse(content=response_xml)
+
+
+def response_get(request):
+    """
+    Processes a GET request with post ID list and returns the posts XML list.
+    Request should contain an 'xml' post attribute with the actual request XML.
+    """
+
+    request_xml = request.body
+
+    if request_xml is None:
+        return HttpResponse(content='Use the API')
+
+    posts = []
+
+    root_tag = et.fromstring(request_xml)
+    model_tag = root_tag[0]
+    for id_tag in model_tag:
+        global_id, exists = GlobalId.from_xml_element(id_tag)
+        if exists:
+            posts.append(Post.objects.get(global_id=global_id))
+
+    response_xml = SyncManager.generate_response_get(posts)
+
+    return HttpResponse(content=response_xml)
+
+
+def get_post_sync_data(request, post_id):
+    try:
+        post = Post.objects.get(id=post_id)
+    except Post.DoesNotExist:
+        raise Http404()
+
+    xml_str = SyncManager.generate_response_get([post])
+
+    xml_repr = xml.dom.minidom.parseString(xml_str)
+    xml_repr = xml_repr.toprettyxml()
+
+    content = '=Global ID=\n%s\n\n=XML=\n%s' \
+              % (post.global_id, xml_repr)
+
+    return HttpResponse(
+        content_type='text/plain',
+        content=content,
+    )
  No newline at end of file

development/run.sh

@@ -0,0 +1,3 b''
+#! /usr/bin/env sh
+
+python3 manage.py runserver [::]:8000

development/test.sh

@@ -0,0 +1,3 b''
+#! /usr/bin/env sh
+
+python3 manage.py test

docs/dip-1.markdown

@@ -0,0 +1,203 b''
+# 0 Title #
+
+DIP-1 Common protocol description
+
+# 1 Intro #
+
+This document describes the Data Interchange Protocol (DIP), designed to
+exchange filtered data that can be stored as a graph structure between
+network nodes.
+
+# 2 Purpose #
+
+This protocol will be used to share the models (originally imageboard posts)
+across multiple servers. The main differnce of this protocol is that the node
+can specify what models it wants to get and from whom. The nodes can get
+models from a specific server, or from all except some specific servers. Also
+the models can be filtered by timestamps or tags.
+
+# 3 Protocol description #
+
+The node requests other node's changes list since some time (since epoch if
+this is the start). The other node sends a list of post ids or posts in the
+XML format.
+
+Protocol version is the version of the sync api. Model version is the version
+of data models. If at least one of them is different, the sync cannot be
+performed.
+
+The node signs the data with its keys. The receiving node saves the key at the
+first sync and checks it every time. If the key has changed, the info won't be
+saved from the node (or the node id must be changed). A model can be signed 
+with several keys but at least one of them must be the same as in the global 
+ID to verify the sender.
+
+Each node can have several keys. Nodes can have shared keys to serve as a pool
+(several nodes with the same key).
+
+Each post has an ID in the unique format: key-type::key::local-id
+
+All requests pass a request type, protocol and model versions, and a list of
+optional arguments used for filtering.
+
+Each request has its own version. Version consists of 2 numbers: first is
+incompatible version (1.3 and 2.0 are not compatible and must not be in sync)
+and the second one is minor and compatible (for example, new optional field
+is added which will be igroned by those who don't support it yet).
+
+Post edits and reflinks are not saved to the sync model. The replied post ID 
+can be got from the post text, and reflinks can be computed when loading 
+posts. The edit time is not saved because a foreign post can be 'edited' (new
+replies are added) but the signature must not change (so we can't update the
+content). The inner posts can be edited, and the signature will change then
+but the local-id won't, so the other node can detect that and replace the post
+instead of adding a new one.
+
+## 3.1 Requests ##
+
+There is no constraint on how the server should calculate the request. The
+server can return any information by any filter and the requesting node is
+responsible for validating it.
+
+The server is required to return the status of request. See 3.2 for details.
+
+### 3.1.1 pull ###
+
+"pull" request gets the desired model id list by the given filter (e.g. thread, tags,
+author)
+
+Sample request is as follows:
+
+    <?xml version="1.1" encoding="UTF-8" ?>
+    <request version="1.0" type="pull">
+        <model version="1.0" name="post">
+            <timestamp_from>0</timestamp_from>
+            <timestamp_to>0</timestamp_to>
+            <tags>
+                <tag>tag1</tag>
+            </tags>
+            <sender>
+                <allow>
+                    <key>abcehy3h9t</key>
+                    <key>ehoehyoe</key>
+                </allow>
+                <!-- There can be only allow block (all other are denied) or deny block (all other are allowed) -->
+            </sender>
+        </model>
+    </request>
+
+Under the <model> tag there are filters. Filters for the "post" model can
+be found in DIP-2.
+
+Sample response:
+
+    <?xml version="1.1" encoding="UTF-8" ?>
+    <response>
+        <status>success</status>
+        <models>
+            <id key="id1" type="ecdsa" local-id="1" />
+            <id key="id1" type="ecdsa" local-id="2" />
+            <id key="id2" type="ecdsa" local-id="1" />
+            <id key="id2" type="ecdsa" local-id="5" />
+        </models>
+    </response>
+
+### 3.1.2 get ###
+
+"get" gets models by id list.
+
+Sample request:
+
+    <?xml version="1.1" encoding="UTF-8" ?>
+    <request version="1.0" type="get">
+        <model version="1.0" name="post">
+            <id key="id1" type="ecdsa" local-id="1" />
+            <id key="id1" type="ecdsa" local-id="2" />
+        </model>
+    </request>
+
+Id consists of a key, key type and local id. This key is used for signing and
+validating of data in the model content.
+
+Sample response:
+
+    <?xml version="1.1" encoding="UTF-8" ?>
+    <response>
+        <!--
+             Valid statuses are 'success' and 'error'.
+        -->
+        <status>success</status>
+        <models>
+            <model name="post">
+                <!--
+                    Content tag is the data that is signed by signatures and must
+                    not be changed for the post from other node.
+                -->
+                <content>
+                    <id key="id1" type="ecdsa" local-id="1" />
+                    <title>13</title>
+                    <text>Thirteen</text>
+                    <thread><id key="id1" type="ecdsa" local-id="2" /></thread>
+                    <pub-time>12</pub-time>
+                    <!--
+                        Images are saved as attachments and included in the
+                        signature.
+                    -->
+                    <attachments>
+                        <attachment mimetype="image/png">TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5I</attachment>
+                    </attachments>
+                </content>
+                <!--
+                    There can be several signatures for one model. At least one
+                    signature must be made with the key used in global ID.
+                 -->
+                <signatures>
+                    <signature key="id1" type="ecdsa" value="dhefhtreh" />
+                    <signature key="id45" type="ecdsa" value="dsgfgdhefhtreh" />
+                </signatures>
+                <attachment-refs>
+                    <attachment-ref ref="TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5IGhpcyByZWFzb24sIGJ1dCBieSB0"
+                        url="/media/images/12345.png" />
+                </attachment-refs>
+            </model>
+            <model name="post">
+                <content>
+                    <id key="id1" type="ecdsa" local-id="id2" />
+                    <title>13</title>
+                    <text>Thirteen</text>
+                    <pub-time>12</pub-time>
+                    <edit-time>13</edit-time>
+                    <tags>
+                        <tag>tag1</tag>
+                    </tags>
+                </content>
+                <signatures>
+                    <signature key="id2" type="ecdsa" value="dehdfh" />
+                </signatures>
+            </model>
+        </models>
+    </response>
+
+### 3.1.3 put ###
+
+"put" gives a model to the given node (you have no guarantee the node takes
+it, consider you are just advising the node to take your post). This request
+type is useful in pool where all the nodes try to duplicate all of their data
+across the pool.
+
+## 3.2 Responses ##
+
+### 3.2.1 "not supported" ###
+
+If the request if completely not supported, a "not supported" status will be
+returned.
+
+### 3.2.2 "success" ###
+
+"success" status means the request was processed and the result is returned.
+
+### 3.2.3 "error" ###
+
+If the server knows for sure that the operation cannot be processed, it sends
+the "error" status. Additional tags describing the error may be <description>
+and <stack>.

docs/dip-2.markdown

@@ -0,0 +1,29 b''
+# 0 Title #
+
+"post" model reference
+
+# 1 Description #
+
+"post" is a model that defines an imageboard message, or post.
+
+# 2 Fields #
+
+# 2.1 Mandatory fields #
+
+* title -- text field.
+* text -- text field.
+* pub-time -- timestamp (TBD: Define format).
+
+# 2.2 Optional fields #
+
+* attachments -- defines attachments such as images.
+* attachment -- contains and attachment or link to attachment to be downloaded
+manually. Required attributes are mimetype and name.
+
+This field is used for the opening post (thread):
+
+* tags -- text tag name list.
+
+This field is used for non-opening post:
+
+* thread -- ID of a post model that this post is related to.

boards/__init__.py

@@ -0,0 +1,1 b''
+default_app_config = 'boards.apps.BoardsAppConfig'
  No newline at end of file

boards/admin.py

@@ -1,114 +1,127 b''
 from django.contrib import admin
 from django.utils.translation import ugettext_lazy as _
 from django.core.urlresolvers import reverse
-from boards.models import Post, Tag, Ban, Thread, Banner, PostImage
+from boards.models import Post, Tag, Ban, Thread, Banner, PostImage, KeyPair, GlobalId
 
 
 @admin.register(Post)
 class PostAdmin(admin.ModelAdmin):
 
     list_display = ('id', 'title', 'text', 'poster_ip', 'linked_images')
     list_filter = ('pub_time',)
     search_fields = ('id', 'title', 'text', 'poster_ip')
     exclude = ('referenced_posts', 'refmap', 'images')
     readonly_fields = ('poster_ip', 'threads', 'thread', 'linked_images',
             'attachments', 'uid', 'url', 'pub_time', 'opening')
 
     def ban_poster(self, request, queryset):
         bans = 0
         for post in queryset:
             poster_ip = post.poster_ip
             ban, created = Ban.objects.get_or_create(ip=poster_ip)
             if created:
                 bans += 1
         self.message_user(request, _('{} posters were banned').format(bans))
 
     def ban_with_hiding(self, request, queryset):
         bans = 0
         hidden = 0
         for post in queryset:
             poster_ip = post.poster_ip
             ban, created = Ban.objects.get_or_create(ip=poster_ip)
             if created:
                 bans += 1
             posts = Post.objects.filter(poster_ip=poster_ip, id__gte=post.id)
             hidden += posts.count()
             posts.update(hidden=True)
         self.message_user(request, _('{} posters were banned, {} messages were hidden').format(bans, hidden))
 
     def linked_images(self, obj: Post):
         images = obj.images.all()
         image_urls = ['<a href="{}"><img src="{}" /></a>'.format(
             reverse('admin:%s_%s_change' % (image._meta.app_label,
                                             image._meta.model_name),
                     args=[image.id]), image.image.url_200x150) for image in images]
         return ', '.join(image_urls)
     linked_images.allow_tags = True
 
 
     actions = ['ban_poster', 'ban_with_hiding']
 
 
 @admin.register(Tag)
 class TagAdmin(admin.ModelAdmin):
 
     def thread_count(self, obj: Tag) -> int:
         return obj.get_thread_count()
 
     def display_children(self, obj: Tag):
         return ', '.join([str(child) for child in obj.get_children().all()])
 
     def save_model(self, request, obj, form, change):
         super().save_model(request, obj, form, change)
         for thread in obj.get_threads().all():
             thread.refresh_tags()
-
     list_display = ('name', 'thread_count', 'display_children')
     search_fields = ('name',)
 
 
 @admin.register(Thread)
 class ThreadAdmin(admin.ModelAdmin):
 
     def title(self, obj: Thread) -> str:
         return obj.get_opening_post().get_title()
 
     def reply_count(self, obj: Thread) -> int:
         return obj.get_reply_count()
 
     def ip(self, obj: Thread):
         return obj.get_opening_post().poster_ip
 
     def display_tags(self, obj: Thread):
         return ', '.join([str(tag) for tag in obj.get_tags().all()])
 
     def op(self, obj: Thread):
         return obj.get_opening_post_id()
 
     # Save parent tags when editing tags
     def save_related(self, request, form, formsets, change):
         super().save_related(request, form, formsets, change)
         form.instance.refresh_tags()
-
     list_display = ('id', 'op', 'title', 'reply_count', 'status', 'ip',
             'display_tags')
     list_filter = ('bump_time', 'status')
     search_fields = ('id', 'title')
     filter_horizontal = ('tags',)
 
 
[e-mail](KeyPair)
+class KeyPairAdmin(admin.ModelAdmin):
+    list_display = ('public_key', 'primary')
+    list_filter = ('primary',)
+    search_fields = ('public_key',)
+
+
 @admin.register(Ban)
 class BanAdmin(admin.ModelAdmin):
     list_display = ('ip', 'can_read')
     list_filter = ('can_read',)
     search_fields = ('ip',)
 
 
 @admin.register(Banner)
 class BannerAdmin(admin.ModelAdmin):
     list_display = ('title', 'text')
 
 
 @admin.register(PostImage)
 class PostImageAdmin(admin.ModelAdmin):
     search_fields = ('alias',)
+
+
[e-mail](GlobalId)
+class GlobalIdAdmin(admin.ModelAdmin):
+    def is_linked(self, obj):
+        return Post.objects.filter(global_id=obj).exists()
+
+    list_display = ('__str__', 'is_linked',)
  No newline at end of file

boards/forms.py

@@ -1,469 +1,464 b''
 import hashlib
 import re
 import time
 import logging
 
 import pytz
 
 from django import forms
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.core.exceptions import ObjectDoesNotExist
 from django.forms.utils import ErrorList
 from django.utils.translation import ugettext_lazy as _, ungettext_lazy
 from django.utils import timezone
 
 from boards.abstracts.settingsmanager import get_settings_manager
 from boards.abstracts.attachment_alias import get_image_by_alias
 from boards.mdx_neboard import formatters
-from boards.models.attachment.downloaders import Downloader
+from boards.models.attachment.downloaders import download
 from boards.models.post import TITLE_MAX_LENGTH
 from boards.models import Tag, Post
 from boards.utils import validate_file_size, get_file_mimetype, \
     FILE_EXTENSION_DELIMITER
 from neboard import settings
 import boards.settings as board_settings
 import neboard
 
 POW_HASH_LENGTH = 16
 POW_LIFE_MINUTES = 5
 
 REGEX_TAGS = re.compile(r'^[\w\s\d]+$', re.UNICODE)
 REGEX_USERNAMES = re.compile(r'^[\w\s\d,]+$', re.UNICODE)
 
 VETERAN_POSTING_DELAY = 5
 
 ATTRIBUTE_PLACEHOLDER = 'placeholder'
 ATTRIBUTE_ROWS = 'rows'
 
 LAST_POST_TIME = 'last_post_time'
 LAST_LOGIN_TIME = 'last_login_time'
 TEXT_PLACEHOLDER = _('Type message here. Use formatting panel for more advanced usage.')
 TAGS_PLACEHOLDER = _('music images i_dont_like_tags')
 
 LABEL_TITLE = _('Title')
 LABEL_TEXT = _('Text')
 LABEL_TAG = _('Tag')
 LABEL_SEARCH = _('Search')
 
 ERROR_SPEED = 'Please wait %(delay)d second before sending message'
 ERROR_SPEED_PLURAL = 'Please wait %(delay)d seconds before sending message'
 
 TAG_MAX_LENGTH = 20
 
 TEXTAREA_ROWS = 4
 
 TRIPCODE_DELIM = '#'
 
 # TODO Maybe this may be converted into the database table?
 MIMETYPE_EXTENSIONS = {
     'image/jpeg': 'jpeg',
     'image/png': 'png',
     'image/gif': 'gif',
     'video/webm': 'webm',
     'application/pdf': 'pdf',
     'x-diff': 'diff',
     'image/svg+xml': 'svg',
     'application/x-shockwave-flash': 'swf',
     'image/x-ms-bmp': 'bmp',
     'image/bmp': 'bmp',
 }
 
 
 def get_timezones():
     timezones = []
     for tz in pytz.common_timezones:
         timezones.append((tz, tz),)
     return timezones
 
 
 class FormatPanel(forms.Textarea):
     """
     Panel for text formatting. Consists of buttons to add different tags to the
     form text area.
     """
 
     def render(self, name, value, attrs=None):
         output = '<div id="mark-panel">'
         for formatter in formatters:
             output += '<span class="mark_btn"' + \
                       ' onClick="addMarkToMsg(\'' + formatter.format_left + \
                       '\', \'' + formatter.format_right + '\')">' + \
                       formatter.preview_left + formatter.name + \
                       formatter.preview_right + '</span>'
 
         output += '</div>'
         output += super(FormatPanel, self).render(name, value, attrs=attrs)
 
         return output
 
 
 class PlainErrorList(ErrorList):
     def __unicode__(self):
         return self.as_text()
 
     def as_text(self):
         return ''.join(['(!) %s ' % e for e in self])
 
 
 class NeboardForm(forms.Form):
     """
     Form with neboard-specific formatting.
     """
 
     def as_div(self):
         """
         Returns this form rendered as HTML <as_div>s.
         """
 
         return self._html_output(
             # TODO Do not show hidden rows in the list here
             normal_row='<div class="form-row">'
                        '<div class="form-label">'
                        '%(label)s'
                        '</div>'
                        '<div class="form-input">'
                        '%(field)s'
                        '</div>'
                        '</div>'
                        '<div class="form-row">'
                        '%(help_text)s'
                        '</div>',
             error_row='<div class="form-row">'
                       '<div class="form-label"></div>'
                       '<div class="form-errors">%s</div>'
                       '</div>',
             row_ender='</div>',
             help_text_html='%s',
             errors_on_separate_row=True)
 
     def as_json_errors(self):
         errors = []
 
         for name, field in list(self.fields.items()):
             if self[name].errors:
                 errors.append({
                     'field': name,
                     'errors': self[name].errors.as_text(),
                 })
 
         return errors
 
 
 class PostForm(NeboardForm):
 
     title = forms.CharField(max_length=TITLE_MAX_LENGTH, required=False,
                             label=LABEL_TITLE,
                             widget=forms.TextInput(
                                 attrs={ATTRIBUTE_PLACEHOLDER:
                                            'test#tripcode'}))
     text = forms.CharField(
         widget=FormatPanel(attrs={
             ATTRIBUTE_PLACEHOLDER: TEXT_PLACEHOLDER,
             ATTRIBUTE_ROWS: TEXTAREA_ROWS,
         }),
         required=False, label=LABEL_TEXT)
     file = forms.FileField(required=False, label=_('File'),
                              widget=forms.ClearableFileInput(
                                  attrs={'accept': 'file/*'}))
     file_url = forms.CharField(required=False, label=_('File URL'),
                                 widget=forms.TextInput(
                                     attrs={ATTRIBUTE_PLACEHOLDER:
                                                'http://example.com/image.png'}))
 
     # This field is for spam prevention only
     email = forms.CharField(max_length=100, required=False, label=_('e-mail'),
                             widget=forms.TextInput(attrs={
                                 'class': 'form-email'}))
     threads = forms.CharField(required=False, label=_('Additional threads'),
             widget=forms.TextInput(attrs={ATTRIBUTE_PLACEHOLDER:
                 '123 456 789'}))
 
     guess = forms.CharField(widget=forms.HiddenInput(), required=False)
     timestamp = forms.CharField(widget=forms.HiddenInput(), required=False)
     iteration = forms.CharField(widget=forms.HiddenInput(), required=False)
 
     session = None
     need_to_ban = False
     image = None
 
     def _update_file_extension(self, file):
         if file:
             mimetype = get_file_mimetype(file)
             extension = MIMETYPE_EXTENSIONS.get(mimetype)
             if extension:
                 filename = file.name.split(FILE_EXTENSION_DELIMITER, 1)[0]
                 new_filename = filename + FILE_EXTENSION_DELIMITER + extension
 
                 file.name = new_filename
             else:
                 logger = logging.getLogger('boards.forms.extension')
 
                 logger.info('Unrecognized file mimetype: {}'.format(mimetype))
 
     def clean_title(self):
         title = self.cleaned_data['title']
         if title:
             if len(title) > TITLE_MAX_LENGTH:
                 raise forms.ValidationError(_('Title must have less than %s '
                                               'characters') %
                                             str(TITLE_MAX_LENGTH))
         return title
 
     def clean_text(self):
         text = self.cleaned_data['text'].strip()
         if text:
             max_length = board_settings.get_int('Forms', 'MaxTextLength')
             if len(text) > max_length:
                 raise forms.ValidationError(_('Text must have less than %s '
                                               'characters') % str(max_length))
         return text
 
     def clean_file(self):
         file = self.cleaned_data['file']
 
         if file:
             validate_file_size(file.size)
         self._update_file_extension(file)
 
         return file
 
     def clean_file_url(self):
         url = self.cleaned_data['file_url']
 
         file = None
 
         if url:
             file = get_image_by_alias(url, self.session)
             self.image = file
 
             if file is not None:
                 return
 
             if file is None:
                 file = self._get_file_from_url(url)
                 if not file:
                     raise forms.ValidationError(_('Invalid URL'))
                 else:
                     validate_file_size(file.size)
         self._update_file_extension(file)
 
         return file
 
     def clean_threads(self):
         threads_str = self.cleaned_data['threads']
 
         if len(threads_str) > 0:
             threads_id_list = threads_str.split(' ')
 
             threads = list()
 
             for thread_id in threads_id_list:
                 try:
                     thread = Post.objects.get(id=int(thread_id))
                     if not thread.is_opening() or thread.get_thread().is_archived():
                         raise ObjectDoesNotExist()
                     threads.append(thread)
                 except (ObjectDoesNotExist, ValueError):
                     raise forms.ValidationError(_('Invalid additional thread list'))
 
             return threads
 
     def clean(self):
         cleaned_data = super(PostForm, self).clean()
 
         if cleaned_data['email']:
             self.need_to_ban = True
             raise forms.ValidationError('A human cannot enter a hidden field')
 
         if not self.errors:
             self._clean_text_file()
 
         limit_speed = board_settings.get_bool('Forms', 'LimitPostingSpeed')
 
         settings_manager = get_settings_manager(self)
         if not self.errors and limit_speed and not settings_manager.get_setting('confirmed_user'):
             pow_difficulty = board_settings.get_int('Forms', 'PowDifficulty')
             if pow_difficulty > 0:
                 # Limit only first post
                 if cleaned_data['timestamp'] \
                         and cleaned_data['iteration'] and cleaned_data['guess'] \
                         and not settings_manager.get_setting('confirmed_user'):
                     self._validate_hash(cleaned_data['timestamp'], cleaned_data['iteration'], cleaned_data['guess'], cleaned_data['text'])
             else:
                 # Limit every post
                 self._validate_posting_speed()
             settings_manager.set_setting('confirmed_user', True)
 
 
         return cleaned_data
 
     def get_file(self):
         """
         Gets file from form or URL.
         """
 
         file = self.cleaned_data['file']
         return file or self.cleaned_data['file_url']
 
     def get_tripcode(self):
         title = self.cleaned_data['title']
         if title is not None and TRIPCODE_DELIM in title:
             code = title.split(TRIPCODE_DELIM, maxsplit=1)[1] + neboard.settings.SECRET_KEY
             tripcode = hashlib.md5(code.encode()).hexdigest()
         else:
             tripcode = ''
         return tripcode
 
     def get_title(self):
         title = self.cleaned_data['title']
         if title is not None and TRIPCODE_DELIM in title:
             return title.split(TRIPCODE_DELIM, maxsplit=1)[0]
         else:
             return title
 
     def get_images(self):
         if self.image:
             return [self.image]
         else:
             return []
 
     def _clean_text_file(self):
         text = self.cleaned_data.get('text')
         file = self.get_file()
         images = self.get_images()
 
         if (not text) and (not file) and len(images) == 0:
             error_message = _('Either text or file must be entered.')
             self._errors['text'] = self.error_class([error_message])
 
     def _validate_posting_speed(self):
         can_post = True
 
         posting_delay = settings.POSTING_DELAY
 
         if board_settings.get_bool('Forms', 'LimitPostingSpeed'):
             now = time.time()
 
             current_delay = 0
 
             if LAST_POST_TIME not in self.session:
                 self.session[LAST_POST_TIME] = now
 
                 need_delay = True
             else:
                 last_post_time = self.session.get(LAST_POST_TIME)
                 current_delay = int(now - last_post_time)
 
                 need_delay = current_delay < posting_delay
 
             if need_delay:
                 delay = posting_delay - current_delay
                 error_message = ungettext_lazy(ERROR_SPEED, ERROR_SPEED_PLURAL,
                                                delay) % {'delay': delay}
                 self._errors['text'] = self.error_class([error_message])
 
                 can_post = False
 
             if can_post:
                 self.session[LAST_POST_TIME] = now
 
     def _get_file_from_url(self, url: str) -> SimpleUploadedFile:
         """
         Gets an file file from URL.
         """
 
         img_temp = None
 
         try:
-            for downloader in Downloader.__subclasses__():
-                if downloader.handles(url):
-                    return downloader.download(url)
-            # If nobody of the specific downloaders handles this, use generic
-            # one
-            return Downloader.download(url)
+            download(url)
         except forms.ValidationError as e:
             raise e
         except Exception as e:
             raise forms.ValidationError(e)
 
     def _validate_hash(self, timestamp: str, iteration: str, guess: str, message: str):
         post_time = timezone.datetime.fromtimestamp(
             int(timestamp[:-3]), tz=timezone.get_current_timezone())
 
         payload = timestamp + message.replace('\r\n', '\n')
         difficulty = board_settings.get_int('Forms', 'PowDifficulty')
         target = str(int(2 ** (POW_HASH_LENGTH * 3) / difficulty))
         if len(target) < POW_HASH_LENGTH:
             target = '0' * (POW_HASH_LENGTH - len(target)) + target
 
         computed_guess = hashlib.sha256((payload + iteration).encode())\
                              .hexdigest()[0:POW_HASH_LENGTH]
         if guess != computed_guess or guess > target:
             self._errors['text'] = self.error_class(
                 [_('Invalid PoW.')])
 
 
 
 class ThreadForm(PostForm):
 
     tags = forms.CharField(
         widget=forms.TextInput(attrs={ATTRIBUTE_PLACEHOLDER: TAGS_PLACEHOLDER}),
         max_length=100, label=_('Tags'), required=True)
     monochrome = forms.BooleanField(label=_('Monochrome'), required=False)
 
     def clean_tags(self):
         tags = self.cleaned_data['tags'].strip()
 
         if not tags or not REGEX_TAGS.match(tags):
             raise forms.ValidationError(
                 _('Inappropriate characters in tags.'))
 
         required_tag_exists = False
         tag_set = set()
         for tag_string in tags.split():
             tag, created = Tag.objects.get_or_create(name=tag_string.strip().lower())
             tag_set.add(tag)
 
             # If this is a new tag, don't check for its parents because nobody
             # added them yet
             if not created:
                 tag_set |= set(tag.get_all_parents())
 
         for tag in tag_set:
             if tag.required:
                 required_tag_exists = True
                 break
 
         if not required_tag_exists:
             raise forms.ValidationError(
                 _('Need at least one section.'))
 
         return tag_set
 
     def clean(self):
         cleaned_data = super(ThreadForm, self).clean()
 
         return cleaned_data
 
     def is_monochrome(self):
         return self.cleaned_data['monochrome']
 
 
 class SettingsForm(NeboardForm):
 
     theme = forms.ChoiceField(choices=settings.THEMES, label=_('Theme'))
     image_viewer = forms.ChoiceField(choices=settings.IMAGE_VIEWERS, label=_('Image view mode'))
     username = forms.CharField(label=_('User name'), required=False)
     timezone = forms.ChoiceField(choices=get_timezones(), label=_('Time zone'))
 
     def clean_username(self):
         username = self.cleaned_data['username']
 
         if username and not REGEX_USERNAMES.match(username):
             raise forms.ValidationError(_('Inappropriate characters.'))
 
         return username
 
 
 class SearchForm(NeboardForm):
     query = forms.CharField(max_length=500, label=LABEL_SEARCH, required=False)

boards/management/commands/enforce_privacy.py

@@ -1,16 +1,15 b''
 from django.core.management import BaseCommand
 from django.db import transaction
 
 from boards.models import Post
-from boards.models.post import NO_IP
-
+from boards.models.post.manager import NO_IP
 
 __author__ = 'neko259'
 
 
 class Command(BaseCommand):
     help = 'Removes user and IP data from all posts'
 
     @transaction.atomic
     def handle(self, *args, **options):
         Post.objects.all().update(poster_ip=NO_IP)
  No newline at end of file

boards/mdx_neboard.py

@@ -1,244 +1,254 b''
 # coding=utf-8
 
 import re
 import random
 import bbcode
 
 from urllib.parse import unquote
 
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.urlresolvers import reverse
 
 import boards
 
 
 __author__ = 'neko259'
 
 
 REFLINK_PATTERN = re.compile(r'^\d+$')
+GLOBAL_REFLINK_PATTERN = re.compile(r'(\w+)::([^:]+)::(\d+)')
 MULTI_NEWLINES_PATTERN = re.compile(r'(\r?\n){2,}')
 ONE_NEWLINE = '\n'
 REGEX_URL = re.compile(r'https?\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(/\S*)?')
 LINE_BREAK_HTML = '<div class="br"></div>'
 SPOILER_SPACE = '&nbsp;'
 
 MAX_SPOILER_MULTIPLIER = 2
 
 
 class TextFormatter():
     """
     An interface for formatter that can be used in the text format panel
     """
 
     def __init__(self):
         pass
 
     name = ''
 
     # Left and right tags for the button preview
     preview_left = ''
     preview_right = ''
 
     # Left and right characters for the textarea input
     format_left = ''
     format_right = ''
 
 
 class AutolinkPattern():
     def handleMatch(self, m):
         link_element = etree.Element('a')
         href = m.group(2)
         link_element.set('href', href)
         link_element.text = href
 
         return link_element
 
 
 class QuotePattern(TextFormatter):
     name = '>q'
     preview_left = '<span class="quote">'
     preview_right = '</span>'
 
     format_left = '[quote]'
     format_right = '[/quote]'
 
 
 class SpoilerPattern(TextFormatter):
     name = 'spoiler'
     preview_left = '<span class="spoiler">'
     preview_right = '</span>'
 
     format_left = '[spoiler]'
     format_right = '[/spoiler]'
 
 
 class CommentPattern(TextFormatter):
     name = ''
     preview_left = '<span class="comment">// '
     preview_right = '</span>'
 
     format_left = '[comment]'
     format_right = '[/comment]'
 
 
 # TODO Use <s> tag here
 class StrikeThroughPattern(TextFormatter):
     name = 's'
     preview_left = '<span class="strikethrough">'
     preview_right = '</span>'
 
     format_left = '[s]'
     format_right = '[/s]'
 
 
 class ItalicPattern(TextFormatter):
     name = 'i'
     preview_left = '<i>'
     preview_right = '</i>'
 
     format_left = '[i]'
     format_right = '[/i]'
 
 
 class BoldPattern(TextFormatter):
     name = 'b'
     preview_left = '<b>'
     preview_right = '</b>'
 
     format_left = '[b]'
     format_right = '[/b]'
 
 
 class CodePattern(TextFormatter):
     name = 'code'
     preview_left = '<code>'
     preview_right = '</code>'
 
     format_left = '[code]'
     format_right = '[/code]'
 
 
 def render_reflink(tag_name, value, options, parent, context):
     result = '>>%s' % value
 
+    post = None
     if REFLINK_PATTERN.match(value):
         post_id = int(value)
 
         try:
             post = boards.models.Post.objects.get(id=post_id)
 
-            result = post.get_link_view()
         except ObjectDoesNotExist:
             pass
+    elif GLOBAL_REFLINK_PATTERN.match(value):
+        match = GLOBAL_REFLINK_PATTERN.search(value)
+        try:
+            global_id = boards.models.GlobalId.objects.get(
+                key_type=match.group(1), key=match.group(2),
+                local_id=match.group(3))
+            post = global_id.post
+        except ObjectDoesNotExist:
+            pass
+
+    if post is not None:
+        result = post.get_link_view()
 
     return result
 
 
 def render_quote(tag_name, value, options, parent, context):
     source = ''
     if 'source' in options:
         source = options['source']
     elif 'quote' in options:
         source = options['quote']
 
     if source:
         result = '<div class="multiquote"><div class="quote-header">%s</div><div class="quote-text">%s</div></div>' % (source, value)
     else:
         # Insert a ">" at the start of every line
         result = '<span class="quote">&gt;{}</span>'.format(
                 value.replace(LINE_BREAK_HTML,
                     '{}&gt;'.format(LINE_BREAK_HTML)))
 
     return result
 
 
 def render_notification(tag_name, value, options, parent, content):
     username = value.lower()
 
     return '<a href="{}" class="user-cast">@{}</a>'.format(
         reverse('notifications', kwargs={'username': username}), username)
 
 
 def render_tag(tag_name, value, options, parent, context):
     tag_name = value.lower()
 
     try:
         url = boards.models.Tag.objects.get(name=tag_name).get_view()
     except ObjectDoesNotExist:
         url = tag_name
 
     return url
 
 
 def render_spoiler(tag_name, value, options, parent, context):
     text_len = len(value)
     space_count = random.randint(0, text_len * MAX_SPOILER_MULTIPLIER)
     side_spaces = SPOILER_SPACE * (space_count // 2)
     return '<span class="spoiler">{}{}{}</span>'.format(side_spaces, value,
             side_spaces)
 
-    return quote_element
-
-
 
 formatters = [
     QuotePattern,
     SpoilerPattern,
     ItalicPattern,
     BoldPattern,
     CommentPattern,
     StrikeThroughPattern,
     CodePattern,
 ]
 
 
 PREPARSE_PATTERNS = {
     r'(?<!>)>>(\d+)': r'[post]\1[/post]',  # Reflink ">>123"
     r'^>([^>].+)': r'[quote]\1[/quote]',  # Quote ">text"
     r'^//(.+)': r'[comment]\1[/comment]',  # Comment "//text"
     r'\B@(\w+)': r'[user]\1[/user]',  # User notification "@user"
 }
 
 
 class Parser:
     def __init__(self):
         # The newline hack is added because br's margin does not work in all
         # browsers except firefox, when the div's does.
         self.parser = bbcode.Parser(newline=LINE_BREAK_HTML)
 
         self.parser.add_formatter('post', render_reflink, strip=True)
         self.parser.add_formatter('quote', render_quote, strip=True)
         self.parser.add_formatter('user', render_notification, strip=True)
         self.parser.add_formatter('tag', render_tag, strip=True)
         self.parser.add_formatter('spoiler', render_spoiler, strip=True)
         self.parser.add_simple_formatter(
             'comment', '<span class="comment">//%(value)s</span>')
         self.parser.add_simple_formatter(
             's', '<span class="strikethrough">%(value)s</span>')
         # TODO Why not use built-in tag?
         self.parser.add_simple_formatter('code',
                                          '<pre><code>%(value)s</pre></code>',
                                          render_embedded=False)
 
     def preparse(self, text):
         """
         Performs manual parsing before the bbcode parser is used.
         Preparsed text is saved as raw and the text before preparsing is lost.
         """
         new_text = MULTI_NEWLINES_PATTERN.sub(ONE_NEWLINE, text)
 
         for key, value in PREPARSE_PATTERNS.items():
             new_text = re.sub(key, value, new_text, flags=re.MULTILINE)
 
         for link in REGEX_URL.findall(text):
             new_text = new_text.replace(link, unquote(link))
 
         return new_text
 
     def parse(self, text):
         return self.parser.format(text)
 
 
 parser = Parser()
 def get_parser():
     return parser

boards/models/__init__.py

@@ -1,12 +1,14 b''
 STATUS_ACTIVE = 'active'
 STATUS_BUMPLIMIT = 'bumplimit'
 STATUS_ARCHIVE = 'archived'
 
 
+from boards.models.signature import GlobalId, Signature
+from boards.models.sync_key import KeyPair
 from boards.models.image import PostImage
 from boards.models.attachment import Attachment
 from boards.models.thread import Thread
 from boards.models.post import Post
 from boards.models.tag import Tag
 from boards.models.user import Ban
 from boards.models.banner import Banner

boards/models/attachment/downloaders.py

@@ -1,70 +1,79 b''
 import os
 import re
 
 from django.core.files.uploadedfile import SimpleUploadedFile, \
     TemporaryUploadedFile
 from pytube import YouTube
 import requests
 
 from boards.utils import validate_file_size
 
 YOUTUBE_VIDEO_FORMAT = 'webm'
 
 HTTP_RESULT_OK = 200
 
 HEADER_CONTENT_LENGTH = 'content-length'
 HEADER_CONTENT_TYPE = 'content-type'
 
 FILE_DOWNLOAD_CHUNK_BYTES = 200000
 
 YOUTUBE_URL = re.compile(r'https?://((www\.)?youtube\.com/watch\?v=|youtu.be/)[-\w]+')
 
 
 class Downloader:
     @staticmethod
     def handles(url: str) -> bool:
         return False
 
     @staticmethod
     def download(url: str):
         # Verify content headers
         response_head = requests.head(url, verify=False)
         content_type = response_head.headers[HEADER_CONTENT_TYPE].split(';')[0]
         length_header = response_head.headers.get(HEADER_CONTENT_LENGTH)
         if length_header:
             length = int(length_header)
             validate_file_size(length)
         # Get the actual content into memory
         response = requests.get(url, verify=False, stream=True)
 
         # Download file, stop if the size exceeds limit
         size = 0
 
         # Set a dummy file name that will be replaced
         # anyway, just keep the valid extension
         filename = 'file.' + content_type.split('/')[1]
 
         file = TemporaryUploadedFile(filename, content_type, 0, None, None)
         for chunk in response.iter_content(FILE_DOWNLOAD_CHUNK_BYTES):
             size += len(chunk)
             validate_file_size(size)
             file.write(chunk)
 
         if response.status_code == HTTP_RESULT_OK:
             return file
 
 
+def download(url):
+    for downloader in Downloader.__subclasses__():
+        if downloader.handles(url):
+            return downloader.download(url)
+    # If nobody of the specific downloaders handles this, use generic
+    # one
+    return Downloader.download(url)
+
+
 class YouTubeDownloader(Downloader):
     @staticmethod
     def download(url: str):
         yt = YouTube()
         yt.from_url(url)
         videos = yt.filter(YOUTUBE_VIDEO_FORMAT)
         if len(videos) > 0:
             video = videos[0]
             return Downloader.download(video.url)
 
     @staticmethod
     def handles(url: str) -> bool:
         return YOUTUBE_URL.match(url)
 

boards/models/post/__init__.py

@@ -1,369 +1,381 b''
-import logging
-import re
 import uuid
 
+import re
+from boards import settings
+from boards.abstracts.tripcode import Tripcode
+from boards.models import PostImage, Attachment, KeyPair, GlobalId
+from boards.models.base import Viewable
+from boards.models.post.export import get_exporter, DIFF_TYPE_JSON
+from boards.models.post.manager import PostManager
 from boards.utils import datetime_to_epoch
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.urlresolvers import reverse
 from django.db import models
 from django.db.models import TextField, QuerySet
-from django.template.defaultfilters import striptags, truncatewords
+from django.template.defaultfilters import truncatewords, striptags
 from django.template.loader import render_to_string
-from django.utils import timezone
-from django.db.models.signals import post_save, pre_save
-from django.dispatch import receiver
-
-from boards import settings
-from boards.abstracts.tripcode import Tripcode
-from boards.mdx_neboard import get_parser
-from boards.models import PostImage, Attachment
-from boards.models.base import Viewable
-from boards.models.post.export import get_exporter, DIFF_TYPE_JSON
-from boards.models.post.manager import PostManager
-from boards.models.user import Notification
 
 CSS_CLS_HIDDEN_POST = 'hidden_post'
 CSS_CLS_DEAD_POST = 'dead_post'
 CSS_CLS_ARCHIVE_POST = 'archive_post'
 CSS_CLS_POST = 'post'
 CSS_CLS_MONOCHROME = 'monochrome'
 
 TITLE_MAX_WORDS = 10
 
 APP_LABEL_BOARDS = 'boards'
 
 BAN_REASON_AUTO = 'Auto'
 
 IMAGE_THUMB_SIZE = (200, 150)
 
 TITLE_MAX_LENGTH = 200
 
 REGEX_REPLY = re.compile(r'\[post\](\d+)\[/post\]')
+REGEX_GLOBAL_REPLY = re.compile(r'\[post\](\w+)::([^:]+)::(\d+)\[/post\]')
+REGEX_URL = re.compile(r'https?\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(/\S*)?')
 REGEX_NOTIFICATION = re.compile(r'\[user\](\w+)\[/user\]')
 
 PARAMETER_TRUNCATED = 'truncated'
 PARAMETER_TAG = 'tag'
 PARAMETER_OFFSET = 'offset'
 PARAMETER_DIFF_TYPE = 'type'
 PARAMETER_CSS_CLASS = 'css_class'
 PARAMETER_THREAD = 'thread'
 PARAMETER_IS_OPENING = 'is_opening'
 PARAMETER_POST = 'post'
 PARAMETER_OP_ID = 'opening_post_id'
 PARAMETER_NEED_OPEN_LINK = 'need_open_link'
 PARAMETER_REPLY_LINK = 'reply_link'
 PARAMETER_NEED_OP_DATA = 'need_op_data'
 
 POST_VIEW_PARAMS = (
     'need_op_data',
     'reply_link',
     'need_open_link',
     'truncated',
     'mode_tree',
     'perms',
     'tree_depth',
 )
 
 
 class Post(models.Model, Viewable):
     """A post is a message."""
 
     objects = PostManager()
 
     class Meta:
         app_label = APP_LABEL_BOARDS
         ordering = ('id',)
 
     title = models.CharField(max_length=TITLE_MAX_LENGTH, null=True, blank=True)
     pub_time = models.DateTimeField()
     text = TextField(blank=True, null=True)
     _text_rendered = TextField(blank=True, null=True, editable=False)
 
     images = models.ManyToManyField(PostImage, null=True, blank=True,
                                     related_name='post_images', db_index=True)
     attachments = models.ManyToManyField(Attachment, null=True, blank=True,
                                          related_name='attachment_posts')
 
     poster_ip = models.GenericIPAddressField()
 
     # TODO This field can be removed cause UID is used for update now
     last_edit_time = models.DateTimeField()
 
     referenced_posts = models.ManyToManyField('Post', symmetrical=False,
                                               null=True,
                                               blank=True, related_name='refposts',
                                               db_index=True)
     refmap = models.TextField(null=True, blank=True)
     threads = models.ManyToManyField('Thread', db_index=True,
                                      related_name='multi_replies')
     thread = models.ForeignKey('Thread', db_index=True, related_name='pt+')
 
     url = models.TextField()
     uid = models.TextField(db_index=True)
 
+    # Global ID with author key. If the message was downloaded from another
+    # server, this indicates the server.
+    global_id = models.OneToOneField(GlobalId, null=True, blank=True,
+                                     on_delete=models.CASCADE)
+
     tripcode = models.CharField(max_length=50, blank=True, default='')
     opening = models.BooleanField(db_index=True)
     hidden = models.BooleanField(default=False)
 
     def __str__(self):
         return 'P#{}/{}'.format(self.id, self.get_title())
 
     def get_title(self) -> str:
         return self.title
 
     def get_title_or_text(self):
         title = self.get_title()
         if not title:
             title = truncatewords(striptags(self.get_text()), TITLE_MAX_WORDS)
 
         return title
 
     def build_refmap(self) -> None:
         """
         Builds a replies map string from replies list. This is a cache to stop
         the server from recalculating the map on every post show.
         """
 
         post_urls = [refpost.get_link_view()
                      for refpost in self.referenced_posts.all()]
 
         self.refmap = ', '.join(post_urls)
 
     def is_referenced(self) -> bool:
         return self.refmap and len(self.refmap) > 0
 
     def is_opening(self) -> bool:
         """
         Checks if this is an opening post or just a reply.
         """
 
         return self.opening
 
     def get_absolute_url(self, thread=None):
         url = None
 
         if thread is None:
             thread = self.get_thread()
 
             # Url is cached only for the "main" thread. When getting url
             # for other threads, do it manually.
             if self.url:
                 url = self.url
 
         if url is None:
             opening = self.is_opening()
             opening_id = self.id if opening else thread.get_opening_post_id()
             url = reverse('thread', kwargs={'post_id': opening_id})
             if not opening:
                 url += '#' + str(self.id)
 
         return url
 
     def get_thread(self):
         return self.thread
 
     def get_thread_id(self):
         return self.thread_id
 
     def get_threads(self) -> QuerySet:
         """
         Gets post's thread.
         """
 
         return self.threads
 
     def _get_cache_key(self):
         return [datetime_to_epoch(self.last_edit_time)]
 
     def get_view(self, *args, **kwargs) -> str:
         """
         Renders post's HTML view. Some of the post params can be passed over
         kwargs for the means of caching (if we view the thread, some params
         are same for every post and don't need to be computed over and over.
         """
 
         thread = self.get_thread()
 
         css_classes = [CSS_CLS_POST]
         if thread.is_archived():
             css_classes.append(CSS_CLS_ARCHIVE_POST)
         elif not thread.can_bump():
             css_classes.append(CSS_CLS_DEAD_POST)
         if self.is_hidden():
             css_classes.append(CSS_CLS_HIDDEN_POST)
         if thread.is_monochrome():
             css_classes.append(CSS_CLS_MONOCHROME)
 
         params = dict()
         for param in POST_VIEW_PARAMS:
             if param in kwargs:
                 params[param] = kwargs[param]
 
         params.update({
             PARAMETER_POST: self,
             PARAMETER_IS_OPENING: self.is_opening(),
             PARAMETER_THREAD: thread,
             PARAMETER_CSS_CLASS: ' '.join(css_classes),
         })
 
         return render_to_string('boards/post.html', params)
 
     def get_search_view(self, *args, **kwargs):
         return self.get_view(need_op_data=True, *args, **kwargs)
 
     def get_first_image(self) -> PostImage:
         return self.images.earliest('id')
 
-    def delete(self, using=None):
+    def set_global_id(self, key_pair=None):
         """
-        Deletes all post images and the post itself.
+        Sets global id based on the given key pair. If no key pair is given,
+        default one is used.
         """
 
-        for image in self.images.all():
-            image_refs_count = image.post_images.count()
-            if image_refs_count == 1:
-                image.delete()
+        if key_pair:
+            key = key_pair
+        else:
+            try:
+                key = KeyPair.objects.get(primary=True)
+            except KeyPair.DoesNotExist:
+                # Do not update the global id because there is no key defined
+                return
+        global_id = GlobalId(key_type=key.key_type,
+                             key=key.public_key,
+                             local_id=self.id)
+        global_id.save()
+
+        self.global_id = global_id
+
+        self.save(update_fields=['global_id'])
+
+    def get_pub_time_str(self):
+        return str(self.pub_time)
 
-        for attachment in self.attachments.all():
-            attachment_refs_count = attachment.attachment_posts.count()
-            if attachment_refs_count == 1:
-                attachment.delete()
+    def get_replied_ids(self):
+        """
+        Gets ID list of the posts that this post replies.
+        """
+
+        raw_text = self.get_raw_text()
 
-        thread = self.get_thread()
-        thread.last_edit_time = timezone.now()
-        thread.save()
+        local_replied = REGEX_REPLY.findall(raw_text)
+        global_replied = []
+        for match in REGEX_GLOBAL_REPLY.findall(raw_text):
+            key_type = match[0]
+            key = match[1]
+            local_id = match[2]
 
-        super(Post, self).delete(using)
-
-        logging.getLogger('boards.post.delete').info(
-            'Deleted post {}'.format(self))
+            try:
+                global_id = GlobalId.objects.get(key_type=key_type,
+                                                 key=key, local_id=local_id)
+                for post in Post.objects.filter(global_id=global_id).only('id'):
+                    global_replied.append(post.id)
+            except GlobalId.DoesNotExist:
+                pass
+        return local_replied + global_replied
 
     def get_post_data(self, format_type=DIFF_TYPE_JSON, request=None,
                       include_last_update=False) -> str:
         """
         Gets post HTML or JSON data that can be rendered on a page or used by
         API.
         """
 
         return get_exporter(format_type).export(self, request,
                                                 include_last_update)
 
     def notify_clients(self, recursive=True):
         """
         Sends post HTML data to the thread web socket.
         """
 
         if not settings.get_bool('External', 'WebsocketsEnabled'):
             return
 
         thread_ids = list()
         for thread in self.get_threads().all():
             thread_ids.append(thread.id)
 
             thread.notify_clients()
 
         if recursive:
             for reply_number in re.finditer(REGEX_REPLY, self.get_raw_text()):
                 post_id = reply_number.group(1)
 
                 try:
                     ref_post = Post.objects.get(id=post_id)
 
                     if ref_post.get_threads().exclude(id__in=thread_ids).exists():
                         # If post is in this thread, its thread was already notified.
                         # Otherwise, notify its thread separately.
                         ref_post.notify_clients(recursive=False)
                 except ObjectDoesNotExist:
                     pass
 
     def build_url(self):
         self.url = self.get_absolute_url()
         self.save(update_fields=['url'])
 
     def save(self, force_insert=False, force_update=False, using=None,
              update_fields=None):
         new_post = self.id is None
 
         self.uid = str(uuid.uuid4())
         if update_fields is not None and 'uid' not in update_fields:
             update_fields += ['uid']
 
         if not new_post:
             for thread in self.get_threads().all():
                 thread.last_edit_time = self.last_edit_time
 
                 thread.save(update_fields=['last_edit_time', 'status'])
 
         super().save(force_insert, force_update, using, update_fields)
 
         if self.url is None:
             self.build_url()
 
     def get_text(self) -> str:
         return self._text_rendered
 
     def get_raw_text(self) -> str:
         return self.text
 
+    def get_sync_text(self) -> str:
+        """
+        Returns text applicable for sync. It has absolute post reflinks.
+        """
+
+        replacements = dict()
+        for post_id in REGEX_REPLY.findall(self.get_raw_text()):
+            absolute_post_id = str(Post.objects.get(id=post_id).global_id)
+            replacements[post_id] = absolute_post_id
+
+        text = self.get_raw_text() or ''
+        for key in replacements:
+            text = text.replace('[post]{}[/post]'.format(key),
+                                '[post]{}[/post]'.format(replacements[key]))
+        text = text.replace('\r\n', '\n').replace('\r', '\n')
+
+        return text
+
     def connect_threads(self, opening_posts):
         for opening_post in opening_posts:
             threads = opening_post.get_threads().all()
             for thread in threads:
                 if thread.can_bump():
                     thread.update_bump_status()
 
                 thread.last_edit_time = self.last_edit_time
                 thread.save(update_fields=['last_edit_time', 'status'])
             self.threads.add(opening_post.get_thread())
 
     def get_tripcode(self):
         if self.tripcode:
             return Tripcode(self.tripcode)
 
     def get_link_view(self):
         """
         Gets view of a reflink to the post.
         """
         result = '<a href="{}">&gt;&gt;{}</a>'.format(self.get_absolute_url(),
                                                       self.id)
         if self.is_opening():
             result = '<b>{}</b>'.format(result)
 
         return result
 
     def is_hidden(self) -> bool:
         return self.hidden
 
     def set_hidden(self, hidden):
         self.hidden = hidden
-
-
-# SIGNALS (Maybe move to other module?)
-@receiver(post_save, sender=Post)
-def connect_replies(instance, **kwargs):
-    for reply_number in re.finditer(REGEX_REPLY, instance.get_raw_text()):
-        post_id = reply_number.group(1)
-
-        try:
-            referenced_post = Post.objects.get(id=post_id)
-
-            # Connect only to posts that are not connected to already
-            if not referenced_post.referenced_posts.filter(id=instance.id).exists():
-                referenced_post.referenced_posts.add(instance)
-                referenced_post.last_edit_time = instance.pub_time
-                referenced_post.build_refmap()
-                referenced_post.save(update_fields=['refmap', 'last_edit_time'])
-        except ObjectDoesNotExist:
-            pass
-
-
-@receiver(post_save, sender=Post)
-def connect_notifications(instance, **kwargs):
-    for reply_number in re.finditer(REGEX_NOTIFICATION, instance.get_raw_text()):
-        user_name = reply_number.group(1).lower()
-        Notification.objects.get_or_create(name=user_name, post=instance)
-
-
-@receiver(pre_save, sender=Post)
-def preparse_text(instance, **kwargs):
-    instance._text_rendered = get_parser().parse(instance.get_raw_text())

boards/models/post/manager.py

@@ -1,133 +1,160 b''
 import logging
 
 from datetime import datetime, timedelta, date
 from datetime import time as dtime
 
 from django.db import models, transaction
 from django.utils import timezone
 
 import boards
 
 from boards.models.user import Ban
 from boards.mdx_neboard import Parser
 from boards.models import PostImage, Attachment
 from boards import utils
 
 __author__ = 'neko259'
 
 IMAGE_TYPES = (
     'jpeg',
     'jpg',
     'png',
     'bmp',
     'gif',
 )
 
 POSTS_PER_DAY_RANGE = 7
 NO_IP = '0.0.0.0'
 
 
 class PostManager(models.Manager):
     @transaction.atomic
     def create_post(self, title: str, text: str, file=None, thread=None,
                     ip=NO_IP, tags: list=None, opening_posts: list=None,
                     tripcode='', monochrome=False, images=[]):
         """
         Creates new post
         """
 
         if thread is not None and thread.is_archived():
             raise Exception('Cannot post into an archived thread')
 
         if not utils.is_anonymous_mode():
             is_banned = Ban.objects.filter(ip=ip).exists()
         else:
             is_banned = False
 
         # TODO Raise specific exception and catch it in the views
         if is_banned:
             raise Exception("This user is banned")
 
         if not tags:
             tags = []
         if not opening_posts:
             opening_posts = []
 
         posting_time = timezone.now()
         new_thread = False
         if not thread:
             thread = boards.models.thread.Thread.objects.create(
                 bump_time=posting_time, last_edit_time=posting_time,
                     monochrome=monochrome)
             list(map(thread.tags.add, tags))
             boards.models.thread.Thread.objects.process_oldest_threads()
             new_thread = True
 
         pre_text = Parser().preparse(text)
 
         post = self.create(title=title,
                            text=pre_text,
                            pub_time=posting_time,
                            poster_ip=ip,
                            thread=thread,
                            last_edit_time=posting_time,
                            tripcode=tripcode,
                            opening=new_thread)
         post.threads.add(thread)
 
         logger = logging.getLogger('boards.post.create')
 
         logger.info('Created post [{}] with text [{}] by {}'.format(post,
             post.get_text(),post.poster_ip))
 
-        # TODO Move this to other place
         if file:
-            file_type = file.name.split('.')[-1].lower()
-            if file_type in IMAGE_TYPES:
-                post.images.add(PostImage.objects.create_with_hash(file))
-            else:
-                post.attachments.add(Attachment.objects.create_with_hash(file))
+            self._add_file_to_post(file, post)
         for image in images:
             post.images.add(image)
 
         post.connect_threads(opening_posts)
+        post.set_global_id()
 
         # Thread needs to be bumped only when the post is already created
         if not new_thread:
             thread.last_edit_time = posting_time
             thread.bump()
             thread.save()
 
         return post
 
     def delete_posts_by_ip(self, ip):
         """
         Deletes all posts of the author with same IP
         """
 
         posts = self.filter(poster_ip=ip)
         for post in posts:
             post.delete()
 
     @utils.cached_result()
     def get_posts_per_day(self) -> float:
         """
         Gets average count of posts per day for the last 7 days
         """
 
         day_end = date.today()
         day_start = day_end - timedelta(POSTS_PER_DAY_RANGE)
 
         day_time_start = timezone.make_aware(datetime.combine(
             day_start, dtime()), timezone.get_current_timezone())
         day_time_end = timezone.make_aware(datetime.combine(
             day_end, dtime()), timezone.get_current_timezone())
 
         posts_per_period = float(self.filter(
             pub_time__lte=day_time_end,
             pub_time__gte=day_time_start).count())
 
         ppd = posts_per_period / POSTS_PER_DAY_RANGE
 
         return ppd
 
+    @transaction.atomic
+    def import_post(self, title: str, text: str, pub_time: str, global_id,
+                    opening_post=None, tags=list(), files=list()):
+        is_opening = opening_post is None
+        if is_opening:
+            thread = boards.models.thread.Thread.objects.create(
+                bump_time=pub_time, last_edit_time=pub_time)
+            list(map(thread.tags.add, tags))
+        else:
+            thread = opening_post.get_thread()
+
+        post = self.create(title=title, text=text,
+                           pub_time=pub_time,
+                           poster_ip=NO_IP,
+                           last_edit_time=pub_time,
+                           global_id=global_id,
+                           opening=is_opening,
+                           thread=thread)
+
+        # TODO Add files
+        for file in files:
+            self._add_file_to_post(file, post)
+
+        post.threads.add(thread)
+
+    def _add_file_to_post(self, file, post):
+        file_type = file.name.split('.')[-1].lower()
+        if file_type in IMAGE_TYPES:
+            post.images.add(PostImage.objects.create_with_hash(file))
+        else:
+            post.attachments.add(Attachment.objects.create_with_hash(file))

boards/models/user.py

@@ -1,46 +1,45 b''
 from django.db import models
-
-import boards.models.post
+import boards
 
 __author__ = 'neko259'
 
 BAN_REASON_AUTO = 'Auto'
 BAN_REASON_MAX_LENGTH = 200
 
 
 class Ban(models.Model):
 
     class Meta:
         app_label = 'boards'
 
     ip = models.GenericIPAddressField()
     reason = models.CharField(default=BAN_REASON_AUTO,
                               max_length=BAN_REASON_MAX_LENGTH)
     can_read = models.BooleanField(default=True)
 
     def __str__(self):
         return self.ip
 
 
 class NotificationManager(models.Manager):
     def get_notification_posts(self, usernames: list, last: int = None):
         lower_names = [username.lower() for username in usernames]
         posts = boards.models.post.Post.objects.filter(
                 notification__name__in=lower_names).distinct()
         if last is not None:
             posts = posts.filter(id__gt=last)
         posts = posts.order_by('-id')
 
         return posts
 
 
 class Notification(models.Model):
 
     class Meta:
         app_label = 'boards'
 
     objects = NotificationManager()
 
     post = models.ForeignKey('Post')
     name = models.TextField()
 

boards/static/css/md/base_page.css

@@ -1,578 +1,583 b''
 * {
    text-decoration: none;
    font-weight: inherit;
 }
 
 b, strong {
    font-weight: bold;
 }
 
 html {
     background: #555;
     color: #ffffff;
 }
 
 body {
     margin: 0;
 }
 
 #admin_panel {
     background: #FF0000;
     color: #00FF00
 }
 
 .input_field_error {
     color: #FF0000;
 }
 
 .title {
     font-weight: bold;
     color: #ffcc00;
 }
 
 .link, a {
     color: #afdcec;
 }
 
 .block {
     display: inline-block;
     vertical-align: top;
 }
 
 .tag {
     color: #FFD37D;
 }
 
 .post_id {
     color: #fff380;
 }
 
 .post, .dead_post, .archive_post, #posts-table {
     background: #333;
     padding: 10px;
     clear: left;
     word-wrap: break-word;
     border-top: 1px solid #777;
     border-bottom: 1px solid #777;
 }
 
 .post + .post {
     border-top: none;
 }
 
 .dead_post + .dead_post {
     border-top: none;
 }
 
 .archive_post + .archive_post {
     border-top: none;
 }
 
 .metadata {
     padding-top: 5px;
     margin-top: 10px;
     border-top: solid 1px #666;
     color: #ddd;
 }
 
 .navigation_panel, .tag_info {
     background: #222;
     margin-bottom: 5px;
     margin-top: 5px;
     padding: 10px;
     border-bottom: solid 1px #888;
     border-top: solid 1px #888;
     color: #eee;
 }
 
 .navigation_panel .link:first-child {
     border-right: 1px solid #fff;
     font-weight: bold;
     margin-right: 1ex;
     padding-right: 1ex;
 }
 
 .navigation_panel .right-link {
     border-left: 1px solid #fff;
     border-right: none;
     float: right;
     margin-left: 1ex;
     margin-right: 0;
     padding-left: 1ex;
     padding-right: 0;
 }
 
 .navigation_panel .link {
     font-weight: bold;
 }
 
 .navigation_panel::after, .post::after {
     clear: both;
     content: ".";
     display: block;
     height: 0;
     line-height: 0;
     visibility: hidden;
 }
 
 .tag_info {
     text-align: center;
 }
 
 .tag_info > .tag-text-data {
     text-align: left;
     max-width: 30em;
 }
 
 .header {
     border-bottom: solid 2px #ccc;
     margin-bottom: 5px;
     border-top: none;
     margin-top: 0;
 }
 
 .footer {
     border-top: solid 2px #ccc;
     margin-top: 5px;
     border-bottom: none;
     margin-bottom: 0;
 }
 
 p, .br {
     margin-top: .5em;
     margin-bottom: .5em;
 }
 
 .post-form-w {
     background: #333344;
     border-top: solid 1px #888;
     border-bottom: solid 1px #888;
     color: #fff;
     padding: 10px;
     margin-bottom: 5px;
     margin-top: 5px;
 }
 
 .form-row {
     width: 100%;
     display: table-row;
 }
 
 .form-label {
     padding: .25em 1ex .25em 0;
     vertical-align: top;
     display: table-cell;
 }
 
 .form-input {
     padding: .25em 0;
     width: 100%;
     display: table-cell;
 }
 
 .form-errors {
     font-weight: bolder;
     vertical-align: middle;
     display: table-cell;
 }
 
 .post-form input:not([name="image"]):not([type="checkbox"]):not([type="submit"]), .post-form textarea, .post-form select {
     background: #333;
     color: #fff;
     border: solid 1px;
     padding: 0;
     font: medium sans-serif;
     width: 100%;
 }
 
 .post-form textarea {
     resize: vertical;
 }
 
 .form-submit {
     display: table;
     margin-bottom: 1ex;
     margin-top: 1ex;
 }
 
 .form-title {
     font-weight: bold;
     font-size: 2ex;
     margin-bottom: 0.5ex;
 }
 
 input[type="submit"], button {
     background: #222;
     border: solid 2px #fff;
     color: #fff;
     padding: 0.5ex;
     margin-right: 0.5ex;
 }
 
 input[type="submit"]:hover {
     background: #060;
 }
 
 .form-submit > button:hover {
     background: #006;
 }
 
 blockquote {
     border-left: solid 2px;
     padding-left: 5px;
     color: #B1FB17;
     margin: 0;
 }
 
 .post > .image {
     float: left;
     margin: 0 1ex .5ex 0;
     min-width: 1px;
     text-align: center;
     display: table-row;
 }
 
 .post > .metadata {
     clear: left;
 }
 
 .get {
     font-weight: bold;
     color: #d55;
 }
 
 * {
     text-decoration: none;
 }
 
 .dead_post > .post-info {
     font-style: italic;
 }
 
 .archive_post > .post-info {
     text-decoration: line-through;
 }
 
 .mark_btn {
     border: 1px solid;
     padding: 2px 2ex;
     display: inline-block;
     margin: 0 5px 4px 0;
 }
 
 .mark_btn:hover {
     background: #555;
 }
 
 .quote {
     color: #92cf38;
     font-style: italic;
 }
 
 .multiquote {
     padding: 3px;
     display: inline-block;
     background: #222;
     border-style: solid;
     border-width: 1px 1px 1px 4px;
     font-size: 0.9em;
 }
 
 .spoiler {
     background: black;
     color: black;
 }
 
 .spoiler:hover {
     color: #ddd;
 }
 
 .comment {
     color: #eb2;
 }
 
 a:hover {
     text-decoration: underline;
 }
 
 .last-replies {
     margin-left: 3ex;
     margin-right: 3ex;
     border-left: solid 1px #777;
     border-right: solid 1px #777;
 }
 
 .last-replies > .post:first-child {
     border-top: none;
 }
 
 .thread {
     margin-bottom: 3ex;
     margin-top: 1ex;
 }
 
 .post:target {
     border: solid 2px white;
 }
 
 pre{
     white-space:pre-wrap
 }
 
 li {
     list-style-position: inside;
 }
 
 .fancybox-skin {
     position: relative;
     background-color: #fff;
     color: #ddd;
     text-shadow: none;
 }
 
 .fancybox-image {
     border: 1px solid black;
 }
 
 .image-mode-tab {
     background: #444;
     color: #eee;
     margin-top: 5px;
     padding: 5px;
     border-top: 1px solid #888;
     border-bottom: 1px solid #888;
 }
 
 .image-mode-tab > label {
     margin: 0 1ex;
 }
 
 .image-mode-tab > label > input {
     margin-right: .5ex;
 }
 
 #posts-table {
     margin-top: 5px;
     margin-bottom: 5px;
 }
 
 .tag_info > h2 {
     margin: 0;
 }
 
 .post-info {
     color: #ddd;
     margin-bottom: 1ex;
 }
 
 .moderator_info {
     color: #e99d41;
     opacity: 0.4;
 }
 
 .moderator_info:hover {
     opacity: 1;
 }
 
 .refmap {
     font-size: 0.9em;
     color: #ccc;
     margin-top: 1em;
 }
 
 .fav {
     color: yellow;
 }
 
 .not_fav {
     color: #ccc;
 }
 
 .form-email {
     display: none;
 }
 
 .bar-value {
     background: rgba(50, 55, 164, 0.45);
     font-size: 0.9em;
     height: 1.5em;
 }
 
 .bar-bg {
     position: relative;
     border-top: solid 1px #888;
     border-bottom: solid 1px #888;
     margin-top: 5px;
     overflow: hidden;
 }
 
 .bar-text {
     padding: 2px;
     position: absolute;
     left: 0;
     top: 0;
 }
 
 .page_link {
     background: #444;
     border-top: solid 1px #888;
     border-bottom: solid 1px #888;
     padding: 5px;
     color: #eee;
     font-size: 2ex;
     margin-top: .5ex;
     margin-bottom: .5ex;
 }
 
 .skipped_replies {
     padding: 5px;
     margin-left: 3ex;
     margin-right: 3ex;
     border-left: solid 1px #888;
     border-right: solid 1px #888;
     border-bottom: solid 1px #888;
     background: #000;
 }
 
 .current_page {
     padding: 2px;
     background-color: #afdcec;
     color: #000;
 }
 
 .current_mode {
     font-weight: bold;
 }
 
 .gallery_image {
     border: solid 1px;
     margin: 0.5ex;
     text-align: center;
     padding: 1ex;
 }
 
 code {
     border: dashed 1px #ccc;
     background: #111;
     padding: 2px;
     font-size: 1.2em;
     display: inline-block;
 }
 
 pre {
     overflow: auto;
 }
 
 .img-full {
     background: #222;
     border: solid 1px white;
 }
 
 .tag_item {
     display: inline-block;
 }
 
 #id_models li {
     list-style: none;
 }
 
 #id_q {
     margin-left: 1ex;
 }
 
 ul {
     padding-left: 0px;
 }
 
 .quote-header {
     border-bottom: 2px solid #ddd;
     margin-bottom: 1ex;
     padding-bottom: .5ex;
     color: #ddd;
     font-size: 1.2em;
 }
 
+.global-id {
+    font-weight: bolder;
+    opacity: .5;
+}
+
 /* Reflink preview */
 .post_preview {
     border-left: 1px solid #777;
     border-right: 1px solid #777;
     max-width: 600px;
 }
 
 /* Code highlighter */
 .hljs {
     color: #fff;
     background: #000;
     display: inline-block;
 }
 
 .hljs, .hljs-subst, .hljs-tag .hljs-title, .lisp .hljs-title, .clojure .hljs-built_in, .nginx .hljs-title {
     color: #fff;
 }
 
 #up {
     position: fixed;
     bottom: 5px;
     right: 5px;
     border: 1px solid #777;
     background: #000;
     padding: 4px;
     opacity: 0.3;
 }
 
 #up:hover {
     opacity: 1;
 }
 
 .user-cast {
     border: solid #ffffff 1px;
     padding: .2ex;
     background: #152154;
     color: #fff;
 }
 
 .highlight {
     background: #222;
 }
 
 .post-button-form > button:hover {
     text-decoration: underline;
 }
 
 .tree_reply > .post {
     margin-top: 1ex;
     border-left: solid 1px #777;
     padding-right: 0;
 }
 
 #preview-text {
     border: solid 1px white;
     margin: 1ex 0 1ex 0;
     padding: 1ex;
 }
 
 .image-metadata {
     font-size: 0.9em;
 }
 
 .tripcode {
     color: white;
 }
 
 #fav-panel {
     border: 1px solid white;
 }
 
 .post-blink {
     background-color: #000;
 }
 
 #quote-button {
     background-color: black;
     border: solid white 1px;
     padding: 2px;
 }
 
 #quote-button:hover {
     background-color: #2d3955;
 }
  No newline at end of file

boards/templates/boards/post.html

@@ -1,105 +1,107 b''
 {% load i18n %}
 {% load board %}
 
 {% get_current_language as LANGUAGE_CODE %}
 
 <div class="{{ css_class }}" id="{{ post.id }}" data-uid="{{ post.uid }}" {% if tree_depth %}style="margin-left: {{ tree_depth }}em;"{% endif %}>
     <div class="post-info">
         <a class="post_id" href="{{ post.get_absolute_url }}">#{{ post.id }}</a>
         <span class="title">{{ post.title }}</span>
         <span class="pub_time"><time datetime="{{ post.pub_time|date:'c' }}">{{ post.pub_time }}</time></span>
         {% if post.tripcode %}
             /
             {% with tripcode=post.get_tripcode %}
             <a href="{% url 'feed' %}?tripcode={{ tripcode.get_full_text }}"
                class="tripcode" title="{{ tripcode.get_full_text }}"
                style="border: solid 2px #{{ tripcode.get_color }}; border-left: solid 1ex #{{ tripcode.get_color }};">{{ tripcode.get_short_text }}</a>
             {% endwith %}
         {% endif %}
         {% comment %}
             Thread death time needs to be shown only if the thread is alredy archived
             and this is an opening post (thread death time) or a post for popup
             (we don't see OP here so we show the death time in the post itself).
         {% endcomment %}
         {% if thread.is_archived %}
             {% if is_opening %}
             — <time datetime="{{ thread.bump_time|date:'c' }}">{{ thread.bump_time }}</time>
             {% endif %}
         {% endif %}
         {% if is_opening %}
             {% if need_open_link %}
                 {% if thread.is_archived %}
                     <a class="link" href="{% url 'thread' post.id %}">{% trans "Open" %}</a>
                 {% else %}
                     <a class="link" href="{% url 'thread' post.id %}#form">{% trans "Reply" %}</a>
                 {% endif %}
             {% endif %}
         {% else %}
             {% if need_op_data %}
                 {% with thread.get_opening_post as op %}
                     {% trans " in " %}{{ op.get_link_view|safe }} <span class="title">{{ op.get_title_or_text }}</span>
                 {% endwith %}
             {% endif %}
         {% endif %}
         {% if reply_link and not thread.is_archived %}
             <a href="#form" onclick="addQuickReply('{{ post.id }}'); return false;">{% trans 'Reply' %}</a>
         {% endif %}
 
         {% if perms.boards.change_post or perms.boards.delete_post or perms.boards.change_thread or perms_boards.delete_thread %}
             <span class="moderator_info">
                 {% if perms.boards.change_post or perms.boards.delete_post %}
                     | <a href="{% url 'admin:boards_post_change' post.id %}">{% trans 'Edit' %}</a>
                 {% endif %}
                 {% if perms.boards.change_thread or perms_boards.delete_thread %}
                     {% if is_opening %}
                        | <a href="{% url 'admin:boards_thread_change' thread.id %}">{% trans 'Edit thread' %}</a>
                     {% endif %}
                 {% endif %}
-            </form>
+                {% if post.global_id_id %}
+                    | <a href="{% url 'post_sync_data' post.id %}">RAW</a>
+                {% endif %}
             </span>
         {% endif %}
     </div>
     {% comment %}
         Post images. Currently only 1 image can be posted and shown, but post model
         supports multiple.
     {%  endcomment %}
     {% for image in post.images.all %}
         {{ image.get_view|safe }}
     {% endfor %}
     {% for file in post.attachments.all %}
         {{ file.get_view|safe }}
     {% endfor %}
     {% comment %}
         Post message (text)
     {% endcomment %}
     <div class="message">
         {% autoescape off %}
             {% if truncated %}
                 {{ post.get_text|truncatewords_html:50 }}
             {% else %}
                 {{ post.get_text }}
             {% endif %}
         {% endautoescape %}
     </div>
     {% if post.is_referenced %}
         {% if not mode_tree %}
             <div class="refmap">
                 {% trans "Replies" %}: {{ post.refmap|safe }}
             </div>
         {% endif %}
     {% endif %}
     {% comment %}
         Thread metadata: counters, tags etc
     {% endcomment %}
     {% if is_opening %}
         <div class="metadata">
             {% if is_opening and need_open_link %}
                 {% blocktrans count count=thread.get_reply_count %}{{ count }} message{% plural %}{{ count }} messages{% endblocktrans %},
                 {% blocktrans count count=thread.get_images_count %}{{ count }} image{% plural %}{{ count }} images{% endblocktrans %}.
             {% endif %}
             <span class="tags">
                 {{ thread.get_tag_url_list|safe }}
             </span>
         </div>
     {% endif %}
 </div>

boards/tests/test_forms.py

@@ -1,56 +1,56 b''
 from django.test import TestCase, Client
 import time
 from boards import settings
 from boards.models import Post, Tag
 import neboard
 
 
 TEST_TAG = 'test_tag'
 
 PAGE_404 = 'boards/404.html'
 
 TEST_TEXT = 'test text'
 
 NEW_THREAD_PAGE = '/'
 THREAD_PAGE_ONE = '/thread/1/'
 HTTP_CODE_REDIRECT = 302
 
 
 class FormTest(TestCase):
     def test_post_validation(self):
         client = Client()
 
         valid_tags = 'tag1 tag_2 тег_3'
         invalid_tags = '$%_356 ---'
         Tag.objects.create(name='tag1', required=True)
 
         response = client.post(NEW_THREAD_PAGE, {'title': 'test title',
                                                  'text': TEST_TEXT,
                                                  'tags': valid_tags})
         self.assertEqual(response.status_code, HTTP_CODE_REDIRECT,
                          msg='Posting new message failed: got code ' +
                              str(response.status_code))
 
         self.assertEqual(1, Post.objects.count(),
                          msg='No posts were created')
 
         client.post(NEW_THREAD_PAGE, {'text': TEST_TEXT,
                                       'tags': invalid_tags})
         self.assertEqual(1, Post.objects.count(), msg='The validation passed '
                                                       'where it should fail')
 
         # Change posting delay so we don't have to wait for 30 seconds or more
         old_posting_delay = neboard.settings.POSTING_DELAY
         # Wait fot the posting delay or we won't be able to post
-        settings.POSTING_DELAY = 1
+        neboard.settings.POSTING_DELAY = 1
         time.sleep(neboard.settings.POSTING_DELAY + 1)
         response = client.post(THREAD_PAGE_ONE, {'text': TEST_TEXT,
                                                  'tags': valid_tags})
         self.assertEqual(HTTP_CODE_REDIRECT, response.status_code,
                          msg='Posting new message failed: got code ' +
                              str(response.status_code))
         # Restore posting delay
         settings.POSTING_DELAY = old_posting_delay
 
         self.assertEqual(2, Post.objects.count(),
                          msg='No posts were created')

boards/tests/test_post.py

@@ -1,179 +1,200 b''
 from django.core.paginator import Paginator
 from django.test import TestCase
 
 from boards import settings
-from boards.models import Tag, Post, Thread
+from boards.models import Tag, Post, Thread, KeyPair
 from boards.models.thread import STATUS_ARCHIVE
 
 
 class PostTests(TestCase):
 
     def _create_post(self):
         tag, created = Tag.objects.get_or_create(name='test_tag')
         return Post.objects.create_post(title='title', text='text',
                                         tags=[tag])
 
     def test_post_add(self):
         """Test adding post"""
 
         post = self._create_post()
 
         self.assertIsNotNone(post, 'No post was created.')
         self.assertEqual('test_tag', post.get_thread().tags.all()[0].name,
                          'No tags were added to the post.')
 
     def test_delete_post(self):
         """Test post deletion"""
 
         post = self._create_post()
         post_id = post.id
 
         post.delete()
 
         self.assertFalse(Post.objects.filter(id=post_id).exists())
 
     def test_delete_thread(self):
         """Test thread deletion"""
 
         opening_post = self._create_post()
         thread = opening_post.get_thread()
         reply = Post.objects.create_post("", "", thread=thread)
 
         thread.delete()
 
         self.assertFalse(Post.objects.filter(id=reply.id).exists(),
                          'Reply was not deleted with the thread.')
         self.assertFalse(Post.objects.filter(id=opening_post.id).exists(),
                          'Opening post was not deleted with the thread.')
 
     def test_post_to_thread(self):
         """Test adding post to a thread"""
 
         op = self._create_post()
         post = Post.objects.create_post("", "", thread=op.get_thread())
 
         self.assertIsNotNone(post, 'Reply to thread wasn\'t created')
         self.assertEqual(op.get_thread().last_edit_time, post.pub_time,
                          'Post\'s create time doesn\'t match thread last edit'
                          ' time')
 
     def test_delete_posts_by_ip(self):
         """Test deleting posts with the given ip"""
 
         post = self._create_post()
         post_id = post.id
 
         Post.objects.delete_posts_by_ip('0.0.0.0')
 
         self.assertFalse(Post.objects.filter(id=post_id).exists())
 
     def test_get_thread(self):
         """Test getting all posts of a thread"""
 
         opening_post = self._create_post()
 
         for i in range(2):
             Post.objects.create_post('title', 'text',
                                      thread=opening_post.get_thread())
 
         thread = opening_post.get_thread()
 
         self.assertEqual(3, thread.get_replies().count())
 
     def test_create_post_with_tag(self):
         """Test adding tag to post"""
 
         tag = Tag.objects.create(name='test_tag')
         post = Post.objects.create_post(title='title', text='text', tags=[tag])
 
         thread = post.get_thread()
         self.assertIsNotNone(post, 'Post not created')
         self.assertTrue(tag in thread.tags.all(), 'Tag not added to thread')
 
     def test_thread_max_count(self):
         """Test deletion of old posts when the max thread count is reached"""
 
         for i in range(settings.get_int('Messages', 'MaxThreadCount') + 1):
             self._create_post()
 
         self.assertEqual(settings.get_int('Messages', 'MaxThreadCount'),
                          len(Thread.objects.exclude(status=STATUS_ARCHIVE)))
 
     def test_pages(self):
         """Test that the thread list is properly split into pages"""
 
         for i in range(settings.get_int('Messages', 'MaxThreadCount')):
             self._create_post()
 
         all_threads = Thread.objects.exclude(status=STATUS_ARCHIVE)
 
         paginator = Paginator(Thread.objects.exclude(status=STATUS_ARCHIVE),
                               settings.get_int('View', 'ThreadsPerPage'))
         posts_in_second_page = paginator.page(2).object_list
         first_post = posts_in_second_page[0]
 
         self.assertEqual(all_threads[settings.get_int('View', 'ThreadsPerPage')].id,
                          first_post.id)
 
+    def test_reflinks(self):
+        """
+        Tests that reflinks are parsed within post and connecting replies
+        to the replied posts.
+
+        Local reflink example: [post]123[/post]
+        Global reflink example: [post]key_type::key::123[/post]
+        """
+
+        key = KeyPair.objects.generate_key(primary=True)
+
+        tag = Tag.objects.create(name='test_tag')
+
+        post = Post.objects.create_post(title='', text='', tags=[tag])
+        post_local_reflink = Post.objects.create_post(title='',
+                text='[post]%d[/post]' % post.id, thread=post.get_thread())
+
+        self.assertTrue(post_local_reflink in post.referenced_posts.all(),
+            'Local reflink not connecting posts.')
+
+
     def test_thread_replies(self):
         """
         Tests that the replies can be queried from a thread in all possible
         ways.
         """
 
         tag = Tag.objects.create(name='test_tag')
         opening_post = Post.objects.create_post(title='title', text='text',
                                         tags=[tag])
         thread = opening_post.get_thread()
 
         Post.objects.create_post(title='title', text='text', thread=thread)
         Post.objects.create_post(title='title', text='text', thread=thread)
 
         replies = thread.get_replies()
         self.assertTrue(len(replies) > 0, 'No replies found for thread.')
 
         replies = thread.get_replies(view_fields_only=True)
         self.assertTrue(len(replies) > 0,
                 'No replies found for thread with view fields only.')
 
     def test_bumplimit(self):
         """
         Tests that the thread bumpable status is changed and post uids and
         last update times are updated across all post threads.
         """
 
         op1 = Post.objects.create_post(title='title', text='text')
         op2 = Post.objects.create_post(title='title', text='text')
 
         thread1 = op1.get_thread()
         thread1.max_posts = 5
         thread1.save()
 
         uid_1 = op1.uid
         uid_2 = op2.uid
 
         # Create multi reply
         Post.objects.create_post(
             title='title', text='text', thread=thread1,
             opening_posts=[op1, op2])
         thread_update_time_2 = op2.get_thread().last_edit_time
         for i in range(6):
             Post.objects.create_post(title='title', text='text',
                                      thread=thread1)
 
         self.assertFalse(op1.get_thread().can_bump(),
                          'Thread is bumpable when it should not be.')
         self.assertTrue(op2.get_thread().can_bump(),
                         'Thread is not bumpable when it should be.')
         self.assertNotEqual(
             uid_1, Post.objects.get(id=op1.id).uid,
             'UID of the first OP should be changed but it is not.')
         self.assertEqual(
             uid_2, Post.objects.get(id=op2.id).uid,
             'UID of the first OP should not be changed but it is.')
 
         self.assertNotEqual(
             thread_update_time_2,
             Thread.objects.get(id=op2.get_thread().id).last_edit_time,
             'Thread last update time should change when the other thread '
             'changes status.')

boards/tests/test_views.py

@@ -1,45 +1,46 b''
 import logging
 from django.core.urlresolvers import reverse, NoReverseMatch
 from django.test import TestCase, Client
 from boards import urls
 
 
 logger = logging.getLogger(__name__)
 
 HTTP_CODE_OK = 200
 
 EXCLUDED_VIEWS = {
-        'banned',
-        'get_thread_diff',
+    'banned',
+    'get_thread_diff',
+    'api_sync_pull',
 }
 
 
 class ViewTest(TestCase):
 
     def test_all_views(self):
         """
         Try opening all views defined in ulrs.py that don't need additional
         parameters
         """
 
         client = Client()
         for url in urls.urlpatterns:
             try:
                 view_name = url.name
                 if view_name in EXCLUDED_VIEWS:
                     logger.debug('View {} is excluded.'.format(view_name))
                     continue
 
                 logger.debug('Testing view %s' % view_name)
 
                 try:
                     response = client.get(reverse(view_name))
 
                     self.assertEqual(HTTP_CODE_OK, response.status_code,
                             'View not opened: {}'.format(view_name))
                 except NoReverseMatch:
                     # This view just needs additional arguments
                     pass
             except AttributeError:
                 # This is normal, some views do not have names
                 pass

boards/urls.py

@@ -1,89 +1,99 b''
 from django.conf.urls import url
 #from django.views.i18n import javascript_catalog
 
 import neboard
+
 from boards import views
 from boards.rss import AllThreadsFeed, TagThreadsFeed, ThreadPostsFeed
 from boards.views import api, tag_threads, all_threads, \
     settings, all_tags, feed
 from boards.views.authors import AuthorsView
 from boards.views.notifications import NotificationView
+from boards.views.search import BoardSearchView
 from boards.views.static import StaticPageView
 from boards.views.preview import PostPreviewView
+from boards.views.sync import get_post_sync_data, response_get, response_pull
 from boards.views.random import RandomImageView
 from boards.views.tag_gallery import TagGalleryView
 from boards.views.translation import cached_javascript_catalog
 
 
 js_info_dict = {
     'packages': ('boards',),
 }
 
 urlpatterns = [
     # /boards/
     url(r'^$', all_threads.AllThreadsView.as_view(), name='index'),
 
     # /boards/tag/tag_name/
     url(r'^tag/(?P<tag_name>\w+)/$', tag_threads.TagView.as_view(),
         name='tag'),
 
     # /boards/thread/
     url(r'^thread/(?P<post_id>\d+)/$', views.thread.NormalThreadView.as_view(),
         name='thread'),
     url(r'^thread/(?P<post_id>\d+)/mode/gallery/$', views.thread.GalleryThreadView.as_view(),
         name='thread_gallery'),
     url(r'^thread/(?P<post_id>\d+)/mode/tree/$', views.thread.TreeThreadView.as_view(),
         name='thread_tree'),
     # /feed/
     url(r'^feed/$', views.feed.FeedView.as_view(), name='feed'),
 
     url(r'^settings/$', settings.SettingsView.as_view(), name='settings'),
     url(r'^tags/(?P<query>\w+)?/?$', all_tags.AllTagsView.as_view(), name='tags'),
     url(r'^authors/$', AuthorsView.as_view(), name='authors'),
 
     url(r'^banned/$', views.banned.BannedView.as_view(), name='banned'),
     url(r'^staticpage/(?P<name>\w+)/$', StaticPageView.as_view(),
         name='staticpage'),
 
     url(r'^random/$', RandomImageView.as_view(), name='random'),
     url(r'^tag/(?P<tag_name>\w+)/gallery/$', TagGalleryView.as_view(), name='tag_gallery'),
 
     # RSS feeds
     url(r'^rss/$', AllThreadsFeed()),
     url(r'^page/(?P<page>\d+)/rss/$', AllThreadsFeed()),
     url(r'^tag/(?P<tag_name>\w+)/rss/$', TagThreadsFeed()),
     url(r'^tag/(?P<tag_name>\w+)/page/(?P<page>\w+)/rss/$', TagThreadsFeed()),
     url(r'^thread/(?P<post_id>\d+)/rss/$', ThreadPostsFeed()),
 
     # i18n
     url(r'^jsi18n/$', cached_javascript_catalog, js_info_dict,
         name='js_info_dict'),
 
     # API
     url(r'^api/post/(?P<post_id>\d+)/$', api.get_post, name="get_post"),
     url(r'^api/diff_thread/$', api.api_get_threaddiff, name="get_thread_diff"),
     url(r'^api/threads/(?P<count>\w+)/$', api.api_get_threads,
             name='get_threads'),
     url(r'^api/tags/$', api.api_get_tags, name='get_tags'),
     url(r'^api/thread/(?P<opening_post_id>\w+)/$', api.api_get_thread_posts,
             name='get_thread'),
     url(r'^api/add_post/(?P<opening_post_id>\w+)/$', api.api_add_post,
             name='add_post'),
     url(r'^api/notifications/(?P<username>\w+)/$', api.api_get_notifications,
             name='api_notifications'),
     url(r'^api/preview/$', api.api_get_preview, name='preview'),
     url(r'^api/new_posts/$', api.api_get_new_posts, name='new_posts'),
 
+    # Sync protocol API
+    url(r'^api/sync/pull/$', response_pull, name='api_sync_pull'),
+    url(r'^api/sync/get/$', response_get, name='api_sync_pull'),
+    # TODO 'get' request
+
     # Notifications
     url(r'^notifications/(?P<username>\w+)/$', NotificationView.as_view(), name='notifications'),
     url(r'^notifications/$', NotificationView.as_view(), name='notifications'),
 
     # Post preview
     url(r'^preview/$', PostPreviewView.as_view(), name='preview'),
+    url(r'^post_xml/(?P<post_id>\d+)$', get_post_sync_data,
+        name='post_sync_data'),
 ]
 
 # Search
 if 'haystack' in neboard.settings.INSTALLED_APPS:
     from boards.views.search import BoardSearchView
     urlpatterns.append(url(r'^search/$', BoardSearchView.as_view(), name='search'))
 

boards/views/api.py

@@ -1,301 +1,295 b''
-from collections import OrderedDict
 import json
 import logging
 
+from django.core import serializers
 from django.db import transaction
-from django.db.models import Count
 from django.http import HttpResponse
 from django.shortcuts import get_object_or_404
-from django.core import serializers
-from django.template.context_processors import csrf
 from django.views.decorators.csrf import csrf_protect
 
-from boards.abstracts.settingsmanager import get_settings_manager,\
-        FAV_THREAD_NO_UPDATES
-
+from boards.abstracts.settingsmanager import get_settings_manager
 from boards.forms import PostForm, PlainErrorList
+from boards.mdx_neboard import Parser
 from boards.models import Post, Thread, Tag
 from boards.models.thread import STATUS_ARCHIVE
+from boards.models.user import Notification
 from boards.utils import datetime_to_epoch
 from boards.views.thread import ThreadView
-from boards.models.user import Notification
-from boards.mdx_neboard import Parser
-
 
 __author__ = 'neko259'
 
 PARAMETER_TRUNCATED = 'truncated'
 PARAMETER_TAG = 'tag'
 PARAMETER_OFFSET = 'offset'
 PARAMETER_DIFF_TYPE = 'type'
 PARAMETER_POST = 'post'
 PARAMETER_UPDATED = 'updated'
 PARAMETER_LAST_UPDATE = 'last_update'
 PARAMETER_THREAD = 'thread'
 PARAMETER_UIDS = 'uids'
 
 DIFF_TYPE_HTML = 'html'
 DIFF_TYPE_JSON = 'json'
 
 STATUS_OK = 'ok'
 STATUS_ERROR = 'error'
 
 logger = logging.getLogger(__name__)
 
 
 @transaction.atomic
 def api_get_threaddiff(request):
     """
     Gets posts that were changed or added since time
     """
 
     thread_id = request.POST.get(PARAMETER_THREAD)
     uids_str = request.POST.get(PARAMETER_UIDS)
 
     if not thread_id or not uids_str:
         return HttpResponse(content='Invalid request.')
 
     uids = uids_str.strip().split(' ')
 
     opening_post = get_object_or_404(Post, id=thread_id)
     thread = opening_post.get_thread()
 
     json_data = {
         PARAMETER_UPDATED: [],
         PARAMETER_LAST_UPDATE: None,  # TODO Maybe this can be removed already?
     }
     posts = Post.objects.filter(threads__in=[thread]).exclude(uid__in=uids)
 
     diff_type = request.GET.get(PARAMETER_DIFF_TYPE, DIFF_TYPE_HTML)
 
     for post in posts:
         json_data[PARAMETER_UPDATED].append(post.get_post_data(
             format_type=diff_type, request=request))
     json_data[PARAMETER_LAST_UPDATE] = str(thread.last_edit_time)
 
     # If the tag is favorite, update the counter
     settings_manager = get_settings_manager(request)
     favorite = settings_manager.thread_is_fav(opening_post)
     if favorite:
         settings_manager.add_or_read_fav_thread(opening_post)
 
     return HttpResponse(content=json.dumps(json_data))
 
 
 @csrf_protect
 def api_add_post(request, opening_post_id):
     """
     Adds a post and return the JSON response for it
     """
 
     opening_post = get_object_or_404(Post, id=opening_post_id)
 
     logger.info('Adding post via api...')
 
     status = STATUS_OK
     errors = []
 
     if request.method == 'POST':
         form = PostForm(request.POST, request.FILES, error_class=PlainErrorList)
         form.session = request.session
 
         if form.need_to_ban:
             # Ban user because he is suspected to be a bot
             # _ban_current_user(request)
             status = STATUS_ERROR
         if form.is_valid():
             post = ThreadView().new_post(request, form, opening_post,
                                          html_response=False)
             if not post:
                 status = STATUS_ERROR
             else:
                 logger.info('Added post #%d via api.' % post.id)
         else:
             status = STATUS_ERROR
             errors = form.as_json_errors()
 
     response = {
         'status': status,
         'errors': errors,
     }
 
     return HttpResponse(content=json.dumps(response))
 
 
 def get_post(request, post_id):
     """
     Gets the html of a post. Used for popups. Post can be truncated if used
     in threads list with 'truncated' get parameter.
     """
 
     post = get_object_or_404(Post, id=post_id)
     truncated = PARAMETER_TRUNCATED in request.GET
 
     return HttpResponse(content=post.get_view(truncated=truncated, need_op_data=True))
 
 
 def api_get_threads(request, count):
     """
     Gets the JSON thread opening posts list.
     Parameters that can be used for filtering:
     tag, offset (from which thread to get results)
     """
 
     if PARAMETER_TAG in request.GET:
         tag_name = request.GET[PARAMETER_TAG]
         if tag_name is not None:
             tag = get_object_or_404(Tag, name=tag_name)
             threads = tag.get_threads().exclude(status=STATUS_ARCHIVE)
     else: 
         threads = Thread.objects.exclude(status=STATUS_ARCHIVE)
 
     if PARAMETER_OFFSET in request.GET:
         offset = request.GET[PARAMETER_OFFSET]
         offset = int(offset) if offset is not None else 0
     else:
         offset = 0
 
     threads = threads.order_by('-bump_time')
     threads = threads[offset:offset + int(count)]
 
     opening_posts = []
     for thread in threads:
         opening_post = thread.get_opening_post()
 
         # TODO Add tags, replies and images count
         post_data = opening_post.get_post_data(include_last_update=True)
         post_data['status'] = thread.get_status()
 
         opening_posts.append(post_data)
 
     return HttpResponse(content=json.dumps(opening_posts))
 
 
 # TODO Test this
 def api_get_tags(request):
     """
     Gets all tags or user tags.
     """
 
     # TODO Get favorite tags for the given user ID
 
     tags = Tag.objects.get_not_empty_tags()
 
     term = request.GET.get('term')
     if term is not None:
         tags = tags.filter(name__contains=term)
 
     tag_names = [tag.name for tag in tags]
 
     return HttpResponse(content=json.dumps(tag_names))
 
 
 # TODO The result can be cached by the thread last update time
 # TODO Test this
 def api_get_thread_posts(request, opening_post_id):
     """
     Gets the JSON array of thread posts
     """
 
     opening_post = get_object_or_404(Post, id=opening_post_id)
     thread = opening_post.get_thread()
     posts = thread.get_replies()
 
     json_data = {
         'posts': [],
         'last_update': None,
     }
     json_post_list = []
 
     for post in posts:
         json_post_list.append(post.get_post_data())
     json_data['last_update'] = datetime_to_epoch(thread.last_edit_time)
     json_data['posts'] = json_post_list
 
     return HttpResponse(content=json.dumps(json_data))
 
 
 def api_get_notifications(request, username):
     last_notification_id_str = request.GET.get('last', None)
     last_id = int(last_notification_id_str) if last_notification_id_str is not None else None
 
     posts = Notification.objects.get_notification_posts(usernames=[username],
                                                         last=last_id)
 
     json_post_list = []
     for post in posts:
         json_post_list.append(post.get_post_data())
     return HttpResponse(content=json.dumps(json_post_list))
 
 
 def api_get_post(request, post_id):
     """
     Gets the JSON of a post. This can be
     used as and API for external clients.
     """
 
     post = get_object_or_404(Post, id=post_id)
 
     json = serializers.serialize("json", [post], fields=(
         "pub_time", "_text_rendered", "title", "text", "image",
         "image_width", "image_height", "replies", "tags"
     ))
 
     return HttpResponse(content=json)
 
 
 def api_get_preview(request):
     raw_text = request.POST['raw_text']
 
     parser = Parser()
     return HttpResponse(content=parser.parse(parser.preparse(raw_text)))
 
 
 def api_get_new_posts(request):
     """
     Gets favorite threads and unread posts count.
     """
     posts = list()
 
     include_posts = 'include_posts' in request.GET
 
     settings_manager = get_settings_manager(request)
     fav_threads = settings_manager.get_fav_threads()
     fav_thread_ops = Post.objects.filter(id__in=fav_threads.keys())\
         .order_by('-pub_time').prefetch_related('thread')
 
     ops = [{'op': op, 'last_id': fav_threads[str(op.id)]} for op in fav_thread_ops]
     if include_posts:
         new_post_threads = Thread.objects.get_new_posts(ops)
         if new_post_threads:
             thread_ids = {thread.id: thread for thread in new_post_threads}
         else:
             thread_ids = dict()
 
         for op in fav_thread_ops:
             fav_thread_dict = dict()
 
             op_thread = op.get_thread()
             if op_thread.id in thread_ids:
                 thread = thread_ids[op_thread.id]
                 new_post_count = thread.new_post_count
                 fav_thread_dict['newest_post_link'] = thread.get_replies()\
                         .filter(id__gt=fav_threads[str(op.id)])\
                         .first().get_absolute_url(thread=thread)
             else:
                 new_post_count = 0
             fav_thread_dict['new_post_count'] = new_post_count
 
             fav_thread_dict['id'] = op.id
 
             fav_thread_dict['post_url'] = op.get_link_view()
             fav_thread_dict['title'] = op.title
 
             posts.append(fav_thread_dict)
     else:
         fav_thread_dict = dict()
         fav_thread_dict['new_post_count'] = \
             Thread.objects.get_new_post_count(ops)
         posts.append(fav_thread_dict)
 
     return HttpResponse(content=json.dumps(posts))

neboard/settings.py

@@ -1,241 +1,232 b''
 # Django settings for neboard project.
 import os
 
 DEBUG = True
-TEMPLATE_DEBUG = DEBUG
 
 ADMINS = (
     # ('Your Name', '[e-mail]'),
     ('admin', '[e-mail]')
 )
 
 MANAGERS = ADMINS
 
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
         'NAME': 'database.db',                      # Or path to database file if using sqlite3.
         'USER': '',                      # Not used with sqlite3.
         'PASSWORD': '',                  # Not used with sqlite3.
         'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
         'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
 	'CONN_MAX_AGE': None,
     }
 }
 
 # Local time zone for this installation. Choices can be found here:
 # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
 # although not all choices may be available on all operating systems.
 # In a Windows environment this must be set to your system time zone.
 TIME_ZONE = 'Europe/Kiev'
 
 # Language code for this installation. All choices can be found here:
 # http://www.i18nguy.com/unicode/language-identifiers.html
 LANGUAGE_CODE = 'en'
 
 SITE_ID = 1
 
 # If you set this to False, Django will make some optimizations so as not
 # to load the internationalization machinery.
 USE_I18N = True
 
 # If you set this to False, Django will not format dates, numbers and
 # calendars according to the current locale.
 USE_L10N = True
 
 # If you set this to False, Django will not use timezone-aware datetimes.
 USE_TZ = True
 
 USE_ETAGS = True
 
 # Absolute filesystem path to the directory that will hold user-uploaded files.
 # Example: "/home/media/media.lawrence.com/media/"
 MEDIA_ROOT = './media/' 
 
 # URL that handles the media served from MEDIA_ROOT. Make sure to use a
 # trailing slash.
 # Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
 MEDIA_URL = '/media/'
 
 # Absolute path to the directory static files should be collected to.
 # Don't put anything in this directory yourself; store your static files
 # in apps' "static/" subdirectories and in STATICFILES_DIRS.
 # Example: "/home/media/media.lawrence.com/static/"
 STATIC_ROOT = ''
 
 # URL prefix for static files.
 # Example: "http://media.lawrence.com/static/"
 STATIC_URL = '/static/'
 
-# Additional locations of static files
-# It is really a hack, put real paths, not related
-STATICFILES_DIRS = (
-    os.path.dirname(__file__) + '/boards/static',
-
-#    '/d/work/python/django/neboard/neboard/boards/static',
-    # Put strings here, like "/home/html/static" or "C:/www/django/static".
-    # Always use forward slashes, even on Windows.
-    # Don't forget to use absolute paths, not relative paths.
-)
+STATICFILES_DIRS = []
 
 # List of finder classes that know how to find static files in
 # various locations.
 STATICFILES_FINDERS = (
     'django.contrib.staticfiles.finders.FileSystemFinder',
     'django.contrib.staticfiles.finders.AppDirectoriesFinder',
 )
 
 if DEBUG:
     STATICFILES_STORAGE = \
         'django.contrib.staticfiles.storage.StaticFilesStorage'
 else:
     STATICFILES_STORAGE = \
         'django.contrib.staticfiles.storage.CachedStaticFilesStorage'
 
 # Make this unique, and don't share it with anybody.
 SECRET_KEY = '@1rc$o(7=tt#kd+4s$u6wchm**z^)4x90)7f6z(i&55@o11*8o'
 
 TEMPLATES = [{
     'BACKEND': 'django.template.backends.django.DjangoTemplates',
     'DIRS': ['templates'],
     'OPTIONS': {
         'loaders': [
             ('django.template.loaders.cached.Loader', [
                 'django.template.loaders.filesystem.Loader',
                 'django.template.loaders.app_directories.Loader',
             ]),
         ],
         'context_processors': [
             'django.template.context_processors.csrf',
             'django.core.context_processors.media',
             'django.core.context_processors.static',
             'django.core.context_processors.request',
             'django.contrib.auth.context_processors.auth',
             'boards.context_processors.user_and_ui_processor',
         ],
     },
 }]
 
 
 MIDDLEWARE_CLASSES = [
     'django.middleware.http.ConditionalGetMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.locale.LocaleMiddleware',
     'django.middleware.common.CommonMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'boards.middlewares.BanMiddleware',
     'boards.middlewares.TimezoneMiddleware',
 ]
 
 ROOT_URLCONF = 'neboard.urls'
 
 # Python dotted path to the WSGI application used by Django's runserver.
 WSGI_APPLICATION = 'neboard.wsgi.application'
 
 INSTALLED_APPS = (
     'django.contrib.auth',
     'django.contrib.contenttypes',
     'django.contrib.sessions',
     # 'django.contrib.sites',
     'django.contrib.messages',
     'django.contrib.staticfiles',
     # Uncomment the next line to enable the admin:
     'django.contrib.admin',
     # Uncomment the next line to enable admin documentation:
     # 'django.contrib.admindocs',
     #'django.contrib.humanize',
 
     'debug_toolbar',
 
     # Search
     'haystack',
 
     'boards',
 )
 
 # A sample logging configuration. The only tangible logging
 # performed by this configuration is to send an email to
 # the site admins on every HTTP 500 error when DEBUG=False.
 # See http://docs.djangoproject.com/en/dev/topics/logging for
 # more details on how to customize your logging configuration.
 LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,
     'formatters': {
         'verbose': {
             'format': '%(levelname)s %(asctime)s %(name)s %(process)d %(thread)d %(message)s'
         },
         'simple': {
             'format': '%(levelname)s %(asctime)s [%(name)s] %(message)s'
         },
     },
     'filters': {
         'require_debug_false': {
             '()': 'django.utils.log.RequireDebugFalse'
         }
     },
     'handlers': {
         'console': {
             'level': 'DEBUG',
             'class': 'logging.StreamHandler',
             'formatter': 'simple'
         },
     },
     'loggers': {
         'boards': {
             'handlers': ['console'],
             'level': 'DEBUG',
         }
     },
 }
 
 HAYSTACK_CONNECTIONS = {
     'default': {
         'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine',
         'PATH': os.path.join(os.path.dirname(__file__), 'whoosh_index'),
     },
 }
 
 THEMES = [
     ('md', 'Mystic Dark'),
     ('md_centered', 'Mystic Dark (centered)'),
     ('sw', 'Snow White'),
     ('pg', 'Photon Gray'),
 ]
 
 IMAGE_VIEWERS = [
     ('simple', 'Simple'),
     ('popup', 'Popup'),
 ]
 
 ALLOWED_HOSTS = ['*']
 
 POSTING_DELAY = 20  # seconds
 
 # Websocket settins
 CENTRIFUGE_HOST = 'localhost'
 CENTRIFUGE_PORT = '9090'
 
 CENTRIFUGE_ADDRESS = 'http://{}:{}'.format(CENTRIFUGE_HOST, CENTRIFUGE_PORT)
 CENTRIFUGE_PROJECT_ID = '<project id here>'
 CENTRIFUGE_PROJECT_SECRET = '<project secret here>'
 CENTRIFUGE_TIMEOUT = 5
 
 # Debug middlewares
 MIDDLEWARE_CLASSES += [
     'debug_toolbar.middleware.DebugToolbarMiddleware',
 ]
 
+
 def custom_show_toolbar(request):
     return request.user.has_perm('admin.debug')
 
 DEBUG_TOOLBAR_CONFIG = {
     'ENABLE_STACKTRACES': True,
     'SHOW_TOOLBAR_CALLBACK': 'neboard.settings.custom_show_toolbar',
 }
 
 # FIXME Uncommenting this fails somehow. Need to investigate this
 #DEBUG_TOOLBAR_PANELS += (
 #    'debug_toolbar.panels.profiling.ProfilingDebugPanel',
 #)

readme.markdown

@@ -1,37 +1,70 b''
 # INTRO #
 
 This project aims to create centralized forum-like discussion platform with
 anonymity in mind.
 
 Main repository: https://bitbucket.org/neko259/neboard/
 
 Site: http://neboard.me/
 
 # INSTALLATION #
 
-1. Install all dependencies over pip or system-wide
-2. Setup a database in `neboard/settings.py`
-3. Run `./manage.py migrate` to apply all south migrations
-4. Apply config changes to `boards/config/config.ini`. You can see the default settings in `boards/config/default_config.ini`
+1. Download application and move inside it:
+
+`hg clone https://bitbucket.org/neko259/neboard`
+
+`cd neboard`
+
+If you wish to use *decentral* version, change branch to *decentral*:
+
+`hg up decentral`
+
+2. Install all application dependencies:
+
+Some minimal system-wide depenencies:
+
+* python3
+* pip/pip3
+* jpeg
+
+Python dependencies:
+
+`pip3 install -r requirements.txt`
+
+You can use virtualenv to speed up the process or avoid conflicts.
+
+3. Setup a database in `neboard/settings.py`. You can also change other settings like search engine.
+
+Depending on configured database and search engine, you need to install corresponding dependencies manually.
+
+Default database is *sqlite*, default search engine is *whoosh*.
+
+4. Setup SECRET_KEY to a secret value in `neboard/settings.py
+5. Run `./manage.py migrate` to apply all migrations
+6. Apply config changes to `boards/config/config.ini`. You can see the default settings in `boards/config/default_config.ini`
+7. If you want to use decetral engine, run `./manage.py generate_keypair` to generate keys
 
 # RUNNING #
 
-You can run the server using django default embedded webserver by running
+You can run the server using django default embedded webserver by running:
 
     ./manage.py runserver <address>:<port>
 
-See django-admin command help for details
+See django-admin command help for details.
 
 Also consider using wsgi or fcgi interfaces on production servers.
 
+When running for the first time, you need to setup at least one section tag.
+Go to the admin page and manually create one tag with "required" property set.
+
 # UPGRADE #
 
 1. Backup your project data.
 2. Copy the project contents over the old project directory
 3. Run migrations by `./manage.py migrate`
 
 You can also just clone the mercurial project and pull it to update
 
 # CONCLUSION #
 
-Enjoy our software and thank you!
+Enjoy our software and thank you!
  No newline at end of file

requirements.txt

@@ -1,10 +1,13 b''
 python-magic
+httplib2
+simplejson
 pytube
 requests
 adjacent
 django-haystack
 pillow
 django>=1.8
 bbcode
 django-debug-toolbar
 pytz
+ecdsa
  No newline at end of file

todo.txt

@@ -1,26 +1,27 b''
 = Features =
 * Tree view (JS)
 * Adding tags to images filename
 * Federative network for s2s communication
 * XMPP gate
 * Bitmessage gate
 * Notification engine
 * Group tags by first letter in all tags list
 * [JS] Character counter in the post field
 * Statistics module. Count views (optional, may result in bad
  performance), posts per day/week/month, IPs
 * Ban confirmation page (or alert) with reason
 * Post deletion confirmation page (or alert)
 * Get thread graph image using pygraphviz
 * Subscribing to tag via AJAX
 * Add buttons to insert a named link or a named quote to the markup panel
 * Add support for "attention posts" that are shown in the header"
+* Use absolute post reflinks in the raw text
 * Use default boards/settings when it is not defined. Maybe default_settings.py module?
 
 = Bugs =
 * Search sort order is confusing
 
 = Testing =
 * Make tests for every view
 * Make tests for every model
 * Make tests for every form