diff --git a/boards/__init__.py b/boards/__init__.py
--- a/boards/__init__.py
+++ b/boards/__init__.py
@@ -0,0 +1,1 @@
+default_app_config = 'boards.apps.BoardsAppConfig'
\ No newline at end of file
diff --git a/boards/admin.py b/boards/admin.py
--- a/boards/admin.py
+++ b/boards/admin.py
@@ -1,7 +1,7 @@
from django.contrib import admin
from django.utils.translation import ugettext_lazy as _
from django.core.urlresolvers import reverse
-from boards.models import Post, Tag, Ban, Thread, Banner, PostImage
+from boards.models import Post, Tag, Ban, Thread, Banner, PostImage, KeyPair, GlobalId
@admin.register(Post)
@@ -62,7 +62,6 @@ class TagAdmin(admin.ModelAdmin):
super().save_model(request, obj, form, change)
for thread in obj.get_threads().all():
thread.refresh_tags()
-
list_display = ('name', 'thread_count', 'display_children')
search_fields = ('name',)
@@ -89,7 +88,6 @@ class ThreadAdmin(admin.ModelAdmin):
def save_related(self, request, form, formsets, change):
super().save_related(request, form, formsets, change)
form.instance.refresh_tags()
-
list_display = ('id', 'op', 'title', 'reply_count', 'status', 'ip',
'display_tags')
list_filter = ('bump_time', 'status')
@@ -97,6 +95,13 @@ class ThreadAdmin(admin.ModelAdmin):
filter_horizontal = ('tags',)
+@admin.register(KeyPair)
+class KeyPairAdmin(admin.ModelAdmin):
+ list_display = ('public_key', 'primary')
+ list_filter = ('primary',)
+ search_fields = ('public_key',)
+
+
@admin.register(Ban)
class BanAdmin(admin.ModelAdmin):
list_display = ('ip', 'can_read')
@@ -112,3 +117,11 @@ class BannerAdmin(admin.ModelAdmin):
@admin.register(PostImage)
class PostImageAdmin(admin.ModelAdmin):
search_fields = ('alias',)
+
+
+@admin.register(GlobalId)
+class GlobalIdAdmin(admin.ModelAdmin):
+ def is_linked(self, obj):
+ return Post.objects.filter(global_id=obj).exists()
+
+ list_display = ('__str__', 'is_linked',)
\ No newline at end of file
diff --git a/boards/apps.py b/boards/apps.py
new file mode 100644
--- /dev/null
+++ b/boards/apps.py
@@ -0,0 +1,11 @@
+from django.apps import AppConfig
+
+
+class BoardsAppConfig(AppConfig):
+ name = 'boards'
+ verbose_name = 'Boards'
+
+ def ready(self):
+ super().ready()
+
+ import boards.signals
\ No newline at end of file
diff --git a/boards/forms.py b/boards/forms.py
--- a/boards/forms.py
+++ b/boards/forms.py
@@ -15,7 +15,7 @@ from django.utils import timezone
from boards.abstracts.settingsmanager import get_settings_manager
from boards.abstracts.attachment_alias import get_image_by_alias
from boards.mdx_neboard import formatters
-from boards.models.attachment.downloaders import Downloader
+from boards.models.attachment.downloaders import download
from boards.models.post import TITLE_MAX_LENGTH
from boards.models import Tag, Post
from boards.utils import validate_file_size, get_file_mimetype, \
@@ -375,12 +375,7 @@ class PostForm(NeboardForm):
img_temp = None
try:
- for downloader in Downloader.__subclasses__():
- if downloader.handles(url):
- return downloader.download(url)
- # If nobody of the specific downloaders handles this, use generic
- # one
- return Downloader.download(url)
+ download(url)
except forms.ValidationError as e:
raise e
except Exception as e:
diff --git a/boards/management/commands/enforce_privacy.py b/boards/management/commands/enforce_privacy.py
--- a/boards/management/commands/enforce_privacy.py
+++ b/boards/management/commands/enforce_privacy.py
@@ -2,8 +2,7 @@ from django.core.management import BaseC
from django.db import transaction
from boards.models import Post
-from boards.models.post import NO_IP
-
+from boards.models.post.manager import NO_IP
__author__ = 'neko259'
diff --git a/boards/management/commands/generate_keypair.py b/boards/management/commands/generate_keypair.py
new file mode 100644
--- /dev/null
+++ b/boards/management/commands/generate_keypair.py
@@ -0,0 +1,21 @@
+__author__ = 'neko259'
+
+
+from django.core.management import BaseCommand
+from django.db import transaction
+
+from boards.models import KeyPair, Post
+
+
+class Command(BaseCommand):
+ help = 'Generates the new keypair. The first one will be primary.'
+
+ @transaction.atomic
+ def handle(self, *args, **options):
+ first_key = not KeyPair.objects.has_primary()
+ key = KeyPair.objects.generate_key(
+ primary=first_key)
+ print(key)
+
+ for post in Post.objects.filter(global_id=None):
+ post.set_global_id()
diff --git a/boards/management/commands/sync_with_server.py b/boards/management/commands/sync_with_server.py
new file mode 100644
--- /dev/null
+++ b/boards/management/commands/sync_with_server.py
@@ -0,0 +1,80 @@
+import re
+import xml.etree.ElementTree as ET
+
+import httplib2
+from django.core.management import BaseCommand
+
+from boards.models import GlobalId
+from boards.models.post.sync import SyncManager
+
+__author__ = 'neko259'
+
+
+REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)')
+
+
+class Command(BaseCommand):
+ help = 'Send a sync or get request to the server.'
+
+ def add_arguments(self, parser):
+ parser.add_argument('url', type=str, help='Server root url')
+ parser.add_argument('--global-id', type=str, default='',
+ help='Post global ID')
+
+ def handle(self, *args, **options):
+ url = options.get('url')
+
+ pull_url = url + 'api/sync/pull/'
+ get_url = url + 'api/sync/get/'
+ file_url = url[:-1]
+
+ global_id_str = options.get('global_id')
+ if global_id_str:
+ match = REGEX_GLOBAL_ID.match(global_id_str)
+ if match:
+ key_type = match.group(1)
+ key = match.group(2)
+ local_id = match.group(3)
+
+ global_id = GlobalId(key_type=key_type, key=key,
+ local_id=local_id)
+
+ xml = GlobalId.objects.generate_request_get([global_id])
+ # body = urllib.parse.urlencode(data)
+ h = httplib2.Http()
+ response, content = h.request(get_url, method="POST", body=xml)
+
+ SyncManager.parse_response_get(content, file_url)
+ else:
+ raise Exception('Invalid global ID')
+ else:
+ h = httplib2.Http()
+ xml = GlobalId.objects.generate_request_pull()
+ response, content = h.request(pull_url, method="POST", body=xml)
+
+ print(content.decode() + '\n')
+
+ root = ET.fromstring(content)
+ status = root.findall('status')[0].text
+ if status == 'success':
+ ids_to_sync = list()
+
+ models = root.findall('models')[0]
+ for model in models:
+ global_id, exists = GlobalId.from_xml_element(model)
+ if not exists:
+ print(global_id)
+ ids_to_sync.append(global_id)
+ print()
+
+ if len(ids_to_sync) > 0:
+ xml = GlobalId.objects.generate_request_get(ids_to_sync)
+ # body = urllib.parse.urlencode(data)
+ h = httplib2.Http()
+ response, content = h.request(get_url, method="POST", body=xml)
+
+ SyncManager.parse_response_get(content, file_url)
+ else:
+ print('Nothing to get, everything synced')
+ else:
+ raise Exception('Invalid response status')
diff --git a/boards/mdx_neboard.py b/boards/mdx_neboard.py
--- a/boards/mdx_neboard.py
+++ b/boards/mdx_neboard.py
@@ -16,6 +16,7 @@ import boards
REFLINK_PATTERN = re.compile(r'^\d+$')
+GLOBAL_REFLINK_PATTERN = re.compile(r'(\w+)::([^:]+)::(\d+)')
MULTI_NEWLINES_PATTERN = re.compile(r'(\r?\n){2,}')
ONE_NEWLINE = '\n'
REGEX_URL = re.compile(r'https?\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(/\S*)?')
@@ -121,15 +122,27 @@ class CodePattern(TextFormatter):
def render_reflink(tag_name, value, options, parent, context):
result = '>>%s' % value
+ post = None
if REFLINK_PATTERN.match(value):
post_id = int(value)
try:
post = boards.models.Post.objects.get(id=post_id)
- result = post.get_link_view()
except ObjectDoesNotExist:
pass
+ elif GLOBAL_REFLINK_PATTERN.match(value):
+ match = GLOBAL_REFLINK_PATTERN.search(value)
+ try:
+ global_id = boards.models.GlobalId.objects.get(
+ key_type=match.group(1), key=match.group(2),
+ local_id=match.group(3))
+ post = global_id.post
+ except ObjectDoesNotExist:
+ pass
+
+ if post is not None:
+ result = post.get_link_view()
return result
@@ -177,9 +190,6 @@ def render_spoiler(tag_name, value, opti
return '{}{}{}'.format(side_spaces, value,
side_spaces)
- return quote_element
-
-
formatters = [
QuotePattern,
diff --git a/boards/migrations/0026_auto_20150830_2006.py b/boards/migrations/0026_auto_20150830_2006.py
new file mode 100644
--- /dev/null
+++ b/boards/migrations/0026_auto_20150830_2006.py
@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('boards', '0025_auto_20150825_2049'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='GlobalId',
+ fields=[
+ ('id', models.AutoField(serialize=False, verbose_name='ID', primary_key=True, auto_created=True)),
+ ('key', models.TextField()),
+ ('key_type', models.TextField()),
+ ('local_id', models.IntegerField()),
+ ],
+ ),
+ migrations.CreateModel(
+ name='KeyPair',
+ fields=[
+ ('id', models.AutoField(serialize=False, verbose_name='ID', primary_key=True, auto_created=True)),
+ ('public_key', models.TextField()),
+ ('private_key', models.TextField()),
+ ('key_type', models.TextField()),
+ ('primary', models.BooleanField(default=False)),
+ ],
+ ),
+ migrations.CreateModel(
+ name='Signature',
+ fields=[
+ ('id', models.AutoField(serialize=False, verbose_name='ID', primary_key=True, auto_created=True)),
+ ('key_type', models.TextField()),
+ ('key', models.TextField()),
+ ('signature', models.TextField()),
+ ('global_id', models.ForeignKey(to='boards.GlobalId')),
+ ],
+ ),
+ migrations.AddField(
+ model_name='post',
+ name='global_id',
+ field=models.OneToOneField(to='boards.GlobalId', null=True, blank=True),
+ ),
+ ]
diff --git a/boards/migrations/0031_merge.py b/boards/migrations/0031_merge.py
new file mode 100644
--- /dev/null
+++ b/boards/migrations/0031_merge.py
@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('boards', '0030_auto_20150929_1816'),
+ ('boards', '0026_auto_20150830_2006'),
+ ]
+
+ operations = [
+ ]
diff --git a/boards/migrations/0036_merge.py b/boards/migrations/0036_merge.py
new file mode 100644
--- /dev/null
+++ b/boards/migrations/0036_merge.py
@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('boards', '0031_merge'),
+ ('boards', '0035_auto_20151021_1346'),
+ ]
+
+ operations = [
+ ]
diff --git a/boards/migrations/0041_merge.py b/boards/migrations/0041_merge.py
new file mode 100644
--- /dev/null
+++ b/boards/migrations/0041_merge.py
@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('boards', '0040_thread_monochrome'),
+ ('boards', '0036_merge'),
+ ]
+
+ operations = [
+ ]
diff --git a/boards/migrations/0043_merge.py b/boards/migrations/0043_merge.py
new file mode 100644
--- /dev/null
+++ b/boards/migrations/0043_merge.py
@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.5 on 2016-04-29 15:52
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('boards', '0041_merge'),
+ ('boards', '0042_auto_20160422_1053'),
+ ]
+
+ operations = [
+ ]
diff --git a/boards/migrations/0044_globalid_content.py b/boards/migrations/0044_globalid_content.py
new file mode 100644
--- /dev/null
+++ b/boards/migrations/0044_globalid_content.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.5 on 2016-05-04 15:36
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('boards', '0043_merge'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='globalid',
+ name='content',
+ field=models.TextField(blank=True, null=True),
+ ),
+ ]
diff --git a/boards/models/__init__.py b/boards/models/__init__.py
--- a/boards/models/__init__.py
+++ b/boards/models/__init__.py
@@ -3,6 +3,8 @@ STATUS_BUMPLIMIT = 'bumplimit'
STATUS_ARCHIVE = 'archived'
+from boards.models.signature import GlobalId, Signature
+from boards.models.sync_key import KeyPair
from boards.models.image import PostImage
from boards.models.attachment import Attachment
from boards.models.thread import Thread
diff --git a/boards/models/attachment/downloaders.py b/boards/models/attachment/downloaders.py
--- a/boards/models/attachment/downloaders.py
+++ b/boards/models/attachment/downloaders.py
@@ -54,6 +54,15 @@ class Downloader:
return file
+def download(url):
+ for downloader in Downloader.__subclasses__():
+ if downloader.handles(url):
+ return downloader.download(url)
+ # If nobody of the specific downloaders handles this, use generic
+ # one
+ return Downloader.download(url)
+
+
class YouTubeDownloader(Downloader):
@staticmethod
def download(url: str):
diff --git a/boards/models/post/__init__.py b/boards/models/post/__init__.py
--- a/boards/models/post/__init__.py
+++ b/boards/models/post/__init__.py
@@ -1,26 +1,19 @@
-import logging
-import re
import uuid
+import re
+from boards import settings
+from boards.abstracts.tripcode import Tripcode
+from boards.models import PostImage, Attachment, KeyPair, GlobalId
+from boards.models.base import Viewable
+from boards.models.post.export import get_exporter, DIFF_TYPE_JSON
+from boards.models.post.manager import PostManager
from boards.utils import datetime_to_epoch
from django.core.exceptions import ObjectDoesNotExist
from django.core.urlresolvers import reverse
from django.db import models
from django.db.models import TextField, QuerySet
-from django.template.defaultfilters import striptags, truncatewords
+from django.template.defaultfilters import truncatewords, striptags
from django.template.loader import render_to_string
-from django.utils import timezone
-from django.db.models.signals import post_save, pre_save
-from django.dispatch import receiver
-
-from boards import settings
-from boards.abstracts.tripcode import Tripcode
-from boards.mdx_neboard import get_parser
-from boards.models import PostImage, Attachment
-from boards.models.base import Viewable
-from boards.models.post.export import get_exporter, DIFF_TYPE_JSON
-from boards.models.post.manager import PostManager
-from boards.models.user import Notification
CSS_CLS_HIDDEN_POST = 'hidden_post'
CSS_CLS_DEAD_POST = 'dead_post'
@@ -39,6 +32,8 @@ IMAGE_THUMB_SIZE = (200, 150)
TITLE_MAX_LENGTH = 200
REGEX_REPLY = re.compile(r'\[post\](\d+)\[/post\]')
+REGEX_GLOBAL_REPLY = re.compile(r'\[post\](\w+)::([^:]+)::(\d+)\[/post\]')
+REGEX_URL = re.compile(r'https?\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(/\S*)?')
REGEX_NOTIFICATION = re.compile(r'\[user\](\w+)\[/user\]')
PARAMETER_TRUNCATED = 'truncated'
@@ -101,6 +96,11 @@ class Post(models.Model, Viewable):
url = models.TextField()
uid = models.TextField(db_index=True)
+ # Global ID with author key. If the message was downloaded from another
+ # server, this indicates the server.
+ global_id = models.OneToOneField(GlobalId, null=True, blank=True,
+ on_delete=models.CASCADE)
+
tripcode = models.CharField(max_length=50, blank=True, default='')
opening = models.BooleanField(db_index=True)
hidden = models.BooleanField(default=False)
@@ -214,29 +214,54 @@ class Post(models.Model, Viewable):
def get_first_image(self) -> PostImage:
return self.images.earliest('id')
- def delete(self, using=None):
+ def set_global_id(self, key_pair=None):
"""
- Deletes all post images and the post itself.
+ Sets global id based on the given key pair. If no key pair is given,
+ default one is used.
"""
- for image in self.images.all():
- image_refs_count = image.post_images.count()
- if image_refs_count == 1:
- image.delete()
+ if key_pair:
+ key = key_pair
+ else:
+ try:
+ key = KeyPair.objects.get(primary=True)
+ except KeyPair.DoesNotExist:
+ # Do not update the global id because there is no key defined
+ return
+ global_id = GlobalId(key_type=key.key_type,
+ key=key.public_key,
+ local_id=self.id)
+ global_id.save()
+
+ self.global_id = global_id
+
+ self.save(update_fields=['global_id'])
+
+ def get_pub_time_str(self):
+ return str(self.pub_time)
- for attachment in self.attachments.all():
- attachment_refs_count = attachment.attachment_posts.count()
- if attachment_refs_count == 1:
- attachment.delete()
+ def get_replied_ids(self):
+ """
+ Gets ID list of the posts that this post replies.
+ """
+
+ raw_text = self.get_raw_text()
- thread = self.get_thread()
- thread.last_edit_time = timezone.now()
- thread.save()
+ local_replied = REGEX_REPLY.findall(raw_text)
+ global_replied = []
+ for match in REGEX_GLOBAL_REPLY.findall(raw_text):
+ key_type = match[0]
+ key = match[1]
+ local_id = match[2]
- super(Post, self).delete(using)
-
- logging.getLogger('boards.post.delete').info(
- 'Deleted post {}'.format(self))
+ try:
+ global_id = GlobalId.objects.get(key_type=key_type,
+ key=key, local_id=local_id)
+ for post in Post.objects.filter(global_id=global_id).only('id'):
+ global_replied.append(post.id)
+ except GlobalId.DoesNotExist:
+ pass
+ return local_replied + global_replied
def get_post_data(self, format_type=DIFF_TYPE_JSON, request=None,
include_last_update=False) -> str:
@@ -305,6 +330,24 @@ class Post(models.Model, Viewable):
def get_raw_text(self) -> str:
return self.text
+ def get_sync_text(self) -> str:
+ """
+ Returns text applicable for sync. It has absolute post reflinks.
+ """
+
+ replacements = dict()
+ for post_id in REGEX_REPLY.findall(self.get_raw_text()):
+ absolute_post_id = str(Post.objects.get(id=post_id).global_id)
+ replacements[post_id] = absolute_post_id
+
+ text = self.get_raw_text() or ''
+ for key in replacements:
+ text = text.replace('[post]{}[/post]'.format(key),
+ '[post]{}[/post]'.format(replacements[key]))
+ text = text.replace('\r\n', '\n').replace('\r', '\n')
+
+ return text
+
def connect_threads(self, opening_posts):
for opening_post in opening_posts:
threads = opening_post.get_threads().all()
@@ -336,34 +379,3 @@ class Post(models.Model, Viewable):
def set_hidden(self, hidden):
self.hidden = hidden
-
-
-# SIGNALS (Maybe move to other module?)
-@receiver(post_save, sender=Post)
-def connect_replies(instance, **kwargs):
- for reply_number in re.finditer(REGEX_REPLY, instance.get_raw_text()):
- post_id = reply_number.group(1)
-
- try:
- referenced_post = Post.objects.get(id=post_id)
-
- # Connect only to posts that are not connected to already
- if not referenced_post.referenced_posts.filter(id=instance.id).exists():
- referenced_post.referenced_posts.add(instance)
- referenced_post.last_edit_time = instance.pub_time
- referenced_post.build_refmap()
- referenced_post.save(update_fields=['refmap', 'last_edit_time'])
- except ObjectDoesNotExist:
- pass
-
-
-@receiver(post_save, sender=Post)
-def connect_notifications(instance, **kwargs):
- for reply_number in re.finditer(REGEX_NOTIFICATION, instance.get_raw_text()):
- user_name = reply_number.group(1).lower()
- Notification.objects.get_or_create(name=user_name, post=instance)
-
-
-@receiver(pre_save, sender=Post)
-def preparse_text(instance, **kwargs):
- instance._text_rendered = get_parser().parse(instance.get_raw_text())
diff --git a/boards/models/post/manager.py b/boards/models/post/manager.py
--- a/boards/models/post/manager.py
+++ b/boards/models/post/manager.py
@@ -80,17 +80,13 @@ class PostManager(models.Manager):
logger.info('Created post [{}] with text [{}] by {}'.format(post,
post.get_text(),post.poster_ip))
- # TODO Move this to other place
if file:
- file_type = file.name.split('.')[-1].lower()
- if file_type in IMAGE_TYPES:
- post.images.add(PostImage.objects.create_with_hash(file))
- else:
- post.attachments.add(Attachment.objects.create_with_hash(file))
+ self._add_file_to_post(file, post)
for image in images:
post.images.add(image)
post.connect_threads(opening_posts)
+ post.set_global_id()
# Thread needs to be bumped only when the post is already created
if not new_thread:
@@ -131,3 +127,34 @@ class PostManager(models.Manager):
return ppd
+ @transaction.atomic
+ def import_post(self, title: str, text: str, pub_time: str, global_id,
+ opening_post=None, tags=list(), files=list()):
+ is_opening = opening_post is None
+ if is_opening:
+ thread = boards.models.thread.Thread.objects.create(
+ bump_time=pub_time, last_edit_time=pub_time)
+ list(map(thread.tags.add, tags))
+ else:
+ thread = opening_post.get_thread()
+
+ post = self.create(title=title, text=text,
+ pub_time=pub_time,
+ poster_ip=NO_IP,
+ last_edit_time=pub_time,
+ global_id=global_id,
+ opening=is_opening,
+ thread=thread)
+
+ # TODO Add files
+ for file in files:
+ self._add_file_to_post(file, post)
+
+ post.threads.add(thread)
+
+ def _add_file_to_post(self, file, post):
+ file_type = file.name.split('.')[-1].lower()
+ if file_type in IMAGE_TYPES:
+ post.images.add(PostImage.objects.create_with_hash(file))
+ else:
+ post.attachments.add(Attachment.objects.create_with_hash(file))
diff --git a/boards/models/post/sync.py b/boards/models/post/sync.py
new file mode 100644
--- /dev/null
+++ b/boards/models/post/sync.py
@@ -0,0 +1,270 @@
+import xml.etree.ElementTree as et
+
+from boards.models.attachment.downloaders import download
+from boards.utils import get_file_mimetype, get_file_hash
+from django.db import transaction
+from boards.models import KeyPair, GlobalId, Signature, Post, Tag
+
+EXCEPTION_NODE = 'Sync node returned an error: {}'
+EXCEPTION_OP = 'Load the OP first'
+EXCEPTION_DOWNLOAD = 'File was not downloaded'
+EXCEPTION_HASH = 'File hash does not match attachment hash'
+EXCEPTION_SIGNATURE = 'Invalid model signature for {}'
+ENCODING_UNICODE = 'unicode'
+
+TAG_MODEL = 'model'
+TAG_REQUEST = 'request'
+TAG_RESPONSE = 'response'
+TAG_ID = 'id'
+TAG_STATUS = 'status'
+TAG_MODELS = 'models'
+TAG_TITLE = 'title'
+TAG_TEXT = 'text'
+TAG_THREAD = 'thread'
+TAG_PUB_TIME = 'pub-time'
+TAG_SIGNATURES = 'signatures'
+TAG_SIGNATURE = 'signature'
+TAG_CONTENT = 'content'
+TAG_ATTACHMENTS = 'attachments'
+TAG_ATTACHMENT = 'attachment'
+TAG_TAGS = 'tags'
+TAG_TAG = 'tag'
+TAG_ATTACHMENT_REFS = 'attachment-refs'
+TAG_ATTACHMENT_REF = 'attachment-ref'
+
+TYPE_GET = 'get'
+
+ATTR_VERSION = 'version'
+ATTR_TYPE = 'type'
+ATTR_NAME = 'name'
+ATTR_VALUE = 'value'
+ATTR_MIMETYPE = 'mimetype'
+ATTR_KEY = 'key'
+ATTR_REF = 'ref'
+ATTR_URL = 'url'
+
+STATUS_SUCCESS = 'success'
+
+
+class SyncException(Exception):
+ pass
+
+
+class SyncManager:
+ @staticmethod
+ def generate_response_get(model_list: list):
+ response = et.Element(TAG_RESPONSE)
+
+ status = et.SubElement(response, TAG_STATUS)
+ status.text = STATUS_SUCCESS
+
+ models = et.SubElement(response, TAG_MODELS)
+
+ for post in model_list:
+ model = et.SubElement(models, TAG_MODEL)
+ model.set(ATTR_NAME, 'post')
+
+ global_id = post.global_id
+
+ images = post.images.all()
+ attachments = post.attachments.all()
+ if global_id.content:
+ model.append(et.fromstring(global_id.content))
+ if len(images) > 0 or len(attachments) > 0:
+ attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS)
+ for image in images:
+ SyncManager._attachment_to_xml(
+ None, attachment_refs, image.image.file,
+ image.hash, image.image.url)
+ for file in attachments:
+ SyncManager._attachment_to_xml(
+ None, attachment_refs, file.file.file,
+ file.hash, file.file.url)
+ else:
+ content_tag = et.SubElement(model, TAG_CONTENT)
+
+ tag_id = et.SubElement(content_tag, TAG_ID)
+ global_id.to_xml_element(tag_id)
+
+ title = et.SubElement(content_tag, TAG_TITLE)
+ title.text = post.title
+
+ text = et.SubElement(content_tag, TAG_TEXT)
+ text.text = post.get_sync_text()
+
+ thread = post.get_thread()
+ if post.is_opening():
+ tag_tags = et.SubElement(content_tag, TAG_TAGS)
+ for tag in thread.get_tags():
+ tag_tag = et.SubElement(tag_tags, TAG_TAG)
+ tag_tag.text = tag.name
+ else:
+ tag_thread = et.SubElement(content_tag, TAG_THREAD)
+ thread_id = et.SubElement(tag_thread, TAG_ID)
+ thread.get_opening_post().global_id.to_xml_element(thread_id)
+
+ pub_time = et.SubElement(content_tag, TAG_PUB_TIME)
+ pub_time.text = str(post.get_pub_time_str())
+
+ if len(images) > 0 or len(attachments) > 0:
+ attachments_tag = et.SubElement(content_tag, TAG_ATTACHMENTS)
+ attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS)
+
+ for image in images:
+ SyncManager._attachment_to_xml(
+ attachments_tag, attachment_refs, image.image.file,
+ image.hash, image.image.url)
+ for file in attachments:
+ SyncManager._attachment_to_xml(
+ attachments_tag, attachment_refs, file.file.file,
+ file.hash, file.file.url)
+
+ global_id.content = et.tostring(content_tag, ENCODING_UNICODE)
+ global_id.save()
+
+ signatures_tag = et.SubElement(model, TAG_SIGNATURES)
+ post_signatures = global_id.signature_set.all()
+ if post_signatures:
+ signatures = post_signatures
+ else:
+ key = KeyPair.objects.get(public_key=global_id.key)
+ signature = Signature(
+ key_type=key.key_type,
+ key=key.public_key,
+ signature=key.sign(global_id.content),
+ global_id=global_id,
+ )
+ signature.save()
+ signatures = [signature]
+ for signature in signatures:
+ signature_tag = et.SubElement(signatures_tag, TAG_SIGNATURE)
+ signature_tag.set(ATTR_TYPE, signature.key_type)
+ signature_tag.set(ATTR_VALUE, signature.signature)
+ signature_tag.set(ATTR_KEY, signature.key)
+
+ return et.tostring(response, ENCODING_UNICODE)
+
+ @staticmethod
+ @transaction.atomic
+ def parse_response_get(response_xml, hostname):
+ tag_root = et.fromstring(response_xml)
+ tag_status = tag_root.find(TAG_STATUS)
+ if STATUS_SUCCESS == tag_status.text:
+ tag_models = tag_root.find(TAG_MODELS)
+ for tag_model in tag_models:
+ tag_content = tag_model.find(TAG_CONTENT)
+
+ content_str = et.tostring(tag_content, ENCODING_UNICODE)
+ signatures = SyncManager._verify_model(content_str, tag_model)
+
+ tag_id = tag_content.find(TAG_ID)
+ global_id, exists = GlobalId.from_xml_element(tag_id)
+
+ if exists:
+ print('Post with same ID already exists')
+ else:
+ global_id.content = content_str
+ global_id.save()
+ for signature in signatures:
+ signature.global_id = global_id
+ signature.save()
+
+ title = tag_content.find(TAG_TITLE).text or ''
+ text = tag_content.find(TAG_TEXT).text or ''
+ pub_time = tag_content.find(TAG_PUB_TIME).text
+
+ thread = tag_content.find(TAG_THREAD)
+ tags = []
+ if thread:
+ thread_id = thread.find(TAG_ID)
+ op_global_id, exists = GlobalId.from_xml_element(thread_id)
+ if exists:
+ opening_post = Post.objects.get(global_id=op_global_id)
+ else:
+ raise SyncException(EXCEPTION_OP)
+ else:
+ opening_post = None
+ tag_tags = tag_content.find(TAG_TAGS)
+ for tag_tag in tag_tags:
+ tag, created = Tag.objects.get_or_create(
+ name=tag_tag.text)
+ tags.append(tag)
+
+ # TODO Check that the replied posts are already present
+ # before adding new ones
+
+ files = []
+ tag_attachments = tag_content.find(TAG_ATTACHMENTS) or list()
+ tag_refs = tag_model.find(TAG_ATTACHMENT_REFS)
+ for attachment in tag_attachments:
+ tag_ref = tag_refs.find("{}[@ref='{}']".format(
+ TAG_ATTACHMENT_REF, attachment.text))
+ url = tag_ref.get(ATTR_URL)
+ attached_file = download(hostname + url)
+ if attached_file is None:
+ raise SyncException(EXCEPTION_DOWNLOAD)
+
+ hash = get_file_hash(attached_file)
+ if hash != attachment.text:
+ raise SyncException(EXCEPTION_HASH)
+
+ files.append(attached_file)
+
+ Post.objects.import_post(
+ title=title, text=text, pub_time=pub_time,
+ opening_post=opening_post, tags=tags,
+ global_id=global_id, files=files)
+ else:
+ raise SyncException(EXCEPTION_NODE.format(tag_status.text))
+
+ @staticmethod
+ def generate_response_pull():
+ response = et.Element(TAG_RESPONSE)
+
+ status = et.SubElement(response, TAG_STATUS)
+ status.text = STATUS_SUCCESS
+
+ models = et.SubElement(response, TAG_MODELS)
+
+ for post in Post.objects.all():
+ tag_id = et.SubElement(models, TAG_ID)
+ post.global_id.to_xml_element(tag_id)
+
+ return et.tostring(response, ENCODING_UNICODE)
+
+ @staticmethod
+ def _verify_model(content_str, tag_model):
+ """
+ Verifies all signatures for a single model.
+ """
+
+ signatures = []
+
+ tag_signatures = tag_model.find(TAG_SIGNATURES)
+ for tag_signature in tag_signatures:
+ signature_type = tag_signature.get(ATTR_TYPE)
+ signature_value = tag_signature.get(ATTR_VALUE)
+ signature_key = tag_signature.get(ATTR_KEY)
+
+ signature = Signature(key_type=signature_type,
+ key=signature_key,
+ signature=signature_value)
+
+ if not KeyPair.objects.verify(signature, content_str):
+ raise SyncException(EXCEPTION_SIGNATURE.format(content_str))
+
+ signatures.append(signature)
+
+ return signatures
+
+ @staticmethod
+ def _attachment_to_xml(tag_attachments, tag_refs, file, hash, url):
+ if tag_attachments is not None:
+ mimetype = get_file_mimetype(file)
+ attachment = et.SubElement(tag_attachments, TAG_ATTACHMENT)
+ attachment.set(ATTR_MIMETYPE, mimetype)
+ attachment.text = hash
+
+ attachment_ref = et.SubElement(tag_refs, TAG_ATTACHMENT_REF)
+ attachment_ref.set(ATTR_REF, hash)
+ attachment_ref.set(ATTR_URL, url)
diff --git a/boards/models/signature.py b/boards/models/signature.py
new file mode 100644
--- /dev/null
+++ b/boards/models/signature.py
@@ -0,0 +1,144 @@
+import xml.etree.ElementTree as et
+from django.db import models
+
+
+TAG_MODEL = 'model'
+TAG_REQUEST = 'request'
+TAG_ID = 'id'
+
+TYPE_GET = 'get'
+TYPE_PULL = 'pull'
+
+ATTR_VERSION = 'version'
+ATTR_TYPE = 'type'
+ATTR_NAME = 'name'
+
+ATTR_KEY = 'key'
+ATTR_KEY_TYPE = 'type'
+ATTR_LOCAL_ID = 'local-id'
+
+
+class GlobalIdManager(models.Manager):
+ def generate_request_get(self, global_id_list: list):
+ """
+ Form a get request from a list of ModelId objects.
+ """
+
+ request = et.Element(TAG_REQUEST)
+ request.set(ATTR_TYPE, TYPE_GET)
+ request.set(ATTR_VERSION, '1.0')
+
+ model = et.SubElement(request, TAG_MODEL)
+ model.set(ATTR_VERSION, '1.0')
+ model.set(ATTR_NAME, 'post')
+
+ for global_id in global_id_list:
+ tag_id = et.SubElement(model, TAG_ID)
+ global_id.to_xml_element(tag_id)
+
+ return et.tostring(request, 'unicode')
+
+ def generate_request_pull(self):
+ """
+ Form a pull request from a list of ModelId objects.
+ """
+
+ request = et.Element(TAG_REQUEST)
+ request.set(ATTR_TYPE, TYPE_PULL)
+ request.set(ATTR_VERSION, '1.0')
+
+ model = et.SubElement(request, TAG_MODEL)
+ model.set(ATTR_VERSION, '1.0')
+ model.set(ATTR_NAME, 'post')
+
+ return et.tostring(request, 'unicode')
+
+ def global_id_exists(self, global_id):
+ """
+ Checks if the same global id already exists in the system.
+ """
+
+ return self.filter(key=global_id.key,
+ key_type=global_id.key_type,
+ local_id=global_id.local_id).exists()
+
+
+class GlobalId(models.Model):
+ """
+ Global model ID and cache.
+ Key, key type and local ID make a single global identificator of the model.
+ Content is an XML cache of the model that can be passed along between nodes
+ without manual serialization each time.
+ """
+ class Meta:
+ app_label = 'boards'
+
+ objects = GlobalIdManager()
+
+ def __init__(self, *args, **kwargs):
+ models.Model.__init__(self, *args, **kwargs)
+
+ if 'key' in kwargs and 'key_type' in kwargs and 'local_id' in kwargs:
+ self.key = kwargs['key']
+ self.key_type = kwargs['key_type']
+ self.local_id = kwargs['local_id']
+
+ key = models.TextField()
+ key_type = models.TextField()
+ local_id = models.IntegerField()
+ content = models.TextField(blank=True, null=True)
+
+ def __str__(self):
+ return '%s::%s::%d' % (self.key_type, self.key, self.local_id)
+
+ def to_xml_element(self, element: et.Element):
+ """
+ Exports global id to an XML element.
+ """
+
+ element.set(ATTR_KEY, self.key)
+ element.set(ATTR_KEY_TYPE, self.key_type)
+ element.set(ATTR_LOCAL_ID, str(self.local_id))
+
+ @staticmethod
+ def from_xml_element(element: et.Element):
+ """
+ Parses XML id tag and gets global id from it.
+
+ Arguments:
+ element -- the XML 'id' element
+
+ Returns:
+ global_id -- id itself
+ exists -- True if the global id was taken from database, False if it
+ did not exist and was created.
+ """
+
+ try:
+ return GlobalId.objects.get(key=element.get(ATTR_KEY),
+ key_type=element.get(ATTR_KEY_TYPE),
+ local_id=int(element.get(
+ ATTR_LOCAL_ID))), True
+ except GlobalId.DoesNotExist:
+ return GlobalId(key=element.get(ATTR_KEY),
+ key_type=element.get(ATTR_KEY_TYPE),
+ local_id=int(element.get(ATTR_LOCAL_ID))), False
+
+
+class Signature(models.Model):
+ class Meta:
+ app_label = 'boards'
+
+ def __init__(self, *args, **kwargs):
+ models.Model.__init__(self, *args, **kwargs)
+
+ if 'key' in kwargs and 'key_type' in kwargs and 'signature' in kwargs:
+ self.key_type = kwargs['key_type']
+ self.key = kwargs['key']
+ self.signature = kwargs['signature']
+
+ key_type = models.TextField()
+ key = models.TextField()
+ signature = models.TextField()
+
+ global_id = models.ForeignKey('GlobalId')
diff --git a/boards/models/sync_key.py b/boards/models/sync_key.py
new file mode 100644
--- /dev/null
+++ b/boards/models/sync_key.py
@@ -0,0 +1,61 @@
+import base64
+from ecdsa import SigningKey, VerifyingKey, BadSignatureError
+from django.db import models
+
+TYPE_ECDSA = 'ecdsa'
+
+APP_LABEL_BOARDS = 'boards'
+
+
+class KeyPairManager(models.Manager):
+ def generate_key(self, key_type=TYPE_ECDSA, primary=False):
+ if primary and self.filter(primary=True).exists():
+ raise Exception('There can be only one primary key')
+
+ if key_type == TYPE_ECDSA:
+ private = SigningKey.generate()
+ public = private.get_verifying_key()
+
+ private_key_str = base64.b64encode(private.to_string()).decode()
+ public_key_str = base64.b64encode(public.to_string()).decode()
+
+ return self.create(public_key=public_key_str,
+ private_key=private_key_str,
+ key_type=TYPE_ECDSA, primary=primary)
+ else:
+ raise Exception('Key type not supported')
+
+ def verify(self, signature, string):
+ if signature.key_type == TYPE_ECDSA:
+ public = VerifyingKey.from_string(base64.b64decode(signature.key))
+ signature_byte = base64.b64decode(signature.signature)
+ try:
+ return public.verify(signature_byte, string.encode())
+ except BadSignatureError:
+ return False
+ else:
+ raise Exception('Key type not supported')
+
+ def has_primary(self):
+ return self.filter(primary=True).exists()
+
+
+class KeyPair(models.Model):
+ class Meta:
+ app_label = APP_LABEL_BOARDS
+
+ objects = KeyPairManager()
+
+ public_key = models.TextField()
+ private_key = models.TextField()
+ key_type = models.TextField()
+ primary = models.BooleanField(default=False)
+
+ def __str__(self):
+ return '%s::%s' % (self.key_type, self.public_key)
+
+ def sign(self, string):
+ private = SigningKey.from_string(base64.b64decode(
+ self.private_key.encode()))
+ signature_byte = private.sign_deterministic(string.encode())
+ return base64.b64encode(signature_byte).decode()
diff --git a/boards/models/user.py b/boards/models/user.py
--- a/boards/models/user.py
+++ b/boards/models/user.py
@@ -1,6 +1,5 @@
from django.db import models
-
-import boards.models.post
+import boards
__author__ = 'neko259'
diff --git a/boards/signals.py b/boards/signals.py
new file mode 100644
--- /dev/null
+++ b/boards/signals.py
@@ -0,0 +1,89 @@
+import re
+from boards.mdx_neboard import get_parser
+
+from boards.models import Post, GlobalId
+from boards.models.post import REGEX_NOTIFICATION
+from boards.models.post import REGEX_REPLY, REGEX_GLOBAL_REPLY
+from boards.models.user import Notification
+from django.db.models.signals import post_save, pre_save, pre_delete, \
+ post_delete
+from django.dispatch import receiver
+from django.utils import timezone
+
+
+@receiver(post_save, sender=Post)
+def connect_replies(instance, **kwargs):
+ for reply_number in re.finditer(REGEX_REPLY, instance.get_raw_text()):
+ post_id = reply_number.group(1)
+
+ try:
+ referenced_post = Post.objects.get(id=post_id)
+
+ if not referenced_post.referenced_posts.filter(
+ id=instance.id).exists():
+ referenced_post.referenced_posts.add(instance)
+ referenced_post.last_edit_time = instance.pub_time
+ referenced_post.build_refmap()
+ referenced_post.save(update_fields=['refmap', 'last_edit_time'])
+ except Post.DoesNotExist:
+ pass
+
+
+@receiver(post_save, sender=Post)
+def connect_global_replies(instance, **kwargs):
+ for reply_number in re.finditer(REGEX_GLOBAL_REPLY, instance.get_raw_text()):
+ key_type = reply_number.group(1)
+ key = reply_number.group(2)
+ local_id = reply_number.group(3)
+
+ try:
+ global_id = GlobalId.objects.get(key_type=key_type, key=key,
+ local_id=local_id)
+ referenced_post = Post.objects.get(global_id=global_id)
+ referenced_post.referenced_posts.add(instance)
+ referenced_post.last_edit_time = instance.pub_time
+ referenced_post.build_refmap()
+ referenced_post.save(update_fields=['refmap', 'last_edit_time'])
+ except (GlobalId.DoesNotExist, Post.DoesNotExist):
+ pass
+
+
+@receiver(post_save, sender=Post)
+def connect_notifications(instance, **kwargs):
+ for reply_number in re.finditer(REGEX_NOTIFICATION, instance.get_raw_text()):
+ user_name = reply_number.group(1).lower()
+ Notification.objects.get_or_create(name=user_name, post=instance)
+
+
+@receiver(pre_save, sender=Post)
+def preparse_text(instance, **kwargs):
+ instance._text_rendered = get_parser().parse(instance.get_raw_text())
+
+
+@receiver(pre_delete, sender=Post)
+def delete_images(instance, **kwargs):
+ for image in instance.images.all():
+ image_refs_count = image.post_images.count()
+ if image_refs_count == 1:
+ image.delete()
+
+
+@receiver(pre_delete, sender=Post)
+def delete_attachments(instance, **kwargs):
+ for attachment in instance.attachments.all():
+ attachment_refs_count = attachment.attachment_posts.count()
+ if attachment_refs_count == 1:
+ attachment.delete()
+
+
+@receiver(post_delete, sender=Post)
+def update_thread_on_delete(instance, **kwargs):
+ thread = instance.get_thread()
+ thread.last_edit_time = timezone.now()
+ thread.save()
+
+
+@receiver(post_delete, sender=Post)
+def delete_global_id(instance, **kwargs):
+ if instance.global_id and instance.global_id.id:
+ instance.global_id.delete()
diff --git a/boards/static/css/md/base_page.css b/boards/static/css/md/base_page.css
--- a/boards/static/css/md/base_page.css
+++ b/boards/static/css/md/base_page.css
@@ -492,6 +492,11 @@ ul {
font-size: 1.2em;
}
+.global-id {
+ font-weight: bolder;
+ opacity: .5;
+}
+
/* Reflink preview */
.post_preview {
border-left: 1px solid #777;
diff --git a/boards/templates/boards/post.html b/boards/templates/boards/post.html
--- a/boards/templates/boards/post.html
+++ b/boards/templates/boards/post.html
@@ -55,7 +55,9 @@
| {% trans 'Edit thread' %}
{% endif %}
{% endif %}
-
+ {% if post.global_id_id %}
+ | RAW
+ {% endif %}
{% endif %}
diff --git a/boards/tests/test_forms.py b/boards/tests/test_forms.py
--- a/boards/tests/test_forms.py
+++ b/boards/tests/test_forms.py
@@ -42,7 +42,7 @@ class FormTest(TestCase):
# Change posting delay so we don't have to wait for 30 seconds or more
old_posting_delay = neboard.settings.POSTING_DELAY
# Wait fot the posting delay or we won't be able to post
- settings.POSTING_DELAY = 1
+ neboard.settings.POSTING_DELAY = 1
time.sleep(neboard.settings.POSTING_DELAY + 1)
response = client.post(THREAD_PAGE_ONE, {'text': TEST_TEXT,
'tags': valid_tags})
diff --git a/boards/tests/test_keys.py b/boards/tests/test_keys.py
new file mode 100644
--- /dev/null
+++ b/boards/tests/test_keys.py
@@ -0,0 +1,89 @@
+from base64 import b64encode
+import logging
+
+from django.test import TestCase
+from boards.models import KeyPair, GlobalId, Post, Signature
+from boards.models.post.sync import SyncManager
+
+logger = logging.getLogger(__name__)
+
+
+class KeyTest(TestCase):
+ def test_create_key(self):
+ key = KeyPair.objects.generate_key('ecdsa')
+
+ self.assertIsNotNone(key, 'The key was not created.')
+
+ def test_validation(self):
+ key = KeyPair.objects.generate_key(key_type='ecdsa')
+ message = 'msg'
+ signature_value = key.sign(message)
+
+ signature = Signature(key_type='ecdsa', key=key.public_key,
+ signature=signature_value)
+ valid = KeyPair.objects.verify(signature, message)
+
+ self.assertTrue(valid, 'Message verification failed.')
+
+ def test_primary_constraint(self):
+ KeyPair.objects.generate_key(key_type='ecdsa', primary=True)
+
+ with self.assertRaises(Exception):
+ KeyPair.objects.generate_key(key_type='ecdsa', primary=True)
+
+ def test_model_id_save(self):
+ model_id = GlobalId(key_type='test', key='test key', local_id='1')
+ model_id.save()
+
+ def test_request_get(self):
+ post = self._create_post_with_key()
+
+ request = GlobalId.objects.generate_request_get([post.global_id])
+ logger.debug(request)
+
+ key = KeyPair.objects.get(primary=True)
+ self.assertTrue(''
+ ''
+ ''
+ ''
+ '' % (
+ key.public_key,
+ key.key_type,
+ ) in request,
+ 'Wrong XML generated for the GET request.')
+
+ def test_response_get(self):
+ post = self._create_post_with_key()
+ reply_post = Post.objects.create_post(title='test_title',
+ text='[post]%d[/post]' % post.id,
+ thread=post.get_thread())
+
+ response = SyncManager.generate_response_get([reply_post])
+ logger.debug(response)
+
+ key = KeyPair.objects.get(primary=True)
+ self.assertTrue('success'
+ ''
+ ''
+ ''
+ ''
+ 'test_title'
+ '[post]%s[/post]'
+ ''
+ '%s'
+ '' % (
+ key.public_key,
+ reply_post.id,
+ key.key_type,
+ str(post.global_id),
+ key.public_key,
+ post.id,
+ key.key_type,
+ str(reply_post.get_pub_time_str()),
+ ) in response,
+ 'Wrong XML generated for the GET response.')
+
+ def _create_post_with_key(self):
+ KeyPair.objects.generate_key(primary=True)
+
+ return Post.objects.create_post(title='test_title', text='test_text')
diff --git a/boards/tests/test_post.py b/boards/tests/test_post.py
--- a/boards/tests/test_post.py
+++ b/boards/tests/test_post.py
@@ -2,7 +2,7 @@ from django.core.paginator import Pagina
from django.test import TestCase
from boards import settings
-from boards.models import Tag, Post, Thread
+from boards.models import Tag, Post, Thread, KeyPair
from boards.models.thread import STATUS_ARCHIVE
@@ -115,6 +115,27 @@ class PostTests(TestCase):
self.assertEqual(all_threads[settings.get_int('View', 'ThreadsPerPage')].id,
first_post.id)
+ def test_reflinks(self):
+ """
+ Tests that reflinks are parsed within post and connecting replies
+ to the replied posts.
+
+ Local reflink example: [post]123[/post]
+ Global reflink example: [post]key_type::key::123[/post]
+ """
+
+ key = KeyPair.objects.generate_key(primary=True)
+
+ tag = Tag.objects.create(name='test_tag')
+
+ post = Post.objects.create_post(title='', text='', tags=[tag])
+ post_local_reflink = Post.objects.create_post(title='',
+ text='[post]%d[/post]' % post.id, thread=post.get_thread())
+
+ self.assertTrue(post_local_reflink in post.referenced_posts.all(),
+ 'Local reflink not connecting posts.')
+
+
def test_thread_replies(self):
"""
Tests that the replies can be queried from a thread in all possible
diff --git a/boards/tests/test_sync.py b/boards/tests/test_sync.py
new file mode 100644
--- /dev/null
+++ b/boards/tests/test_sync.py
@@ -0,0 +1,102 @@
+from boards.models import KeyPair, Post, Tag
+from boards.models.post.sync import SyncManager
+from boards.tests.mocks import MockRequest
+from boards.views.sync import response_get
+
+__author__ = 'neko259'
+
+
+from django.test import TestCase
+
+
+class SyncTest(TestCase):
+ def test_get(self):
+ """
+ Forms a GET request of a post and checks the response.
+ """
+
+ key = KeyPair.objects.generate_key(primary=True)
+ tag = Tag.objects.create(name='tag1')
+ post = Post.objects.create_post(title='test_title',
+ text='test_text\rline two',
+ tags=[tag])
+
+ request = MockRequest()
+ request.body = (
+ ''
+ ''
+ ''
+ ''
+ '' % (post.global_id.key,
+ post.id,
+ post.global_id.key_type)
+ )
+
+ response = response_get(request).content.decode()
+ self.assertTrue(
+ 'success'
+ ''
+ ''
+ ''
+ ''
+ '%s'
+ '%s'
+ '%s'
+ '%s'
+ '' % (
+ post.global_id.key,
+ post.global_id.local_id,
+ post.global_id.key_type,
+ post.title,
+ post.get_sync_text(),
+ post.get_thread().get_tags().first().name,
+ post.get_pub_time_str(),
+ ) in response,
+ 'Wrong response generated for the GET request.')
+
+ post.delete()
+ key.delete()
+
+ KeyPair.objects.generate_key(primary=True)
+
+ SyncManager.parse_response_get(response, None)
+ self.assertEqual(1, Post.objects.count(),
+ 'Post was not created from XML response.')
+
+ parsed_post = Post.objects.first()
+ self.assertEqual('tag1',
+ parsed_post.get_thread().get_tags().first().name,
+ 'Invalid tag was parsed.')
+
+ SyncManager.parse_response_get(response, None)
+ self.assertEqual(1, Post.objects.count(),
+ 'The same post was imported twice.')
+
+ self.assertEqual(1, parsed_post.global_id.signature_set.count(),
+ 'Signature was not saved.')
+
+ post = parsed_post
+
+ # Trying to sync the same once more
+ response = response_get(request).content.decode()
+
+ self.assertTrue(
+ 'success'
+ ''
+ ''
+ ''
+ ''
+ '%s'
+ '%s'
+ '%s'
+ '%s'
+ '' % (
+ post.global_id.key,
+ post.global_id.local_id,
+ post.global_id.key_type,
+ post.title,
+ post.get_sync_text(),
+ post.get_thread().get_tags().first().name,
+ post.get_pub_time_str(),
+ ) in response,
+ 'Wrong response generated for the GET request.')
diff --git a/boards/tests/test_views.py b/boards/tests/test_views.py
--- a/boards/tests/test_views.py
+++ b/boards/tests/test_views.py
@@ -9,8 +9,9 @@ logger = logging.getLogger(__name__)
HTTP_CODE_OK = 200
EXCLUDED_VIEWS = {
- 'banned',
- 'get_thread_diff',
+ 'banned',
+ 'get_thread_diff',
+ 'api_sync_pull',
}
diff --git a/boards/urls.py b/boards/urls.py
--- a/boards/urls.py
+++ b/boards/urls.py
@@ -2,14 +2,17 @@ from django.conf.urls import url
#from django.views.i18n import javascript_catalog
import neboard
+
from boards import views
from boards.rss import AllThreadsFeed, TagThreadsFeed, ThreadPostsFeed
from boards.views import api, tag_threads, all_threads, \
settings, all_tags, feed
from boards.views.authors import AuthorsView
from boards.views.notifications import NotificationView
+from boards.views.search import BoardSearchView
from boards.views.static import StaticPageView
from boards.views.preview import PostPreviewView
+from boards.views.sync import get_post_sync_data, response_get, response_pull
from boards.views.random import RandomImageView
from boards.views.tag_gallery import TagGalleryView
from boards.views.translation import cached_javascript_catalog
@@ -74,12 +77,19 @@ urlpatterns = [
url(r'^api/preview/$', api.api_get_preview, name='preview'),
url(r'^api/new_posts/$', api.api_get_new_posts, name='new_posts'),
+ # Sync protocol API
+ url(r'^api/sync/pull/$', response_pull, name='api_sync_pull'),
+ url(r'^api/sync/get/$', response_get, name='api_sync_pull'),
+ # TODO 'get' request
+
# Notifications
url(r'^notifications/(?P\w+)/$', NotificationView.as_view(), name='notifications'),
url(r'^notifications/$', NotificationView.as_view(), name='notifications'),
# Post preview
url(r'^preview/$', PostPreviewView.as_view(), name='preview'),
+ url(r'^post_xml/(?P\d+)$', get_post_sync_data,
+ name='post_sync_data'),
]
# Search
diff --git a/boards/views/api.py b/boards/views/api.py
--- a/boards/views/api.py
+++ b/boards/views/api.py
@@ -1,26 +1,20 @@
-from collections import OrderedDict
import json
import logging
+from django.core import serializers
from django.db import transaction
-from django.db.models import Count
from django.http import HttpResponse
from django.shortcuts import get_object_or_404
-from django.core import serializers
-from django.template.context_processors import csrf
from django.views.decorators.csrf import csrf_protect
-from boards.abstracts.settingsmanager import get_settings_manager,\
- FAV_THREAD_NO_UPDATES
-
+from boards.abstracts.settingsmanager import get_settings_manager
from boards.forms import PostForm, PlainErrorList
+from boards.mdx_neboard import Parser
from boards.models import Post, Thread, Tag
from boards.models.thread import STATUS_ARCHIVE
+from boards.models.user import Notification
from boards.utils import datetime_to_epoch
from boards.views.thread import ThreadView
-from boards.models.user import Notification
-from boards.mdx_neboard import Parser
-
__author__ = 'neko259'
diff --git a/boards/views/sync.py b/boards/views/sync.py
new file mode 100644
--- /dev/null
+++ b/boards/views/sync.py
@@ -0,0 +1,62 @@
+import xml.etree.ElementTree as et
+import xml.dom.minidom
+
+from django.http import HttpResponse, Http404
+from boards.models import GlobalId, Post
+from boards.models.post.sync import SyncManager
+
+
+def response_pull(request):
+ request_xml = request.body
+
+ if request_xml is None:
+ return HttpResponse(content='Use the API')
+
+ response_xml = SyncManager.generate_response_pull()
+
+ return HttpResponse(content=response_xml)
+
+
+def response_get(request):
+ """
+ Processes a GET request with post ID list and returns the posts XML list.
+ Request should contain an 'xml' post attribute with the actual request XML.
+ """
+
+ request_xml = request.body
+
+ if request_xml is None:
+ return HttpResponse(content='Use the API')
+
+ posts = []
+
+ root_tag = et.fromstring(request_xml)
+ model_tag = root_tag[0]
+ for id_tag in model_tag:
+ global_id, exists = GlobalId.from_xml_element(id_tag)
+ if exists:
+ posts.append(Post.objects.get(global_id=global_id))
+
+ response_xml = SyncManager.generate_response_get(posts)
+
+ return HttpResponse(content=response_xml)
+
+
+def get_post_sync_data(request, post_id):
+ try:
+ post = Post.objects.get(id=post_id)
+ except Post.DoesNotExist:
+ raise Http404()
+
+ xml_str = SyncManager.generate_response_get([post])
+
+ xml_repr = xml.dom.minidom.parseString(xml_str)
+ xml_repr = xml_repr.toprettyxml()
+
+ content = '=Global ID=\n%s\n\n=XML=\n%s' \
+ % (post.global_id, xml_repr)
+
+ return HttpResponse(
+ content_type='text/plain',
+ content=content,
+ )
\ No newline at end of file
diff --git a/development/run.sh b/development/run.sh
new file mode 100755
--- /dev/null
+++ b/development/run.sh
@@ -0,0 +1,3 @@
+#! /usr/bin/env sh
+
+python3 manage.py runserver [::]:8000
diff --git a/development/test.sh b/development/test.sh
new file mode 100755
--- /dev/null
+++ b/development/test.sh
@@ -0,0 +1,3 @@
+#! /usr/bin/env sh
+
+python3 manage.py test
diff --git a/docs/dip-1.markdown b/docs/dip-1.markdown
new file mode 100644
--- /dev/null
+++ b/docs/dip-1.markdown
@@ -0,0 +1,203 @@
+# 0 Title #
+
+DIP-1 Common protocol description
+
+# 1 Intro #
+
+This document describes the Data Interchange Protocol (DIP), designed to
+exchange filtered data that can be stored as a graph structure between
+network nodes.
+
+# 2 Purpose #
+
+This protocol will be used to share the models (originally imageboard posts)
+across multiple servers. The main differnce of this protocol is that the node
+can specify what models it wants to get and from whom. The nodes can get
+models from a specific server, or from all except some specific servers. Also
+the models can be filtered by timestamps or tags.
+
+# 3 Protocol description #
+
+The node requests other node's changes list since some time (since epoch if
+this is the start). The other node sends a list of post ids or posts in the
+XML format.
+
+Protocol version is the version of the sync api. Model version is the version
+of data models. If at least one of them is different, the sync cannot be
+performed.
+
+The node signs the data with its keys. The receiving node saves the key at the
+first sync and checks it every time. If the key has changed, the info won't be
+saved from the node (or the node id must be changed). A model can be signed
+with several keys but at least one of them must be the same as in the global
+ID to verify the sender.
+
+Each node can have several keys. Nodes can have shared keys to serve as a pool
+(several nodes with the same key).
+
+Each post has an ID in the unique format: key-type::key::local-id
+
+All requests pass a request type, protocol and model versions, and a list of
+optional arguments used for filtering.
+
+Each request has its own version. Version consists of 2 numbers: first is
+incompatible version (1.3 and 2.0 are not compatible and must not be in sync)
+and the second one is minor and compatible (for example, new optional field
+is added which will be igroned by those who don't support it yet).
+
+Post edits and reflinks are not saved to the sync model. The replied post ID
+can be got from the post text, and reflinks can be computed when loading
+posts. The edit time is not saved because a foreign post can be 'edited' (new
+replies are added) but the signature must not change (so we can't update the
+content). The inner posts can be edited, and the signature will change then
+but the local-id won't, so the other node can detect that and replace the post
+instead of adding a new one.
+
+## 3.1 Requests ##
+
+There is no constraint on how the server should calculate the request. The
+server can return any information by any filter and the requesting node is
+responsible for validating it.
+
+The server is required to return the status of request. See 3.2 for details.
+
+### 3.1.1 pull ###
+
+"pull" request gets the desired model id list by the given filter (e.g. thread, tags,
+author)
+
+Sample request is as follows:
+
+
+
+
+ 0
+ 0
+
+ tag1
+
+
+
+ abcehy3h9t
+ ehoehyoe
+
+
+
+
+
+
+Under the tag there are filters. Filters for the "post" model can
+be found in DIP-2.
+
+Sample response:
+
+
+
+ success
+
+
+
+
+
+
+
+
+### 3.1.2 get ###
+
+"get" gets models by id list.
+
+Sample request:
+
+
+
+
+
+
+
+
+
+Id consists of a key, key type and local id. This key is used for signing and
+validating of data in the model content.
+
+Sample response:
+
+
+
+
+ success
+
+
+
+
+
+ 13
+ Thirteen
+
+ 12
+
+
+ TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5I
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 13
+ Thirteen
+ 12
+ 13
+
+ tag1
+
+
+
+
+
+
+
+
+
+### 3.1.3 put ###
+
+"put" gives a model to the given node (you have no guarantee the node takes
+it, consider you are just advising the node to take your post). This request
+type is useful in pool where all the nodes try to duplicate all of their data
+across the pool.
+
+## 3.2 Responses ##
+
+### 3.2.1 "not supported" ###
+
+If the request if completely not supported, a "not supported" status will be
+returned.
+
+### 3.2.2 "success" ###
+
+"success" status means the request was processed and the result is returned.
+
+### 3.2.3 "error" ###
+
+If the server knows for sure that the operation cannot be processed, it sends
+the "error" status. Additional tags describing the error may be
+and .
diff --git a/docs/dip-2.markdown b/docs/dip-2.markdown
new file mode 100644
--- /dev/null
+++ b/docs/dip-2.markdown
@@ -0,0 +1,29 @@
+# 0 Title #
+
+"post" model reference
+
+# 1 Description #
+
+"post" is a model that defines an imageboard message, or post.
+
+# 2 Fields #
+
+# 2.1 Mandatory fields #
+
+* title -- text field.
+* text -- text field.
+* pub-time -- timestamp (TBD: Define format).
+
+# 2.2 Optional fields #
+
+* attachments -- defines attachments such as images.
+* attachment -- contains and attachment or link to attachment to be downloaded
+manually. Required attributes are mimetype and name.
+
+This field is used for the opening post (thread):
+
+* tags -- text tag name list.
+
+This field is used for non-opening post:
+
+* thread -- ID of a post model that this post is related to.
diff --git a/neboard/settings.py b/neboard/settings.py
--- a/neboard/settings.py
+++ b/neboard/settings.py
@@ -2,7 +2,6 @@
import os
DEBUG = True
-TEMPLATE_DEBUG = DEBUG
ADMINS = (
# ('Your Name', 'your_email@example.com'),
@@ -67,16 +66,7 @@ STATIC_ROOT = ''
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'
-# Additional locations of static files
-# It is really a hack, put real paths, not related
-STATICFILES_DIRS = (
- os.path.dirname(__file__) + '/boards/static',
-
-# '/d/work/python/django/neboard/neboard/boards/static',
- # Put strings here, like "/home/html/static" or "C:/www/django/static".
- # Always use forward slashes, even on Windows.
- # Don't forget to use absolute paths, not relative paths.
-)
+STATICFILES_DIRS = []
# List of finder classes that know how to find static files in
# various locations.
@@ -227,6 +217,7 @@ MIDDLEWARE_CLASSES += [
'debug_toolbar.middleware.DebugToolbarMiddleware',
]
+
def custom_show_toolbar(request):
return request.user.has_perm('admin.debug')
diff --git a/readme.markdown b/readme.markdown
--- a/readme.markdown
+++ b/readme.markdown
@@ -9,21 +9,54 @@ Site: http://neboard.me/
# INSTALLATION #
-1. Install all dependencies over pip or system-wide
-2. Setup a database in `neboard/settings.py`
-3. Run `./manage.py migrate` to apply all south migrations
-4. Apply config changes to `boards/config/config.ini`. You can see the default settings in `boards/config/default_config.ini`
+1. Download application and move inside it:
+
+`hg clone https://bitbucket.org/neko259/neboard`
+
+`cd neboard`
+
+If you wish to use *decentral* version, change branch to *decentral*:
+
+`hg up decentral`
+
+2. Install all application dependencies:
+
+Some minimal system-wide depenencies:
+
+* python3
+* pip/pip3
+* jpeg
+
+Python dependencies:
+
+`pip3 install -r requirements.txt`
+
+You can use virtualenv to speed up the process or avoid conflicts.
+
+3. Setup a database in `neboard/settings.py`. You can also change other settings like search engine.
+
+Depending on configured database and search engine, you need to install corresponding dependencies manually.
+
+Default database is *sqlite*, default search engine is *whoosh*.
+
+4. Setup SECRET_KEY to a secret value in `neboard/settings.py
+5. Run `./manage.py migrate` to apply all migrations
+6. Apply config changes to `boards/config/config.ini`. You can see the default settings in `boards/config/default_config.ini`
+7. If you want to use decetral engine, run `./manage.py generate_keypair` to generate keys
# RUNNING #
-You can run the server using django default embedded webserver by running
+You can run the server using django default embedded webserver by running:
./manage.py runserver :
-See django-admin command help for details
+See django-admin command help for details.
Also consider using wsgi or fcgi interfaces on production servers.
+When running for the first time, you need to setup at least one section tag.
+Go to the admin page and manually create one tag with "required" property set.
+
# UPGRADE #
1. Backup your project data.
@@ -34,4 +67,4 @@ You can also just clone the mercurial pr
# CONCLUSION #
-Enjoy our software and thank you!
+Enjoy our software and thank you!
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,6 @@
python-magic
+httplib2
+simplejson
pytube
requests
adjacent
@@ -8,3 +10,4 @@ django>=1.8
bbcode
django-debug-toolbar
pytz
+ecdsa
\ No newline at end of file
diff --git a/todo.txt b/todo.txt
--- a/todo.txt
+++ b/todo.txt
@@ -15,6 +15,7 @@
* Subscribing to tag via AJAX
* Add buttons to insert a named link or a named quote to the markup panel
* Add support for "attention posts" that are shown in the header"
+* Use absolute post reflinks in the raw text
* Use default boards/settings when it is not defined. Maybe default_settings.py module?
= Bugs =