# HG changeset patch # User neko259 # Date 2016-05-09 15:59:21 # Node ID 80ed7ef80896ec30d9e4b39dc2e9bba1b959b82c # Parent 5966db37dc85e9252d9d63d3e483680418cf191c # Parent 334c97d60c907dcc09401c0a0ddeb27748043008 Merged the decentral branch into default branch diff --git a/boards/__init__.py b/boards/__init__.py --- a/boards/__init__.py +++ b/boards/__init__.py @@ -0,0 +1,1 @@ +default_app_config = 'boards.apps.BoardsAppConfig' \ No newline at end of file diff --git a/boards/admin.py b/boards/admin.py --- a/boards/admin.py +++ b/boards/admin.py @@ -1,7 +1,7 @@ from django.contrib import admin from django.utils.translation import ugettext_lazy as _ from django.core.urlresolvers import reverse -from boards.models import Post, Tag, Ban, Thread, Banner, PostImage +from boards.models import Post, Tag, Ban, Thread, Banner, PostImage, KeyPair, GlobalId @admin.register(Post) @@ -62,7 +62,6 @@ class TagAdmin(admin.ModelAdmin): super().save_model(request, obj, form, change) for thread in obj.get_threads().all(): thread.refresh_tags() - list_display = ('name', 'thread_count', 'display_children') search_fields = ('name',) @@ -89,7 +88,6 @@ class ThreadAdmin(admin.ModelAdmin): def save_related(self, request, form, formsets, change): super().save_related(request, form, formsets, change) form.instance.refresh_tags() - list_display = ('id', 'op', 'title', 'reply_count', 'status', 'ip', 'display_tags') list_filter = ('bump_time', 'status') @@ -97,6 +95,13 @@ class ThreadAdmin(admin.ModelAdmin): filter_horizontal = ('tags',) +@admin.register(KeyPair) +class KeyPairAdmin(admin.ModelAdmin): + list_display = ('public_key', 'primary') + list_filter = ('primary',) + search_fields = ('public_key',) + + @admin.register(Ban) class BanAdmin(admin.ModelAdmin): list_display = ('ip', 'can_read') @@ -112,3 +117,11 @@ class BannerAdmin(admin.ModelAdmin): @admin.register(PostImage) class PostImageAdmin(admin.ModelAdmin): search_fields = ('alias',) + + +@admin.register(GlobalId) +class GlobalIdAdmin(admin.ModelAdmin): + def is_linked(self, obj): + return Post.objects.filter(global_id=obj).exists() + + list_display = ('__str__', 'is_linked',) \ No newline at end of file diff --git a/boards/apps.py b/boards/apps.py new file mode 100644 --- /dev/null +++ b/boards/apps.py @@ -0,0 +1,11 @@ +from django.apps import AppConfig + + +class BoardsAppConfig(AppConfig): + name = 'boards' + verbose_name = 'Boards' + + def ready(self): + super().ready() + + import boards.signals \ No newline at end of file diff --git a/boards/forms.py b/boards/forms.py --- a/boards/forms.py +++ b/boards/forms.py @@ -15,7 +15,7 @@ from django.utils import timezone from boards.abstracts.settingsmanager import get_settings_manager from boards.abstracts.attachment_alias import get_image_by_alias from boards.mdx_neboard import formatters -from boards.models.attachment.downloaders import Downloader +from boards.models.attachment.downloaders import download from boards.models.post import TITLE_MAX_LENGTH from boards.models import Tag, Post from boards.utils import validate_file_size, get_file_mimetype, \ @@ -375,12 +375,7 @@ class PostForm(NeboardForm): img_temp = None try: - for downloader in Downloader.__subclasses__(): - if downloader.handles(url): - return downloader.download(url) - # If nobody of the specific downloaders handles this, use generic - # one - return Downloader.download(url) + download(url) except forms.ValidationError as e: raise e except Exception as e: diff --git a/boards/management/commands/enforce_privacy.py b/boards/management/commands/enforce_privacy.py --- a/boards/management/commands/enforce_privacy.py +++ b/boards/management/commands/enforce_privacy.py @@ -2,8 +2,7 @@ from django.core.management import BaseC from django.db import transaction from boards.models import Post -from boards.models.post import NO_IP - +from boards.models.post.manager import NO_IP __author__ = 'neko259' diff --git a/boards/management/commands/generate_keypair.py b/boards/management/commands/generate_keypair.py new file mode 100644 --- /dev/null +++ b/boards/management/commands/generate_keypair.py @@ -0,0 +1,21 @@ +__author__ = 'neko259' + + +from django.core.management import BaseCommand +from django.db import transaction + +from boards.models import KeyPair, Post + + +class Command(BaseCommand): + help = 'Generates the new keypair. The first one will be primary.' + + @transaction.atomic + def handle(self, *args, **options): + first_key = not KeyPair.objects.has_primary() + key = KeyPair.objects.generate_key( + primary=first_key) + print(key) + + for post in Post.objects.filter(global_id=None): + post.set_global_id() diff --git a/boards/management/commands/sync_with_server.py b/boards/management/commands/sync_with_server.py new file mode 100644 --- /dev/null +++ b/boards/management/commands/sync_with_server.py @@ -0,0 +1,80 @@ +import re +import xml.etree.ElementTree as ET + +import httplib2 +from django.core.management import BaseCommand + +from boards.models import GlobalId +from boards.models.post.sync import SyncManager + +__author__ = 'neko259' + + +REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)') + + +class Command(BaseCommand): + help = 'Send a sync or get request to the server.' + + def add_arguments(self, parser): + parser.add_argument('url', type=str, help='Server root url') + parser.add_argument('--global-id', type=str, default='', + help='Post global ID') + + def handle(self, *args, **options): + url = options.get('url') + + pull_url = url + 'api/sync/pull/' + get_url = url + 'api/sync/get/' + file_url = url[:-1] + + global_id_str = options.get('global_id') + if global_id_str: + match = REGEX_GLOBAL_ID.match(global_id_str) + if match: + key_type = match.group(1) + key = match.group(2) + local_id = match.group(3) + + global_id = GlobalId(key_type=key_type, key=key, + local_id=local_id) + + xml = GlobalId.objects.generate_request_get([global_id]) + # body = urllib.parse.urlencode(data) + h = httplib2.Http() + response, content = h.request(get_url, method="POST", body=xml) + + SyncManager.parse_response_get(content, file_url) + else: + raise Exception('Invalid global ID') + else: + h = httplib2.Http() + xml = GlobalId.objects.generate_request_pull() + response, content = h.request(pull_url, method="POST", body=xml) + + print(content.decode() + '\n') + + root = ET.fromstring(content) + status = root.findall('status')[0].text + if status == 'success': + ids_to_sync = list() + + models = root.findall('models')[0] + for model in models: + global_id, exists = GlobalId.from_xml_element(model) + if not exists: + print(global_id) + ids_to_sync.append(global_id) + print() + + if len(ids_to_sync) > 0: + xml = GlobalId.objects.generate_request_get(ids_to_sync) + # body = urllib.parse.urlencode(data) + h = httplib2.Http() + response, content = h.request(get_url, method="POST", body=xml) + + SyncManager.parse_response_get(content, file_url) + else: + print('Nothing to get, everything synced') + else: + raise Exception('Invalid response status') diff --git a/boards/mdx_neboard.py b/boards/mdx_neboard.py --- a/boards/mdx_neboard.py +++ b/boards/mdx_neboard.py @@ -16,6 +16,7 @@ import boards REFLINK_PATTERN = re.compile(r'^\d+$') +GLOBAL_REFLINK_PATTERN = re.compile(r'(\w+)::([^:]+)::(\d+)') MULTI_NEWLINES_PATTERN = re.compile(r'(\r?\n){2,}') ONE_NEWLINE = '\n' REGEX_URL = re.compile(r'https?\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(/\S*)?') @@ -121,15 +122,27 @@ class CodePattern(TextFormatter): def render_reflink(tag_name, value, options, parent, context): result = '>>%s' % value + post = None if REFLINK_PATTERN.match(value): post_id = int(value) try: post = boards.models.Post.objects.get(id=post_id) - result = post.get_link_view() except ObjectDoesNotExist: pass + elif GLOBAL_REFLINK_PATTERN.match(value): + match = GLOBAL_REFLINK_PATTERN.search(value) + try: + global_id = boards.models.GlobalId.objects.get( + key_type=match.group(1), key=match.group(2), + local_id=match.group(3)) + post = global_id.post + except ObjectDoesNotExist: + pass + + if post is not None: + result = post.get_link_view() return result @@ -177,9 +190,6 @@ def render_spoiler(tag_name, value, opti return '{}{}{}'.format(side_spaces, value, side_spaces) - return quote_element - - formatters = [ QuotePattern, diff --git a/boards/migrations/0026_auto_20150830_2006.py b/boards/migrations/0026_auto_20150830_2006.py new file mode 100644 --- /dev/null +++ b/boards/migrations/0026_auto_20150830_2006.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('boards', '0025_auto_20150825_2049'), + ] + + operations = [ + migrations.CreateModel( + name='GlobalId', + fields=[ + ('id', models.AutoField(serialize=False, verbose_name='ID', primary_key=True, auto_created=True)), + ('key', models.TextField()), + ('key_type', models.TextField()), + ('local_id', models.IntegerField()), + ], + ), + migrations.CreateModel( + name='KeyPair', + fields=[ + ('id', models.AutoField(serialize=False, verbose_name='ID', primary_key=True, auto_created=True)), + ('public_key', models.TextField()), + ('private_key', models.TextField()), + ('key_type', models.TextField()), + ('primary', models.BooleanField(default=False)), + ], + ), + migrations.CreateModel( + name='Signature', + fields=[ + ('id', models.AutoField(serialize=False, verbose_name='ID', primary_key=True, auto_created=True)), + ('key_type', models.TextField()), + ('key', models.TextField()), + ('signature', models.TextField()), + ('global_id', models.ForeignKey(to='boards.GlobalId')), + ], + ), + migrations.AddField( + model_name='post', + name='global_id', + field=models.OneToOneField(to='boards.GlobalId', null=True, blank=True), + ), + ] diff --git a/boards/migrations/0031_merge.py b/boards/migrations/0031_merge.py new file mode 100644 --- /dev/null +++ b/boards/migrations/0031_merge.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('boards', '0030_auto_20150929_1816'), + ('boards', '0026_auto_20150830_2006'), + ] + + operations = [ + ] diff --git a/boards/migrations/0036_merge.py b/boards/migrations/0036_merge.py new file mode 100644 --- /dev/null +++ b/boards/migrations/0036_merge.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('boards', '0031_merge'), + ('boards', '0035_auto_20151021_1346'), + ] + + operations = [ + ] diff --git a/boards/migrations/0041_merge.py b/boards/migrations/0041_merge.py new file mode 100644 --- /dev/null +++ b/boards/migrations/0041_merge.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('boards', '0040_thread_monochrome'), + ('boards', '0036_merge'), + ] + + operations = [ + ] diff --git a/boards/migrations/0043_merge.py b/boards/migrations/0043_merge.py new file mode 100644 --- /dev/null +++ b/boards/migrations/0043_merge.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-04-29 15:52 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('boards', '0041_merge'), + ('boards', '0042_auto_20160422_1053'), + ] + + operations = [ + ] diff --git a/boards/migrations/0044_globalid_content.py b/boards/migrations/0044_globalid_content.py new file mode 100644 --- /dev/null +++ b/boards/migrations/0044_globalid_content.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-05-04 15:36 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('boards', '0043_merge'), + ] + + operations = [ + migrations.AddField( + model_name='globalid', + name='content', + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/boards/models/__init__.py b/boards/models/__init__.py --- a/boards/models/__init__.py +++ b/boards/models/__init__.py @@ -3,6 +3,8 @@ STATUS_BUMPLIMIT = 'bumplimit' STATUS_ARCHIVE = 'archived' +from boards.models.signature import GlobalId, Signature +from boards.models.sync_key import KeyPair from boards.models.image import PostImage from boards.models.attachment import Attachment from boards.models.thread import Thread diff --git a/boards/models/attachment/downloaders.py b/boards/models/attachment/downloaders.py --- a/boards/models/attachment/downloaders.py +++ b/boards/models/attachment/downloaders.py @@ -54,6 +54,15 @@ class Downloader: return file +def download(url): + for downloader in Downloader.__subclasses__(): + if downloader.handles(url): + return downloader.download(url) + # If nobody of the specific downloaders handles this, use generic + # one + return Downloader.download(url) + + class YouTubeDownloader(Downloader): @staticmethod def download(url: str): diff --git a/boards/models/post/__init__.py b/boards/models/post/__init__.py --- a/boards/models/post/__init__.py +++ b/boards/models/post/__init__.py @@ -1,26 +1,19 @@ -import logging -import re import uuid +import re +from boards import settings +from boards.abstracts.tripcode import Tripcode +from boards.models import PostImage, Attachment, KeyPair, GlobalId +from boards.models.base import Viewable +from boards.models.post.export import get_exporter, DIFF_TYPE_JSON +from boards.models.post.manager import PostManager from boards.utils import datetime_to_epoch from django.core.exceptions import ObjectDoesNotExist from django.core.urlresolvers import reverse from django.db import models from django.db.models import TextField, QuerySet -from django.template.defaultfilters import striptags, truncatewords +from django.template.defaultfilters import truncatewords, striptags from django.template.loader import render_to_string -from django.utils import timezone -from django.db.models.signals import post_save, pre_save -from django.dispatch import receiver - -from boards import settings -from boards.abstracts.tripcode import Tripcode -from boards.mdx_neboard import get_parser -from boards.models import PostImage, Attachment -from boards.models.base import Viewable -from boards.models.post.export import get_exporter, DIFF_TYPE_JSON -from boards.models.post.manager import PostManager -from boards.models.user import Notification CSS_CLS_HIDDEN_POST = 'hidden_post' CSS_CLS_DEAD_POST = 'dead_post' @@ -39,6 +32,8 @@ IMAGE_THUMB_SIZE = (200, 150) TITLE_MAX_LENGTH = 200 REGEX_REPLY = re.compile(r'\[post\](\d+)\[/post\]') +REGEX_GLOBAL_REPLY = re.compile(r'\[post\](\w+)::([^:]+)::(\d+)\[/post\]') +REGEX_URL = re.compile(r'https?\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(/\S*)?') REGEX_NOTIFICATION = re.compile(r'\[user\](\w+)\[/user\]') PARAMETER_TRUNCATED = 'truncated' @@ -101,6 +96,11 @@ class Post(models.Model, Viewable): url = models.TextField() uid = models.TextField(db_index=True) + # Global ID with author key. If the message was downloaded from another + # server, this indicates the server. + global_id = models.OneToOneField(GlobalId, null=True, blank=True, + on_delete=models.CASCADE) + tripcode = models.CharField(max_length=50, blank=True, default='') opening = models.BooleanField(db_index=True) hidden = models.BooleanField(default=False) @@ -214,29 +214,54 @@ class Post(models.Model, Viewable): def get_first_image(self) -> PostImage: return self.images.earliest('id') - def delete(self, using=None): + def set_global_id(self, key_pair=None): """ - Deletes all post images and the post itself. + Sets global id based on the given key pair. If no key pair is given, + default one is used. """ - for image in self.images.all(): - image_refs_count = image.post_images.count() - if image_refs_count == 1: - image.delete() + if key_pair: + key = key_pair + else: + try: + key = KeyPair.objects.get(primary=True) + except KeyPair.DoesNotExist: + # Do not update the global id because there is no key defined + return + global_id = GlobalId(key_type=key.key_type, + key=key.public_key, + local_id=self.id) + global_id.save() + + self.global_id = global_id + + self.save(update_fields=['global_id']) + + def get_pub_time_str(self): + return str(self.pub_time) - for attachment in self.attachments.all(): - attachment_refs_count = attachment.attachment_posts.count() - if attachment_refs_count == 1: - attachment.delete() + def get_replied_ids(self): + """ + Gets ID list of the posts that this post replies. + """ + + raw_text = self.get_raw_text() - thread = self.get_thread() - thread.last_edit_time = timezone.now() - thread.save() + local_replied = REGEX_REPLY.findall(raw_text) + global_replied = [] + for match in REGEX_GLOBAL_REPLY.findall(raw_text): + key_type = match[0] + key = match[1] + local_id = match[2] - super(Post, self).delete(using) - - logging.getLogger('boards.post.delete').info( - 'Deleted post {}'.format(self)) + try: + global_id = GlobalId.objects.get(key_type=key_type, + key=key, local_id=local_id) + for post in Post.objects.filter(global_id=global_id).only('id'): + global_replied.append(post.id) + except GlobalId.DoesNotExist: + pass + return local_replied + global_replied def get_post_data(self, format_type=DIFF_TYPE_JSON, request=None, include_last_update=False) -> str: @@ -305,6 +330,24 @@ class Post(models.Model, Viewable): def get_raw_text(self) -> str: return self.text + def get_sync_text(self) -> str: + """ + Returns text applicable for sync. It has absolute post reflinks. + """ + + replacements = dict() + for post_id in REGEX_REPLY.findall(self.get_raw_text()): + absolute_post_id = str(Post.objects.get(id=post_id).global_id) + replacements[post_id] = absolute_post_id + + text = self.get_raw_text() or '' + for key in replacements: + text = text.replace('[post]{}[/post]'.format(key), + '[post]{}[/post]'.format(replacements[key])) + text = text.replace('\r\n', '\n').replace('\r', '\n') + + return text + def connect_threads(self, opening_posts): for opening_post in opening_posts: threads = opening_post.get_threads().all() @@ -336,34 +379,3 @@ class Post(models.Model, Viewable): def set_hidden(self, hidden): self.hidden = hidden - - -# SIGNALS (Maybe move to other module?) -@receiver(post_save, sender=Post) -def connect_replies(instance, **kwargs): - for reply_number in re.finditer(REGEX_REPLY, instance.get_raw_text()): - post_id = reply_number.group(1) - - try: - referenced_post = Post.objects.get(id=post_id) - - # Connect only to posts that are not connected to already - if not referenced_post.referenced_posts.filter(id=instance.id).exists(): - referenced_post.referenced_posts.add(instance) - referenced_post.last_edit_time = instance.pub_time - referenced_post.build_refmap() - referenced_post.save(update_fields=['refmap', 'last_edit_time']) - except ObjectDoesNotExist: - pass - - -@receiver(post_save, sender=Post) -def connect_notifications(instance, **kwargs): - for reply_number in re.finditer(REGEX_NOTIFICATION, instance.get_raw_text()): - user_name = reply_number.group(1).lower() - Notification.objects.get_or_create(name=user_name, post=instance) - - -@receiver(pre_save, sender=Post) -def preparse_text(instance, **kwargs): - instance._text_rendered = get_parser().parse(instance.get_raw_text()) diff --git a/boards/models/post/manager.py b/boards/models/post/manager.py --- a/boards/models/post/manager.py +++ b/boards/models/post/manager.py @@ -80,17 +80,13 @@ class PostManager(models.Manager): logger.info('Created post [{}] with text [{}] by {}'.format(post, post.get_text(),post.poster_ip)) - # TODO Move this to other place if file: - file_type = file.name.split('.')[-1].lower() - if file_type in IMAGE_TYPES: - post.images.add(PostImage.objects.create_with_hash(file)) - else: - post.attachments.add(Attachment.objects.create_with_hash(file)) + self._add_file_to_post(file, post) for image in images: post.images.add(image) post.connect_threads(opening_posts) + post.set_global_id() # Thread needs to be bumped only when the post is already created if not new_thread: @@ -131,3 +127,34 @@ class PostManager(models.Manager): return ppd + @transaction.atomic + def import_post(self, title: str, text: str, pub_time: str, global_id, + opening_post=None, tags=list(), files=list()): + is_opening = opening_post is None + if is_opening: + thread = boards.models.thread.Thread.objects.create( + bump_time=pub_time, last_edit_time=pub_time) + list(map(thread.tags.add, tags)) + else: + thread = opening_post.get_thread() + + post = self.create(title=title, text=text, + pub_time=pub_time, + poster_ip=NO_IP, + last_edit_time=pub_time, + global_id=global_id, + opening=is_opening, + thread=thread) + + # TODO Add files + for file in files: + self._add_file_to_post(file, post) + + post.threads.add(thread) + + def _add_file_to_post(self, file, post): + file_type = file.name.split('.')[-1].lower() + if file_type in IMAGE_TYPES: + post.images.add(PostImage.objects.create_with_hash(file)) + else: + post.attachments.add(Attachment.objects.create_with_hash(file)) diff --git a/boards/models/post/sync.py b/boards/models/post/sync.py new file mode 100644 --- /dev/null +++ b/boards/models/post/sync.py @@ -0,0 +1,270 @@ +import xml.etree.ElementTree as et + +from boards.models.attachment.downloaders import download +from boards.utils import get_file_mimetype, get_file_hash +from django.db import transaction +from boards.models import KeyPair, GlobalId, Signature, Post, Tag + +EXCEPTION_NODE = 'Sync node returned an error: {}' +EXCEPTION_OP = 'Load the OP first' +EXCEPTION_DOWNLOAD = 'File was not downloaded' +EXCEPTION_HASH = 'File hash does not match attachment hash' +EXCEPTION_SIGNATURE = 'Invalid model signature for {}' +ENCODING_UNICODE = 'unicode' + +TAG_MODEL = 'model' +TAG_REQUEST = 'request' +TAG_RESPONSE = 'response' +TAG_ID = 'id' +TAG_STATUS = 'status' +TAG_MODELS = 'models' +TAG_TITLE = 'title' +TAG_TEXT = 'text' +TAG_THREAD = 'thread' +TAG_PUB_TIME = 'pub-time' +TAG_SIGNATURES = 'signatures' +TAG_SIGNATURE = 'signature' +TAG_CONTENT = 'content' +TAG_ATTACHMENTS = 'attachments' +TAG_ATTACHMENT = 'attachment' +TAG_TAGS = 'tags' +TAG_TAG = 'tag' +TAG_ATTACHMENT_REFS = 'attachment-refs' +TAG_ATTACHMENT_REF = 'attachment-ref' + +TYPE_GET = 'get' + +ATTR_VERSION = 'version' +ATTR_TYPE = 'type' +ATTR_NAME = 'name' +ATTR_VALUE = 'value' +ATTR_MIMETYPE = 'mimetype' +ATTR_KEY = 'key' +ATTR_REF = 'ref' +ATTR_URL = 'url' + +STATUS_SUCCESS = 'success' + + +class SyncException(Exception): + pass + + +class SyncManager: + @staticmethod + def generate_response_get(model_list: list): + response = et.Element(TAG_RESPONSE) + + status = et.SubElement(response, TAG_STATUS) + status.text = STATUS_SUCCESS + + models = et.SubElement(response, TAG_MODELS) + + for post in model_list: + model = et.SubElement(models, TAG_MODEL) + model.set(ATTR_NAME, 'post') + + global_id = post.global_id + + images = post.images.all() + attachments = post.attachments.all() + if global_id.content: + model.append(et.fromstring(global_id.content)) + if len(images) > 0 or len(attachments) > 0: + attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS) + for image in images: + SyncManager._attachment_to_xml( + None, attachment_refs, image.image.file, + image.hash, image.image.url) + for file in attachments: + SyncManager._attachment_to_xml( + None, attachment_refs, file.file.file, + file.hash, file.file.url) + else: + content_tag = et.SubElement(model, TAG_CONTENT) + + tag_id = et.SubElement(content_tag, TAG_ID) + global_id.to_xml_element(tag_id) + + title = et.SubElement(content_tag, TAG_TITLE) + title.text = post.title + + text = et.SubElement(content_tag, TAG_TEXT) + text.text = post.get_sync_text() + + thread = post.get_thread() + if post.is_opening(): + tag_tags = et.SubElement(content_tag, TAG_TAGS) + for tag in thread.get_tags(): + tag_tag = et.SubElement(tag_tags, TAG_TAG) + tag_tag.text = tag.name + else: + tag_thread = et.SubElement(content_tag, TAG_THREAD) + thread_id = et.SubElement(tag_thread, TAG_ID) + thread.get_opening_post().global_id.to_xml_element(thread_id) + + pub_time = et.SubElement(content_tag, TAG_PUB_TIME) + pub_time.text = str(post.get_pub_time_str()) + + if len(images) > 0 or len(attachments) > 0: + attachments_tag = et.SubElement(content_tag, TAG_ATTACHMENTS) + attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS) + + for image in images: + SyncManager._attachment_to_xml( + attachments_tag, attachment_refs, image.image.file, + image.hash, image.image.url) + for file in attachments: + SyncManager._attachment_to_xml( + attachments_tag, attachment_refs, file.file.file, + file.hash, file.file.url) + + global_id.content = et.tostring(content_tag, ENCODING_UNICODE) + global_id.save() + + signatures_tag = et.SubElement(model, TAG_SIGNATURES) + post_signatures = global_id.signature_set.all() + if post_signatures: + signatures = post_signatures + else: + key = KeyPair.objects.get(public_key=global_id.key) + signature = Signature( + key_type=key.key_type, + key=key.public_key, + signature=key.sign(global_id.content), + global_id=global_id, + ) + signature.save() + signatures = [signature] + for signature in signatures: + signature_tag = et.SubElement(signatures_tag, TAG_SIGNATURE) + signature_tag.set(ATTR_TYPE, signature.key_type) + signature_tag.set(ATTR_VALUE, signature.signature) + signature_tag.set(ATTR_KEY, signature.key) + + return et.tostring(response, ENCODING_UNICODE) + + @staticmethod + @transaction.atomic + def parse_response_get(response_xml, hostname): + tag_root = et.fromstring(response_xml) + tag_status = tag_root.find(TAG_STATUS) + if STATUS_SUCCESS == tag_status.text: + tag_models = tag_root.find(TAG_MODELS) + for tag_model in tag_models: + tag_content = tag_model.find(TAG_CONTENT) + + content_str = et.tostring(tag_content, ENCODING_UNICODE) + signatures = SyncManager._verify_model(content_str, tag_model) + + tag_id = tag_content.find(TAG_ID) + global_id, exists = GlobalId.from_xml_element(tag_id) + + if exists: + print('Post with same ID already exists') + else: + global_id.content = content_str + global_id.save() + for signature in signatures: + signature.global_id = global_id + signature.save() + + title = tag_content.find(TAG_TITLE).text or '' + text = tag_content.find(TAG_TEXT).text or '' + pub_time = tag_content.find(TAG_PUB_TIME).text + + thread = tag_content.find(TAG_THREAD) + tags = [] + if thread: + thread_id = thread.find(TAG_ID) + op_global_id, exists = GlobalId.from_xml_element(thread_id) + if exists: + opening_post = Post.objects.get(global_id=op_global_id) + else: + raise SyncException(EXCEPTION_OP) + else: + opening_post = None + tag_tags = tag_content.find(TAG_TAGS) + for tag_tag in tag_tags: + tag, created = Tag.objects.get_or_create( + name=tag_tag.text) + tags.append(tag) + + # TODO Check that the replied posts are already present + # before adding new ones + + files = [] + tag_attachments = tag_content.find(TAG_ATTACHMENTS) or list() + tag_refs = tag_model.find(TAG_ATTACHMENT_REFS) + for attachment in tag_attachments: + tag_ref = tag_refs.find("{}[@ref='{}']".format( + TAG_ATTACHMENT_REF, attachment.text)) + url = tag_ref.get(ATTR_URL) + attached_file = download(hostname + url) + if attached_file is None: + raise SyncException(EXCEPTION_DOWNLOAD) + + hash = get_file_hash(attached_file) + if hash != attachment.text: + raise SyncException(EXCEPTION_HASH) + + files.append(attached_file) + + Post.objects.import_post( + title=title, text=text, pub_time=pub_time, + opening_post=opening_post, tags=tags, + global_id=global_id, files=files) + else: + raise SyncException(EXCEPTION_NODE.format(tag_status.text)) + + @staticmethod + def generate_response_pull(): + response = et.Element(TAG_RESPONSE) + + status = et.SubElement(response, TAG_STATUS) + status.text = STATUS_SUCCESS + + models = et.SubElement(response, TAG_MODELS) + + for post in Post.objects.all(): + tag_id = et.SubElement(models, TAG_ID) + post.global_id.to_xml_element(tag_id) + + return et.tostring(response, ENCODING_UNICODE) + + @staticmethod + def _verify_model(content_str, tag_model): + """ + Verifies all signatures for a single model. + """ + + signatures = [] + + tag_signatures = tag_model.find(TAG_SIGNATURES) + for tag_signature in tag_signatures: + signature_type = tag_signature.get(ATTR_TYPE) + signature_value = tag_signature.get(ATTR_VALUE) + signature_key = tag_signature.get(ATTR_KEY) + + signature = Signature(key_type=signature_type, + key=signature_key, + signature=signature_value) + + if not KeyPair.objects.verify(signature, content_str): + raise SyncException(EXCEPTION_SIGNATURE.format(content_str)) + + signatures.append(signature) + + return signatures + + @staticmethod + def _attachment_to_xml(tag_attachments, tag_refs, file, hash, url): + if tag_attachments is not None: + mimetype = get_file_mimetype(file) + attachment = et.SubElement(tag_attachments, TAG_ATTACHMENT) + attachment.set(ATTR_MIMETYPE, mimetype) + attachment.text = hash + + attachment_ref = et.SubElement(tag_refs, TAG_ATTACHMENT_REF) + attachment_ref.set(ATTR_REF, hash) + attachment_ref.set(ATTR_URL, url) diff --git a/boards/models/signature.py b/boards/models/signature.py new file mode 100644 --- /dev/null +++ b/boards/models/signature.py @@ -0,0 +1,144 @@ +import xml.etree.ElementTree as et +from django.db import models + + +TAG_MODEL = 'model' +TAG_REQUEST = 'request' +TAG_ID = 'id' + +TYPE_GET = 'get' +TYPE_PULL = 'pull' + +ATTR_VERSION = 'version' +ATTR_TYPE = 'type' +ATTR_NAME = 'name' + +ATTR_KEY = 'key' +ATTR_KEY_TYPE = 'type' +ATTR_LOCAL_ID = 'local-id' + + +class GlobalIdManager(models.Manager): + def generate_request_get(self, global_id_list: list): + """ + Form a get request from a list of ModelId objects. + """ + + request = et.Element(TAG_REQUEST) + request.set(ATTR_TYPE, TYPE_GET) + request.set(ATTR_VERSION, '1.0') + + model = et.SubElement(request, TAG_MODEL) + model.set(ATTR_VERSION, '1.0') + model.set(ATTR_NAME, 'post') + + for global_id in global_id_list: + tag_id = et.SubElement(model, TAG_ID) + global_id.to_xml_element(tag_id) + + return et.tostring(request, 'unicode') + + def generate_request_pull(self): + """ + Form a pull request from a list of ModelId objects. + """ + + request = et.Element(TAG_REQUEST) + request.set(ATTR_TYPE, TYPE_PULL) + request.set(ATTR_VERSION, '1.0') + + model = et.SubElement(request, TAG_MODEL) + model.set(ATTR_VERSION, '1.0') + model.set(ATTR_NAME, 'post') + + return et.tostring(request, 'unicode') + + def global_id_exists(self, global_id): + """ + Checks if the same global id already exists in the system. + """ + + return self.filter(key=global_id.key, + key_type=global_id.key_type, + local_id=global_id.local_id).exists() + + +class GlobalId(models.Model): + """ + Global model ID and cache. + Key, key type and local ID make a single global identificator of the model. + Content is an XML cache of the model that can be passed along between nodes + without manual serialization each time. + """ + class Meta: + app_label = 'boards' + + objects = GlobalIdManager() + + def __init__(self, *args, **kwargs): + models.Model.__init__(self, *args, **kwargs) + + if 'key' in kwargs and 'key_type' in kwargs and 'local_id' in kwargs: + self.key = kwargs['key'] + self.key_type = kwargs['key_type'] + self.local_id = kwargs['local_id'] + + key = models.TextField() + key_type = models.TextField() + local_id = models.IntegerField() + content = models.TextField(blank=True, null=True) + + def __str__(self): + return '%s::%s::%d' % (self.key_type, self.key, self.local_id) + + def to_xml_element(self, element: et.Element): + """ + Exports global id to an XML element. + """ + + element.set(ATTR_KEY, self.key) + element.set(ATTR_KEY_TYPE, self.key_type) + element.set(ATTR_LOCAL_ID, str(self.local_id)) + + @staticmethod + def from_xml_element(element: et.Element): + """ + Parses XML id tag and gets global id from it. + + Arguments: + element -- the XML 'id' element + + Returns: + global_id -- id itself + exists -- True if the global id was taken from database, False if it + did not exist and was created. + """ + + try: + return GlobalId.objects.get(key=element.get(ATTR_KEY), + key_type=element.get(ATTR_KEY_TYPE), + local_id=int(element.get( + ATTR_LOCAL_ID))), True + except GlobalId.DoesNotExist: + return GlobalId(key=element.get(ATTR_KEY), + key_type=element.get(ATTR_KEY_TYPE), + local_id=int(element.get(ATTR_LOCAL_ID))), False + + +class Signature(models.Model): + class Meta: + app_label = 'boards' + + def __init__(self, *args, **kwargs): + models.Model.__init__(self, *args, **kwargs) + + if 'key' in kwargs and 'key_type' in kwargs and 'signature' in kwargs: + self.key_type = kwargs['key_type'] + self.key = kwargs['key'] + self.signature = kwargs['signature'] + + key_type = models.TextField() + key = models.TextField() + signature = models.TextField() + + global_id = models.ForeignKey('GlobalId') diff --git a/boards/models/sync_key.py b/boards/models/sync_key.py new file mode 100644 --- /dev/null +++ b/boards/models/sync_key.py @@ -0,0 +1,61 @@ +import base64 +from ecdsa import SigningKey, VerifyingKey, BadSignatureError +from django.db import models + +TYPE_ECDSA = 'ecdsa' + +APP_LABEL_BOARDS = 'boards' + + +class KeyPairManager(models.Manager): + def generate_key(self, key_type=TYPE_ECDSA, primary=False): + if primary and self.filter(primary=True).exists(): + raise Exception('There can be only one primary key') + + if key_type == TYPE_ECDSA: + private = SigningKey.generate() + public = private.get_verifying_key() + + private_key_str = base64.b64encode(private.to_string()).decode() + public_key_str = base64.b64encode(public.to_string()).decode() + + return self.create(public_key=public_key_str, + private_key=private_key_str, + key_type=TYPE_ECDSA, primary=primary) + else: + raise Exception('Key type not supported') + + def verify(self, signature, string): + if signature.key_type == TYPE_ECDSA: + public = VerifyingKey.from_string(base64.b64decode(signature.key)) + signature_byte = base64.b64decode(signature.signature) + try: + return public.verify(signature_byte, string.encode()) + except BadSignatureError: + return False + else: + raise Exception('Key type not supported') + + def has_primary(self): + return self.filter(primary=True).exists() + + +class KeyPair(models.Model): + class Meta: + app_label = APP_LABEL_BOARDS + + objects = KeyPairManager() + + public_key = models.TextField() + private_key = models.TextField() + key_type = models.TextField() + primary = models.BooleanField(default=False) + + def __str__(self): + return '%s::%s' % (self.key_type, self.public_key) + + def sign(self, string): + private = SigningKey.from_string(base64.b64decode( + self.private_key.encode())) + signature_byte = private.sign_deterministic(string.encode()) + return base64.b64encode(signature_byte).decode() diff --git a/boards/models/user.py b/boards/models/user.py --- a/boards/models/user.py +++ b/boards/models/user.py @@ -1,6 +1,5 @@ from django.db import models - -import boards.models.post +import boards __author__ = 'neko259' diff --git a/boards/signals.py b/boards/signals.py new file mode 100644 --- /dev/null +++ b/boards/signals.py @@ -0,0 +1,89 @@ +import re +from boards.mdx_neboard import get_parser + +from boards.models import Post, GlobalId +from boards.models.post import REGEX_NOTIFICATION +from boards.models.post import REGEX_REPLY, REGEX_GLOBAL_REPLY +from boards.models.user import Notification +from django.db.models.signals import post_save, pre_save, pre_delete, \ + post_delete +from django.dispatch import receiver +from django.utils import timezone + + +@receiver(post_save, sender=Post) +def connect_replies(instance, **kwargs): + for reply_number in re.finditer(REGEX_REPLY, instance.get_raw_text()): + post_id = reply_number.group(1) + + try: + referenced_post = Post.objects.get(id=post_id) + + if not referenced_post.referenced_posts.filter( + id=instance.id).exists(): + referenced_post.referenced_posts.add(instance) + referenced_post.last_edit_time = instance.pub_time + referenced_post.build_refmap() + referenced_post.save(update_fields=['refmap', 'last_edit_time']) + except Post.DoesNotExist: + pass + + +@receiver(post_save, sender=Post) +def connect_global_replies(instance, **kwargs): + for reply_number in re.finditer(REGEX_GLOBAL_REPLY, instance.get_raw_text()): + key_type = reply_number.group(1) + key = reply_number.group(2) + local_id = reply_number.group(3) + + try: + global_id = GlobalId.objects.get(key_type=key_type, key=key, + local_id=local_id) + referenced_post = Post.objects.get(global_id=global_id) + referenced_post.referenced_posts.add(instance) + referenced_post.last_edit_time = instance.pub_time + referenced_post.build_refmap() + referenced_post.save(update_fields=['refmap', 'last_edit_time']) + except (GlobalId.DoesNotExist, Post.DoesNotExist): + pass + + +@receiver(post_save, sender=Post) +def connect_notifications(instance, **kwargs): + for reply_number in re.finditer(REGEX_NOTIFICATION, instance.get_raw_text()): + user_name = reply_number.group(1).lower() + Notification.objects.get_or_create(name=user_name, post=instance) + + +@receiver(pre_save, sender=Post) +def preparse_text(instance, **kwargs): + instance._text_rendered = get_parser().parse(instance.get_raw_text()) + + +@receiver(pre_delete, sender=Post) +def delete_images(instance, **kwargs): + for image in instance.images.all(): + image_refs_count = image.post_images.count() + if image_refs_count == 1: + image.delete() + + +@receiver(pre_delete, sender=Post) +def delete_attachments(instance, **kwargs): + for attachment in instance.attachments.all(): + attachment_refs_count = attachment.attachment_posts.count() + if attachment_refs_count == 1: + attachment.delete() + + +@receiver(post_delete, sender=Post) +def update_thread_on_delete(instance, **kwargs): + thread = instance.get_thread() + thread.last_edit_time = timezone.now() + thread.save() + + +@receiver(post_delete, sender=Post) +def delete_global_id(instance, **kwargs): + if instance.global_id and instance.global_id.id: + instance.global_id.delete() diff --git a/boards/static/css/md/base_page.css b/boards/static/css/md/base_page.css --- a/boards/static/css/md/base_page.css +++ b/boards/static/css/md/base_page.css @@ -492,6 +492,11 @@ ul { font-size: 1.2em; } +.global-id { + font-weight: bolder; + opacity: .5; +} + /* Reflink preview */ .post_preview { border-left: 1px solid #777; diff --git a/boards/templates/boards/post.html b/boards/templates/boards/post.html --- a/boards/templates/boards/post.html +++ b/boards/templates/boards/post.html @@ -55,7 +55,9 @@ | {% trans 'Edit thread' %} {% endif %} {% endif %} - + {% if post.global_id_id %} + | RAW + {% endif %} {% endif %} diff --git a/boards/tests/test_forms.py b/boards/tests/test_forms.py --- a/boards/tests/test_forms.py +++ b/boards/tests/test_forms.py @@ -42,7 +42,7 @@ class FormTest(TestCase): # Change posting delay so we don't have to wait for 30 seconds or more old_posting_delay = neboard.settings.POSTING_DELAY # Wait fot the posting delay or we won't be able to post - settings.POSTING_DELAY = 1 + neboard.settings.POSTING_DELAY = 1 time.sleep(neboard.settings.POSTING_DELAY + 1) response = client.post(THREAD_PAGE_ONE, {'text': TEST_TEXT, 'tags': valid_tags}) diff --git a/boards/tests/test_keys.py b/boards/tests/test_keys.py new file mode 100644 --- /dev/null +++ b/boards/tests/test_keys.py @@ -0,0 +1,89 @@ +from base64 import b64encode +import logging + +from django.test import TestCase +from boards.models import KeyPair, GlobalId, Post, Signature +from boards.models.post.sync import SyncManager + +logger = logging.getLogger(__name__) + + +class KeyTest(TestCase): + def test_create_key(self): + key = KeyPair.objects.generate_key('ecdsa') + + self.assertIsNotNone(key, 'The key was not created.') + + def test_validation(self): + key = KeyPair.objects.generate_key(key_type='ecdsa') + message = 'msg' + signature_value = key.sign(message) + + signature = Signature(key_type='ecdsa', key=key.public_key, + signature=signature_value) + valid = KeyPair.objects.verify(signature, message) + + self.assertTrue(valid, 'Message verification failed.') + + def test_primary_constraint(self): + KeyPair.objects.generate_key(key_type='ecdsa', primary=True) + + with self.assertRaises(Exception): + KeyPair.objects.generate_key(key_type='ecdsa', primary=True) + + def test_model_id_save(self): + model_id = GlobalId(key_type='test', key='test key', local_id='1') + model_id.save() + + def test_request_get(self): + post = self._create_post_with_key() + + request = GlobalId.objects.generate_request_get([post.global_id]) + logger.debug(request) + + key = KeyPair.objects.get(primary=True) + self.assertTrue('' + '' + '' + '' + '' % ( + key.public_key, + key.key_type, + ) in request, + 'Wrong XML generated for the GET request.') + + def test_response_get(self): + post = self._create_post_with_key() + reply_post = Post.objects.create_post(title='test_title', + text='[post]%d[/post]' % post.id, + thread=post.get_thread()) + + response = SyncManager.generate_response_get([reply_post]) + logger.debug(response) + + key = KeyPair.objects.get(primary=True) + self.assertTrue('success' + '' + '' + '' + '' + 'test_title' + '[post]%s[/post]' + '' + '%s' + '' % ( + key.public_key, + reply_post.id, + key.key_type, + str(post.global_id), + key.public_key, + post.id, + key.key_type, + str(reply_post.get_pub_time_str()), + ) in response, + 'Wrong XML generated for the GET response.') + + def _create_post_with_key(self): + KeyPair.objects.generate_key(primary=True) + + return Post.objects.create_post(title='test_title', text='test_text') diff --git a/boards/tests/test_post.py b/boards/tests/test_post.py --- a/boards/tests/test_post.py +++ b/boards/tests/test_post.py @@ -2,7 +2,7 @@ from django.core.paginator import Pagina from django.test import TestCase from boards import settings -from boards.models import Tag, Post, Thread +from boards.models import Tag, Post, Thread, KeyPair from boards.models.thread import STATUS_ARCHIVE @@ -115,6 +115,27 @@ class PostTests(TestCase): self.assertEqual(all_threads[settings.get_int('View', 'ThreadsPerPage')].id, first_post.id) + def test_reflinks(self): + """ + Tests that reflinks are parsed within post and connecting replies + to the replied posts. + + Local reflink example: [post]123[/post] + Global reflink example: [post]key_type::key::123[/post] + """ + + key = KeyPair.objects.generate_key(primary=True) + + tag = Tag.objects.create(name='test_tag') + + post = Post.objects.create_post(title='', text='', tags=[tag]) + post_local_reflink = Post.objects.create_post(title='', + text='[post]%d[/post]' % post.id, thread=post.get_thread()) + + self.assertTrue(post_local_reflink in post.referenced_posts.all(), + 'Local reflink not connecting posts.') + + def test_thread_replies(self): """ Tests that the replies can be queried from a thread in all possible diff --git a/boards/tests/test_sync.py b/boards/tests/test_sync.py new file mode 100644 --- /dev/null +++ b/boards/tests/test_sync.py @@ -0,0 +1,102 @@ +from boards.models import KeyPair, Post, Tag +from boards.models.post.sync import SyncManager +from boards.tests.mocks import MockRequest +from boards.views.sync import response_get + +__author__ = 'neko259' + + +from django.test import TestCase + + +class SyncTest(TestCase): + def test_get(self): + """ + Forms a GET request of a post and checks the response. + """ + + key = KeyPair.objects.generate_key(primary=True) + tag = Tag.objects.create(name='tag1') + post = Post.objects.create_post(title='test_title', + text='test_text\rline two', + tags=[tag]) + + request = MockRequest() + request.body = ( + '' + '' + '' + '' + '' % (post.global_id.key, + post.id, + post.global_id.key_type) + ) + + response = response_get(request).content.decode() + self.assertTrue( + 'success' + '' + '' + '' + '' + '%s' + '%s' + '%s' + '%s' + '' % ( + post.global_id.key, + post.global_id.local_id, + post.global_id.key_type, + post.title, + post.get_sync_text(), + post.get_thread().get_tags().first().name, + post.get_pub_time_str(), + ) in response, + 'Wrong response generated for the GET request.') + + post.delete() + key.delete() + + KeyPair.objects.generate_key(primary=True) + + SyncManager.parse_response_get(response, None) + self.assertEqual(1, Post.objects.count(), + 'Post was not created from XML response.') + + parsed_post = Post.objects.first() + self.assertEqual('tag1', + parsed_post.get_thread().get_tags().first().name, + 'Invalid tag was parsed.') + + SyncManager.parse_response_get(response, None) + self.assertEqual(1, Post.objects.count(), + 'The same post was imported twice.') + + self.assertEqual(1, parsed_post.global_id.signature_set.count(), + 'Signature was not saved.') + + post = parsed_post + + # Trying to sync the same once more + response = response_get(request).content.decode() + + self.assertTrue( + 'success' + '' + '' + '' + '' + '%s' + '%s' + '%s' + '%s' + '' % ( + post.global_id.key, + post.global_id.local_id, + post.global_id.key_type, + post.title, + post.get_sync_text(), + post.get_thread().get_tags().first().name, + post.get_pub_time_str(), + ) in response, + 'Wrong response generated for the GET request.') diff --git a/boards/tests/test_views.py b/boards/tests/test_views.py --- a/boards/tests/test_views.py +++ b/boards/tests/test_views.py @@ -9,8 +9,9 @@ logger = logging.getLogger(__name__) HTTP_CODE_OK = 200 EXCLUDED_VIEWS = { - 'banned', - 'get_thread_diff', + 'banned', + 'get_thread_diff', + 'api_sync_pull', } diff --git a/boards/urls.py b/boards/urls.py --- a/boards/urls.py +++ b/boards/urls.py @@ -2,14 +2,17 @@ from django.conf.urls import url #from django.views.i18n import javascript_catalog import neboard + from boards import views from boards.rss import AllThreadsFeed, TagThreadsFeed, ThreadPostsFeed from boards.views import api, tag_threads, all_threads, \ settings, all_tags, feed from boards.views.authors import AuthorsView from boards.views.notifications import NotificationView +from boards.views.search import BoardSearchView from boards.views.static import StaticPageView from boards.views.preview import PostPreviewView +from boards.views.sync import get_post_sync_data, response_get, response_pull from boards.views.random import RandomImageView from boards.views.tag_gallery import TagGalleryView from boards.views.translation import cached_javascript_catalog @@ -74,12 +77,19 @@ urlpatterns = [ url(r'^api/preview/$', api.api_get_preview, name='preview'), url(r'^api/new_posts/$', api.api_get_new_posts, name='new_posts'), + # Sync protocol API + url(r'^api/sync/pull/$', response_pull, name='api_sync_pull'), + url(r'^api/sync/get/$', response_get, name='api_sync_pull'), + # TODO 'get' request + # Notifications url(r'^notifications/(?P\w+)/$', NotificationView.as_view(), name='notifications'), url(r'^notifications/$', NotificationView.as_view(), name='notifications'), # Post preview url(r'^preview/$', PostPreviewView.as_view(), name='preview'), + url(r'^post_xml/(?P\d+)$', get_post_sync_data, + name='post_sync_data'), ] # Search diff --git a/boards/views/api.py b/boards/views/api.py --- a/boards/views/api.py +++ b/boards/views/api.py @@ -1,26 +1,20 @@ -from collections import OrderedDict import json import logging +from django.core import serializers from django.db import transaction -from django.db.models import Count from django.http import HttpResponse from django.shortcuts import get_object_or_404 -from django.core import serializers -from django.template.context_processors import csrf from django.views.decorators.csrf import csrf_protect -from boards.abstracts.settingsmanager import get_settings_manager,\ - FAV_THREAD_NO_UPDATES - +from boards.abstracts.settingsmanager import get_settings_manager from boards.forms import PostForm, PlainErrorList +from boards.mdx_neboard import Parser from boards.models import Post, Thread, Tag from boards.models.thread import STATUS_ARCHIVE +from boards.models.user import Notification from boards.utils import datetime_to_epoch from boards.views.thread import ThreadView -from boards.models.user import Notification -from boards.mdx_neboard import Parser - __author__ = 'neko259' diff --git a/boards/views/sync.py b/boards/views/sync.py new file mode 100644 --- /dev/null +++ b/boards/views/sync.py @@ -0,0 +1,62 @@ +import xml.etree.ElementTree as et +import xml.dom.minidom + +from django.http import HttpResponse, Http404 +from boards.models import GlobalId, Post +from boards.models.post.sync import SyncManager + + +def response_pull(request): + request_xml = request.body + + if request_xml is None: + return HttpResponse(content='Use the API') + + response_xml = SyncManager.generate_response_pull() + + return HttpResponse(content=response_xml) + + +def response_get(request): + """ + Processes a GET request with post ID list and returns the posts XML list. + Request should contain an 'xml' post attribute with the actual request XML. + """ + + request_xml = request.body + + if request_xml is None: + return HttpResponse(content='Use the API') + + posts = [] + + root_tag = et.fromstring(request_xml) + model_tag = root_tag[0] + for id_tag in model_tag: + global_id, exists = GlobalId.from_xml_element(id_tag) + if exists: + posts.append(Post.objects.get(global_id=global_id)) + + response_xml = SyncManager.generate_response_get(posts) + + return HttpResponse(content=response_xml) + + +def get_post_sync_data(request, post_id): + try: + post = Post.objects.get(id=post_id) + except Post.DoesNotExist: + raise Http404() + + xml_str = SyncManager.generate_response_get([post]) + + xml_repr = xml.dom.minidom.parseString(xml_str) + xml_repr = xml_repr.toprettyxml() + + content = '=Global ID=\n%s\n\n=XML=\n%s' \ + % (post.global_id, xml_repr) + + return HttpResponse( + content_type='text/plain', + content=content, + ) \ No newline at end of file diff --git a/development/run.sh b/development/run.sh new file mode 100755 --- /dev/null +++ b/development/run.sh @@ -0,0 +1,3 @@ +#! /usr/bin/env sh + +python3 manage.py runserver [::]:8000 diff --git a/development/test.sh b/development/test.sh new file mode 100755 --- /dev/null +++ b/development/test.sh @@ -0,0 +1,3 @@ +#! /usr/bin/env sh + +python3 manage.py test diff --git a/docs/dip-1.markdown b/docs/dip-1.markdown new file mode 100644 --- /dev/null +++ b/docs/dip-1.markdown @@ -0,0 +1,203 @@ +# 0 Title # + +DIP-1 Common protocol description + +# 1 Intro # + +This document describes the Data Interchange Protocol (DIP), designed to +exchange filtered data that can be stored as a graph structure between +network nodes. + +# 2 Purpose # + +This protocol will be used to share the models (originally imageboard posts) +across multiple servers. The main differnce of this protocol is that the node +can specify what models it wants to get and from whom. The nodes can get +models from a specific server, or from all except some specific servers. Also +the models can be filtered by timestamps or tags. + +# 3 Protocol description # + +The node requests other node's changes list since some time (since epoch if +this is the start). The other node sends a list of post ids or posts in the +XML format. + +Protocol version is the version of the sync api. Model version is the version +of data models. If at least one of them is different, the sync cannot be +performed. + +The node signs the data with its keys. The receiving node saves the key at the +first sync and checks it every time. If the key has changed, the info won't be +saved from the node (or the node id must be changed). A model can be signed +with several keys but at least one of them must be the same as in the global +ID to verify the sender. + +Each node can have several keys. Nodes can have shared keys to serve as a pool +(several nodes with the same key). + +Each post has an ID in the unique format: key-type::key::local-id + +All requests pass a request type, protocol and model versions, and a list of +optional arguments used for filtering. + +Each request has its own version. Version consists of 2 numbers: first is +incompatible version (1.3 and 2.0 are not compatible and must not be in sync) +and the second one is minor and compatible (for example, new optional field +is added which will be igroned by those who don't support it yet). + +Post edits and reflinks are not saved to the sync model. The replied post ID +can be got from the post text, and reflinks can be computed when loading +posts. The edit time is not saved because a foreign post can be 'edited' (new +replies are added) but the signature must not change (so we can't update the +content). The inner posts can be edited, and the signature will change then +but the local-id won't, so the other node can detect that and replace the post +instead of adding a new one. + +## 3.1 Requests ## + +There is no constraint on how the server should calculate the request. The +server can return any information by any filter and the requesting node is +responsible for validating it. + +The server is required to return the status of request. See 3.2 for details. + +### 3.1.1 pull ### + +"pull" request gets the desired model id list by the given filter (e.g. thread, tags, +author) + +Sample request is as follows: + + + + + 0 + 0 + + tag1 + + + + abcehy3h9t + ehoehyoe + + + + + + +Under the tag there are filters. Filters for the "post" model can +be found in DIP-2. + +Sample response: + + + + success + + + + + + + + +### 3.1.2 get ### + +"get" gets models by id list. + +Sample request: + + + + + + + + + +Id consists of a key, key type and local id. This key is used for signing and +validating of data in the model content. + +Sample response: + + + + + success + + + + + + 13 + Thirteen + + 12 + + + TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5I + + + + + + + + + + + + + + + 13 + Thirteen + 12 + 13 + + tag1 + + + + + + + + + +### 3.1.3 put ### + +"put" gives a model to the given node (you have no guarantee the node takes +it, consider you are just advising the node to take your post). This request +type is useful in pool where all the nodes try to duplicate all of their data +across the pool. + +## 3.2 Responses ## + +### 3.2.1 "not supported" ### + +If the request if completely not supported, a "not supported" status will be +returned. + +### 3.2.2 "success" ### + +"success" status means the request was processed and the result is returned. + +### 3.2.3 "error" ### + +If the server knows for sure that the operation cannot be processed, it sends +the "error" status. Additional tags describing the error may be +and . diff --git a/docs/dip-2.markdown b/docs/dip-2.markdown new file mode 100644 --- /dev/null +++ b/docs/dip-2.markdown @@ -0,0 +1,29 @@ +# 0 Title # + +"post" model reference + +# 1 Description # + +"post" is a model that defines an imageboard message, or post. + +# 2 Fields # + +# 2.1 Mandatory fields # + +* title -- text field. +* text -- text field. +* pub-time -- timestamp (TBD: Define format). + +# 2.2 Optional fields # + +* attachments -- defines attachments such as images. +* attachment -- contains and attachment or link to attachment to be downloaded +manually. Required attributes are mimetype and name. + +This field is used for the opening post (thread): + +* tags -- text tag name list. + +This field is used for non-opening post: + +* thread -- ID of a post model that this post is related to. diff --git a/neboard/settings.py b/neboard/settings.py --- a/neboard/settings.py +++ b/neboard/settings.py @@ -2,7 +2,6 @@ import os DEBUG = True -TEMPLATE_DEBUG = DEBUG ADMINS = ( # ('Your Name', 'your_email@example.com'), @@ -67,16 +66,7 @@ STATIC_ROOT = '' # Example: "http://media.lawrence.com/static/" STATIC_URL = '/static/' -# Additional locations of static files -# It is really a hack, put real paths, not related -STATICFILES_DIRS = ( - os.path.dirname(__file__) + '/boards/static', - -# '/d/work/python/django/neboard/neboard/boards/static', - # Put strings here, like "/home/html/static" or "C:/www/django/static". - # Always use forward slashes, even on Windows. - # Don't forget to use absolute paths, not relative paths. -) +STATICFILES_DIRS = [] # List of finder classes that know how to find static files in # various locations. @@ -227,6 +217,7 @@ MIDDLEWARE_CLASSES += [ 'debug_toolbar.middleware.DebugToolbarMiddleware', ] + def custom_show_toolbar(request): return request.user.has_perm('admin.debug') diff --git a/readme.markdown b/readme.markdown --- a/readme.markdown +++ b/readme.markdown @@ -9,21 +9,54 @@ Site: http://neboard.me/ # INSTALLATION # -1. Install all dependencies over pip or system-wide -2. Setup a database in `neboard/settings.py` -3. Run `./manage.py migrate` to apply all south migrations -4. Apply config changes to `boards/config/config.ini`. You can see the default settings in `boards/config/default_config.ini` +1. Download application and move inside it: + +`hg clone https://bitbucket.org/neko259/neboard` + +`cd neboard` + +If you wish to use *decentral* version, change branch to *decentral*: + +`hg up decentral` + +2. Install all application dependencies: + +Some minimal system-wide depenencies: + +* python3 +* pip/pip3 +* jpeg + +Python dependencies: + +`pip3 install -r requirements.txt` + +You can use virtualenv to speed up the process or avoid conflicts. + +3. Setup a database in `neboard/settings.py`. You can also change other settings like search engine. + +Depending on configured database and search engine, you need to install corresponding dependencies manually. + +Default database is *sqlite*, default search engine is *whoosh*. + +4. Setup SECRET_KEY to a secret value in `neboard/settings.py +5. Run `./manage.py migrate` to apply all migrations +6. Apply config changes to `boards/config/config.ini`. You can see the default settings in `boards/config/default_config.ini` +7. If you want to use decetral engine, run `./manage.py generate_keypair` to generate keys # RUNNING # -You can run the server using django default embedded webserver by running +You can run the server using django default embedded webserver by running: ./manage.py runserver
: -See django-admin command help for details +See django-admin command help for details. Also consider using wsgi or fcgi interfaces on production servers. +When running for the first time, you need to setup at least one section tag. +Go to the admin page and manually create one tag with "required" property set. + # UPGRADE # 1. Backup your project data. @@ -34,4 +67,4 @@ You can also just clone the mercurial pr # CONCLUSION # -Enjoy our software and thank you! +Enjoy our software and thank you! \ No newline at end of file diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ python-magic +httplib2 +simplejson pytube requests adjacent @@ -8,3 +10,4 @@ django>=1.8 bbcode django-debug-toolbar pytz +ecdsa \ No newline at end of file diff --git a/todo.txt b/todo.txt --- a/todo.txt +++ b/todo.txt @@ -15,6 +15,7 @@ * Subscribing to tag via AJAX * Add buttons to insert a named link or a named quote to the markup panel * Add support for "attention posts" that are shown in the header" +* Use absolute post reflinks in the raw text * Use default boards/settings when it is not defined. Maybe default_settings.py module? = Bugs =