Merge remote changes

boards/migrations/0045_post_version.py

@@ -0,0 +1,20 b''
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.5 on 2016-05-11 09:23
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('boards', '0044_globalid_content'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='post',
+            name='version',
+            field=models.IntegerField(default=1),
+        ),
+    ]

boards/admin.py

@@ -1,136 +1,151 b''
 from django.contrib import admin
 from django.utils.translation import ugettext_lazy as _
 from django.core.urlresolvers import reverse
+from django.db.models import F
 from boards.models import Post, Tag, Ban, Thread, Banner, PostImage, KeyPair, GlobalId
 
 
 @admin.register(Post)
 class PostAdmin(admin.ModelAdmin):
 
     list_display = ('id', 'title', 'text', 'poster_ip', 'linked_images', 'linked_global_id')
     list_filter = ('pub_time',)
     search_fields = ('id', 'title', 'text', 'poster_ip')
     exclude = ('referenced_posts', 'refmap', 'images', 'global_id')
     readonly_fields = ('poster_ip', 'threads', 'thread', 'linked_images',
-            'attachments', 'uid', 'url', 'pub_time', 'opening', 'linked_global_id')
+            'attachments', 'uid', 'url', 'pub_time', 'opening', 'linked_global_id',
+            'version')
 
     def ban_poster(self, request, queryset):
         bans = 0
         for post in queryset:
             poster_ip = post.poster_ip
             ban, created = Ban.objects.get_or_create(ip=poster_ip)
             if created:
                 bans += 1
         self.message_user(request, _('{} posters were banned').format(bans))
 
     def ban_with_hiding(self, request, queryset):
         bans = 0
         hidden = 0
         for post in queryset:
             poster_ip = post.poster_ip
             ban, created = Ban.objects.get_or_create(ip=poster_ip)
             if created:
                 bans += 1
             posts = Post.objects.filter(poster_ip=poster_ip, id__gte=post.id)
             hidden += posts.count()
             posts.update(hidden=True)
         self.message_user(request, _('{} posters were banned, {} messages were hidden').format(bans, hidden))
 
     def linked_images(self, obj: Post):
         images = obj.images.all()
         image_urls = ['<a href="{}"><img src="{}" /></a>'.format(
             reverse('admin:%s_%s_change' % (image._meta.app_label,
                                             image._meta.model_name),
                     args=[image.id]), image.image.url_200x150) for image in images]
         return ', '.join(image_urls)
     linked_images.allow_tags = True
 
     def linked_global_id(self, obj: Post):
         global_id = obj.global_id
         if global_id is not None:
             return '<a href="{}">{}</a>'.format(
                 reverse('admin:%s_%s_change' % (global_id._meta.app_label,
                                                 global_id._meta.model_name),
                         args=[global_id.id]), str(global_id))
     linked_global_id.allow_tags = True
 
+    def save_model(self, request, obj, form, change):
+        obj.increment_version()
+        obj.save()
+        obj.clear_cache()
+
     actions = ['ban_poster', 'ban_with_hiding']
 
 
 @admin.register(Tag)
 class TagAdmin(admin.ModelAdmin):
 
     def thread_count(self, obj: Tag) -> int:
         return obj.get_thread_count()
 
     def display_children(self, obj: Tag):
         return ', '.join([str(child) for child in obj.get_children().all()])
 
     def save_model(self, request, obj, form, change):
         super().save_model(request, obj, form, change)
         for thread in obj.get_threads().all():
             thread.refresh_tags()
     list_display = ('name', 'thread_count', 'display_children')
     search_fields = ('name',)
 
 
 @admin.register(Thread)
 class ThreadAdmin(admin.ModelAdmin):
 
     def title(self, obj: Thread) -> str:
         return obj.get_opening_post().get_title()
 
     def reply_count(self, obj: Thread) -> int:
         return obj.get_reply_count()
 
     def ip(self, obj: Thread):
         return obj.get_opening_post().poster_ip
 
     def display_tags(self, obj: Thread):
         return ', '.join([str(tag) for tag in obj.get_tags().all()])
 
     def op(self, obj: Thread):
         return obj.get_opening_post_id()
 
     # Save parent tags when editing tags
     def save_related(self, request, form, formsets, change):
         super().save_related(request, form, formsets, change)
         form.instance.refresh_tags()
+
+    def save_model(self, request, obj, form, change):
+        op = obj.get_opening_post()
+        op.increment_version()
+        op.save(update_fields=['version'])
+        obj.save()
+        op.clear_cache()
+
     list_display = ('id', 'op', 'title', 'reply_count', 'status', 'ip',
             'display_tags')
     list_filter = ('bump_time', 'status')
     search_fields = ('id', 'title')
     filter_horizontal = ('tags',)
 
 
 @admin.register(KeyPair)
 class KeyPairAdmin(admin.ModelAdmin):
     list_display = ('public_key', 'primary')
     list_filter = ('primary',)
     search_fields = ('public_key',)
 
 
 @admin.register(Ban)
 class BanAdmin(admin.ModelAdmin):
     list_display = ('ip', 'can_read')
     list_filter = ('can_read',)
     search_fields = ('ip',)
 
 
 @admin.register(Banner)
 class BannerAdmin(admin.ModelAdmin):
     list_display = ('title', 'text')
 
 
 @admin.register(PostImage)
 class PostImageAdmin(admin.ModelAdmin):
     search_fields = ('alias',)
 
 
 @admin.register(GlobalId)
 class GlobalIdAdmin(admin.ModelAdmin):
     def is_linked(self, obj):
         return Post.objects.filter(global_id=obj).exists()
 
     list_display = ('__str__', 'is_linked',)
     readonly_fields = ('content',)

boards/management/commands/invalidate_sync_cache.py

@@ -1,20 +1,22 b''
 from django.core.management import BaseCommand
 from django.db import transaction
 
 from boards.models import GlobalId
 
 __author__ = 'neko259'
 
 
 class Command(BaseCommand):
     help = 'Removes local global ID cache'
 
     @transaction.atomic
     def handle(self, *args, **options):
         count = 0
-        for global_id in GlobalId.objects.all():
+        for global_id in GlobalId.objects.exclude(content__isnull=True).exclude(
+                content=''):
             if global_id.is_local() and global_id.content is not None:
                 global_id.content = None
                 global_id.save()
+                global_id.signature_set.all().delete()
                 count += 1
         print('Invalidated {} caches.'.format(count))

boards/management/commands/sync_with_server.py

@@ -1,85 +1,90 b''
 import re
 import xml.etree.ElementTree as ET
 
 import httplib2
 from django.core.management import BaseCommand
 
 from boards.models import GlobalId
-from boards.models.post.sync import SyncManager
+from boards.models.post.sync import SyncManager, TAG_ID, TAG_VERSION
 
 __author__ = 'neko259'
 
 
 REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)')
 
 
 class Command(BaseCommand):
     help = 'Send a sync or get request to the server.'
 
     def add_arguments(self, parser):
         parser.add_argument('url', type=str, help='Server root url')
         parser.add_argument('--global-id', type=str, default='',
                             help='Post global ID')
         parser.add_argument('--split-query', type=int,
                             help='Split GET query into separate by the given'
                                  ' number of posts in one')
 
     def handle(self, *args, **options):
         url = options.get('url')
 
-        pull_url = url + 'api/sync/pull/'
+        list_url = url + 'api/sync/list/'
         get_url = url + 'api/sync/get/'
         file_url = url[:-1]
 
         global_id_str = options.get('global_id')
         if global_id_str:
             match = REGEX_GLOBAL_ID.match(global_id_str)
             if match:
                 key_type = match.group(1)
                 key = match.group(2)
                 local_id = match.group(3)
 
                 global_id = GlobalId(key_type=key_type, key=key,
                                      local_id=local_id)
 
                 xml = GlobalId.objects.generate_request_get([global_id])
                 # body = urllib.parse.urlencode(data)
                 h = httplib2.Http()
                 response, content = h.request(get_url, method="POST", body=xml)
 
                 SyncManager.parse_response_get(content, file_url)
             else:
                 raise Exception('Invalid global ID')
         else:
             h = httplib2.Http()
-            xml = GlobalId.objects.generate_request_pull()
-            response, content = h.request(pull_url, method="POST", body=xml)
+            xml = GlobalId.objects.generate_request_list()
+            response, content = h.request(list_url, method="POST", body=xml)
 
             print(content.decode() + '\n')
 
             root = ET.fromstring(content)
             status = root.findall('status')[0].text
             if status == 'success':
                 ids_to_sync = list()
 
                 models = root.findall('models')[0]
                 for model in models:
-                    global_id, exists = GlobalId.from_xml_element(model)
-                    if not exists:
-                        print(global_id)
+                    tag_id = model.find(TAG_ID)
+                    global_id, exists = GlobalId.from_xml_element(tag_id)
+                    tag_version = model.find(TAG_VERSION)
+                    if tag_version is not None:
+                        version = int(tag_version.text) or 1
+                    else:
+                        version = 1
+                    if not exists or global_id.post.version < version:
                         ids_to_sync.append(global_id)
-                print()
+                print('Starting sync...')
 
                 if len(ids_to_sync) > 0:
                     limit = options.get('split_query', len(ids_to_sync))
                     for offset in range(0, len(ids_to_sync), limit):
                         xml = GlobalId.objects.generate_request_get(ids_to_sync[offset:offset+limit])
                         # body = urllib.parse.urlencode(data)
                         h = httplib2.Http()
                         response, content = h.request(get_url, method="POST", body=xml)
 
                         SyncManager.parse_response_get(content, file_url)
                 else:
                     print('Nothing to get, everything synced')
             else:
                 raise Exception('Invalid response status')

boards/models/post/__init__.py

@@ -1,381 +1,396 b''
 import uuid
 
 import re
 from boards import settings
 from boards.abstracts.tripcode import Tripcode
 from boards.models import PostImage, Attachment, KeyPair, GlobalId
 from boards.models.base import Viewable
 from boards.models.post.export import get_exporter, DIFF_TYPE_JSON
 from boards.models.post.manager import PostManager
 from boards.utils import datetime_to_epoch
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.urlresolvers import reverse
 from django.db import models
-from django.db.models import TextField, QuerySet
+from django.db.models import TextField, QuerySet, F
 from django.template.defaultfilters import truncatewords, striptags
 from django.template.loader import render_to_string
 
 CSS_CLS_HIDDEN_POST = 'hidden_post'
 CSS_CLS_DEAD_POST = 'dead_post'
 CSS_CLS_ARCHIVE_POST = 'archive_post'
 CSS_CLS_POST = 'post'
 CSS_CLS_MONOCHROME = 'monochrome'
 
 TITLE_MAX_WORDS = 10
 
 APP_LABEL_BOARDS = 'boards'
 
 BAN_REASON_AUTO = 'Auto'
 
 IMAGE_THUMB_SIZE = (200, 150)
 
 TITLE_MAX_LENGTH = 200
 
 REGEX_REPLY = re.compile(r'\[post\](\d+)\[/post\]')
 REGEX_GLOBAL_REPLY = re.compile(r'\[post\](\w+)::([^:]+)::(\d+)\[/post\]')
 REGEX_URL = re.compile(r'https?\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(/\S*)?')
 REGEX_NOTIFICATION = re.compile(r'\[user\](\w+)\[/user\]')
 
 PARAMETER_TRUNCATED = 'truncated'
 PARAMETER_TAG = 'tag'
 PARAMETER_OFFSET = 'offset'
 PARAMETER_DIFF_TYPE = 'type'
 PARAMETER_CSS_CLASS = 'css_class'
 PARAMETER_THREAD = 'thread'
 PARAMETER_IS_OPENING = 'is_opening'
 PARAMETER_POST = 'post'
 PARAMETER_OP_ID = 'opening_post_id'
 PARAMETER_NEED_OPEN_LINK = 'need_open_link'
 PARAMETER_REPLY_LINK = 'reply_link'
 PARAMETER_NEED_OP_DATA = 'need_op_data'
 
 POST_VIEW_PARAMS = (
     'need_op_data',
     'reply_link',
     'need_open_link',
     'truncated',
     'mode_tree',
     'perms',
     'tree_depth',
 )
 
 
 class Post(models.Model, Viewable):
     """A post is a message."""
 
     objects = PostManager()
 
     class Meta:
         app_label = APP_LABEL_BOARDS
         ordering = ('id',)
 
     title = models.CharField(max_length=TITLE_MAX_LENGTH, null=True, blank=True)
     pub_time = models.DateTimeField()
     text = TextField(blank=True, null=True)
     _text_rendered = TextField(blank=True, null=True, editable=False)
 
     images = models.ManyToManyField(PostImage, null=True, blank=True,
                                     related_name='post_images', db_index=True)
     attachments = models.ManyToManyField(Attachment, null=True, blank=True,
                                          related_name='attachment_posts')
 
     poster_ip = models.GenericIPAddressField()
 
     # TODO This field can be removed cause UID is used for update now
     last_edit_time = models.DateTimeField()
 
     referenced_posts = models.ManyToManyField('Post', symmetrical=False,
                                               null=True,
                                               blank=True, related_name='refposts',
                                               db_index=True)
     refmap = models.TextField(null=True, blank=True)
     threads = models.ManyToManyField('Thread', db_index=True,
                                      related_name='multi_replies')
     thread = models.ForeignKey('Thread', db_index=True, related_name='pt+')
 
     url = models.TextField()
     uid = models.TextField(db_index=True)
 
     # Global ID with author key. If the message was downloaded from another
     # server, this indicates the server.
     global_id = models.OneToOneField(GlobalId, null=True, blank=True,
                                      on_delete=models.CASCADE)
 
     tripcode = models.CharField(max_length=50, blank=True, default='')
     opening = models.BooleanField(db_index=True)
     hidden = models.BooleanField(default=False)
+    version = models.IntegerField(default=1)
 
     def __str__(self):
         return 'P#{}/{}'.format(self.id, self.get_title())
 
     def get_title(self) -> str:
         return self.title
 
     def get_title_or_text(self):
         title = self.get_title()
         if not title:
             title = truncatewords(striptags(self.get_text()), TITLE_MAX_WORDS)
 
         return title
 
     def build_refmap(self) -> None:
         """
         Builds a replies map string from replies list. This is a cache to stop
         the server from recalculating the map on every post show.
         """
 
         post_urls = [refpost.get_link_view()
                      for refpost in self.referenced_posts.all()]
 
         self.refmap = ', '.join(post_urls)
 
     def is_referenced(self) -> bool:
         return self.refmap and len(self.refmap) > 0
 
     def is_opening(self) -> bool:
         """
         Checks if this is an opening post or just a reply.
         """
 
         return self.opening
 
     def get_absolute_url(self, thread=None):
         url = None
 
         if thread is None:
             thread = self.get_thread()
 
             # Url is cached only for the "main" thread. When getting url
             # for other threads, do it manually.
             if self.url:
                 url = self.url
 
         if url is None:
             opening = self.is_opening()
             opening_id = self.id if opening else thread.get_opening_post_id()
             url = reverse('thread', kwargs={'post_id': opening_id})
             if not opening:
                 url += '#' + str(self.id)
 
         return url
 
     def get_thread(self):
         return self.thread
 
     def get_thread_id(self):
         return self.thread_id
 
     def get_threads(self) -> QuerySet:
         """
         Gets post's thread.
         """
 
         return self.threads
 
     def _get_cache_key(self):
         return [datetime_to_epoch(self.last_edit_time)]
 
     def get_view(self, *args, **kwargs) -> str:
         """
         Renders post's HTML view. Some of the post params can be passed over
         kwargs for the means of caching (if we view the thread, some params
         are same for every post and don't need to be computed over and over.
         """
 
         thread = self.get_thread()
 
         css_classes = [CSS_CLS_POST]
         if thread.is_archived():
             css_classes.append(CSS_CLS_ARCHIVE_POST)
         elif not thread.can_bump():
             css_classes.append(CSS_CLS_DEAD_POST)
         if self.is_hidden():
             css_classes.append(CSS_CLS_HIDDEN_POST)
         if thread.is_monochrome():
             css_classes.append(CSS_CLS_MONOCHROME)
 
         params = dict()
         for param in POST_VIEW_PARAMS:
             if param in kwargs:
                 params[param] = kwargs[param]
 
         params.update({
             PARAMETER_POST: self,
             PARAMETER_IS_OPENING: self.is_opening(),
             PARAMETER_THREAD: thread,
             PARAMETER_CSS_CLASS: ' '.join(css_classes),
         })
 
         return render_to_string('boards/post.html', params)
 
     def get_search_view(self, *args, **kwargs):
         return self.get_view(need_op_data=True, *args, **kwargs)
 
     def get_first_image(self) -> PostImage:
         return self.images.earliest('id')
 
     def set_global_id(self, key_pair=None):
         """
         Sets global id based on the given key pair. If no key pair is given,
         default one is used.
         """
 
         if key_pair:
             key = key_pair
         else:
             try:
                 key = KeyPair.objects.get(primary=True)
             except KeyPair.DoesNotExist:
                 # Do not update the global id because there is no key defined
                 return
         global_id = GlobalId(key_type=key.key_type,
                              key=key.public_key,
                              local_id=self.id)
         global_id.save()
 
         self.global_id = global_id
 
         self.save(update_fields=['global_id'])
 
     def get_pub_time_str(self):
         return str(self.pub_time)
 
     def get_replied_ids(self):
         """
         Gets ID list of the posts that this post replies.
         """
 
         raw_text = self.get_raw_text()
 
         local_replied = REGEX_REPLY.findall(raw_text)
         global_replied = []
         for match in REGEX_GLOBAL_REPLY.findall(raw_text):
             key_type = match[0]
             key = match[1]
             local_id = match[2]
 
             try:
                 global_id = GlobalId.objects.get(key_type=key_type,
                                                  key=key, local_id=local_id)
                 for post in Post.objects.filter(global_id=global_id).only('id'):
                     global_replied.append(post.id)
             except GlobalId.DoesNotExist:
                 pass
         return local_replied + global_replied
 
     def get_post_data(self, format_type=DIFF_TYPE_JSON, request=None,
                       include_last_update=False) -> str:
         """
         Gets post HTML or JSON data that can be rendered on a page or used by
         API.
         """
 
         return get_exporter(format_type).export(self, request,
                                                 include_last_update)
 
     def notify_clients(self, recursive=True):
         """
         Sends post HTML data to the thread web socket.
         """
 
         if not settings.get_bool('External', 'WebsocketsEnabled'):
             return
 
         thread_ids = list()
         for thread in self.get_threads().all():
             thread_ids.append(thread.id)
 
             thread.notify_clients()
 
         if recursive:
             for reply_number in re.finditer(REGEX_REPLY, self.get_raw_text()):
                 post_id = reply_number.group(1)
 
                 try:
                     ref_post = Post.objects.get(id=post_id)
 
                     if ref_post.get_threads().exclude(id__in=thread_ids).exists():
                         # If post is in this thread, its thread was already notified.
                         # Otherwise, notify its thread separately.
                         ref_post.notify_clients(recursive=False)
                 except ObjectDoesNotExist:
                     pass
 
     def build_url(self):
         self.url = self.get_absolute_url()
         self.save(update_fields=['url'])
 
     def save(self, force_insert=False, force_update=False, using=None,
              update_fields=None):
         new_post = self.id is None
 
         self.uid = str(uuid.uuid4())
         if update_fields is not None and 'uid' not in update_fields:
             update_fields += ['uid']
 
         if not new_post:
             for thread in self.get_threads().all():
                 thread.last_edit_time = self.last_edit_time
 
                 thread.save(update_fields=['last_edit_time', 'status'])
 
         super().save(force_insert, force_update, using, update_fields)
 
         if self.url is None:
             self.build_url()
 
     def get_text(self) -> str:
         return self._text_rendered
 
     def get_raw_text(self) -> str:
         return self.text
 
     def get_sync_text(self) -> str:
         """
         Returns text applicable for sync. It has absolute post reflinks.
         """
 
         replacements = dict()
         for post_id in REGEX_REPLY.findall(self.get_raw_text()):
-            absolute_post_id = str(Post.objects.get(id=post_id).global_id)
-            replacements[post_id] = absolute_post_id
+            try:
+                absolute_post_id = str(Post.objects.get(id=post_id).global_id)
+                replacements[post_id] = absolute_post_id
+            except Post.DoesNotExist:
+                pass
 
         text = self.get_raw_text() or ''
         for key in replacements:
             text = text.replace('[post]{}[/post]'.format(key),
                                 '[post]{}[/post]'.format(replacements[key]))
         text = text.replace('\r\n', '\n').replace('\r', '\n')
 
         return text
 
     def connect_threads(self, opening_posts):
         for opening_post in opening_posts:
             threads = opening_post.get_threads().all()
             for thread in threads:
                 if thread.can_bump():
                     thread.update_bump_status()
 
                 thread.last_edit_time = self.last_edit_time
                 thread.save(update_fields=['last_edit_time', 'status'])
             self.threads.add(opening_post.get_thread())
 
     def get_tripcode(self):
         if self.tripcode:
             return Tripcode(self.tripcode)
 
     def get_link_view(self):
         """
         Gets view of a reflink to the post.
         """
         result = '<a href="{}">&gt;&gt;{}</a>'.format(self.get_absolute_url(),
                                                       self.id)
         if self.is_opening():
             result = '<b>{}</b>'.format(result)
 
         return result
 
     def is_hidden(self) -> bool:
         return self.hidden
 
     def set_hidden(self, hidden):
         self.hidden = hidden
+
+    def increment_version(self):
+        self.version = F('version') + 1
+
+    def clear_cache(self):
+        global_id = self.global_id
+        if global_id is not None and global_id.is_local()\
+                and global_id.content is not None:
+            global_id.content = None
+            global_id.save()
+            global_id.signature_set.all().delete()

boards/models/post/sync.py

@@ -1,284 +1,291 b''
 import xml.etree.ElementTree as et
 
 from boards.models.attachment.downloaders import download
 from boards.utils import get_file_mimetype, get_file_hash
 from django.db import transaction
 from boards.models import KeyPair, GlobalId, Signature, Post, Tag
 
 EXCEPTION_NODE = 'Sync node returned an error: {}'
 EXCEPTION_OP = 'Load the OP first'
 EXCEPTION_DOWNLOAD = 'File was not downloaded'
 EXCEPTION_HASH = 'File hash does not match attachment hash'
 EXCEPTION_SIGNATURE = 'Invalid model signature for {}'
 ENCODING_UNICODE = 'unicode'
 
 TAG_MODEL = 'model'
 TAG_REQUEST = 'request'
 TAG_RESPONSE = 'response'
 TAG_ID = 'id'
 TAG_STATUS = 'status'
 TAG_MODELS = 'models'
 TAG_TITLE = 'title'
 TAG_TEXT = 'text'
 TAG_THREAD = 'thread'
 TAG_PUB_TIME = 'pub-time'
 TAG_SIGNATURES = 'signatures'
 TAG_SIGNATURE = 'signature'
 TAG_CONTENT = 'content'
 TAG_ATTACHMENTS = 'attachments'
 TAG_ATTACHMENT = 'attachment'
 TAG_TAGS = 'tags'
 TAG_TAG = 'tag'
 TAG_ATTACHMENT_REFS = 'attachment-refs'
 TAG_ATTACHMENT_REF = 'attachment-ref'
 TAG_TRIPCODE = 'tripcode'
+TAG_VERSION = 'version'
 
 TYPE_GET = 'get'
 
 ATTR_VERSION = 'version'
 ATTR_TYPE = 'type'
 ATTR_NAME = 'name'
 ATTR_VALUE = 'value'
 ATTR_MIMETYPE = 'mimetype'
 ATTR_KEY = 'key'
 ATTR_REF = 'ref'
 ATTR_URL = 'url'
 ATTR_ID_TYPE = 'id-type'
 
 ID_TYPE_MD5 = 'md5'
 
 STATUS_SUCCESS = 'success'
 
 
 class SyncException(Exception):
     pass
 
 
 class SyncManager:
     @staticmethod
     def generate_response_get(model_list: list):
         response = et.Element(TAG_RESPONSE)
 
         status = et.SubElement(response, TAG_STATUS)
         status.text = STATUS_SUCCESS
 
         models = et.SubElement(response, TAG_MODELS)
 
         for post in model_list:
             model = et.SubElement(models, TAG_MODEL)
             model.set(ATTR_NAME, 'post')
 
             global_id = post.global_id
 
             images = post.images.all()
             attachments = post.attachments.all()
             if global_id.content:
                 model.append(et.fromstring(global_id.content))
                 if len(images) > 0 or len(attachments) > 0:
                     attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS)
                     for image in images:
                         SyncManager._attachment_to_xml(
                             None, attachment_refs, image.image.file,
                             image.hash, image.image.url)
                     for file in attachments:
                         SyncManager._attachment_to_xml(
                             None, attachment_refs, file.file.file,
                             file.hash, file.file.url)
             else:
                 content_tag = et.SubElement(model, TAG_CONTENT)
 
                 tag_id = et.SubElement(content_tag, TAG_ID)
                 global_id.to_xml_element(tag_id)
 
                 title = et.SubElement(content_tag, TAG_TITLE)
                 title.text = post.title
 
                 text = et.SubElement(content_tag, TAG_TEXT)
                 text.text = post.get_sync_text()
 
                 thread = post.get_thread()
                 if post.is_opening():
                     tag_tags = et.SubElement(content_tag, TAG_TAGS)
                     for tag in thread.get_tags():
                         tag_tag = et.SubElement(tag_tags, TAG_TAG)
                         tag_tag.text = tag.name
                 else:
                     tag_thread = et.SubElement(content_tag, TAG_THREAD)
                     thread_id = et.SubElement(tag_thread, TAG_ID)
                     thread.get_opening_post().global_id.to_xml_element(thread_id)
 
                 pub_time = et.SubElement(content_tag, TAG_PUB_TIME)
                 pub_time.text = str(post.get_pub_time_str())
 
                 if post.tripcode:
                     tripcode = et.SubElement(content_tag, TAG_TRIPCODE)
                     tripcode.text = post.tripcode
 
                 if len(images) > 0 or len(attachments) > 0:
                     attachments_tag = et.SubElement(content_tag, TAG_ATTACHMENTS)
                     attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS)
 
                     for image in images:
                         SyncManager._attachment_to_xml(
                             attachments_tag, attachment_refs, image.image.file,
                             image.hash, image.image.url)
                     for file in attachments:
                         SyncManager._attachment_to_xml(
                             attachments_tag, attachment_refs, file.file.file,
                             file.hash, file.file.url)
+                version_tag = et.SubElement(content_tag, TAG_VERSION)
+                version_tag.text = str(post.version)
 
                 global_id.content = et.tostring(content_tag, ENCODING_UNICODE)
                 global_id.save()
 
             signatures_tag = et.SubElement(model, TAG_SIGNATURES)
             post_signatures = global_id.signature_set.all()
             if post_signatures:
                 signatures = post_signatures
             else:
                 key = KeyPair.objects.get(public_key=global_id.key)
                 signature = Signature(
                     key_type=key.key_type,
                     key=key.public_key,
                     signature=key.sign(global_id.content),
                     global_id=global_id,
                 )
                 signature.save()
                 signatures = [signature]
             for signature in signatures:
                 signature_tag = et.SubElement(signatures_tag, TAG_SIGNATURE)
                 signature_tag.set(ATTR_TYPE, signature.key_type)
                 signature_tag.set(ATTR_VALUE, signature.signature)
                 signature_tag.set(ATTR_KEY, signature.key)
 
         return et.tostring(response, ENCODING_UNICODE)
 
     @staticmethod
     @transaction.atomic
     def parse_response_get(response_xml, hostname):
         tag_root = et.fromstring(response_xml)
         tag_status = tag_root.find(TAG_STATUS)
         if STATUS_SUCCESS == tag_status.text:
             tag_models = tag_root.find(TAG_MODELS)
             for tag_model in tag_models:
                 tag_content = tag_model.find(TAG_CONTENT)
 
                 content_str = et.tostring(tag_content, ENCODING_UNICODE)
                 signatures = SyncManager._verify_model(content_str, tag_model)
 
                 tag_id = tag_content.find(TAG_ID)
                 global_id, exists = GlobalId.from_xml_element(tag_id)
 
                 if exists:
                     print('Post with same ID already exists')
                 else:
                     global_id.content = content_str
                     global_id.save()
                     for signature in signatures:
                         signature.global_id = global_id
                         signature.save()
 
                     title = tag_content.find(TAG_TITLE).text or ''
                     text = tag_content.find(TAG_TEXT).text or ''
                     pub_time = tag_content.find(TAG_PUB_TIME).text
                     tripcode_tag = tag_content.find(TAG_TRIPCODE)
                     if tripcode_tag is not None:
                         tripcode = tripcode_tag.text or ''
                     else:
                         tripcode = ''
 
                     thread = tag_content.find(TAG_THREAD)
                     tags = []
                     if thread:
                         thread_id = thread.find(TAG_ID)
                         op_global_id, exists = GlobalId.from_xml_element(thread_id)
                         if exists:
                             opening_post = Post.objects.get(global_id=op_global_id)
                         else:
                             raise SyncException(EXCEPTION_OP)
                     else:
                         opening_post = None
                         tag_tags = tag_content.find(TAG_TAGS)
                         for tag_tag in tag_tags:
                             tag, created = Tag.objects.get_or_create(
                                 name=tag_tag.text)
                             tags.append(tag)
 
                     # TODO Check that the replied posts are already present
                     # before adding new ones
 
                     files = []
                     tag_attachments = tag_content.find(TAG_ATTACHMENTS) or list()
                     tag_refs = tag_model.find(TAG_ATTACHMENT_REFS)
                     for attachment in tag_attachments:
                         tag_ref = tag_refs.find("{}[@ref='{}']".format(
                             TAG_ATTACHMENT_REF, attachment.text))
                         url = tag_ref.get(ATTR_URL)
                         attached_file = download(hostname + url)
                         if attached_file is None:
                             raise SyncException(EXCEPTION_DOWNLOAD)
 
                         hash = get_file_hash(attached_file)
                         if hash != attachment.text:
                             raise SyncException(EXCEPTION_HASH)
 
                         files.append(attached_file)
 
                     Post.objects.import_post(
                         title=title, text=text, pub_time=pub_time,
                         opening_post=opening_post, tags=tags,
                         global_id=global_id, files=files, tripcode=tripcode)
+                print('Parsed post {}'.format(global_id))
         else:
             raise SyncException(EXCEPTION_NODE.format(tag_status.text))
 
     @staticmethod
-    def generate_response_pull():
+    def generate_response_list():
         response = et.Element(TAG_RESPONSE)
 
         status = et.SubElement(response, TAG_STATUS)
         status.text = STATUS_SUCCESS
 
         models = et.SubElement(response, TAG_MODELS)
 
         for post in Post.objects.prefetch_related('global_id').all():
-            tag_id = et.SubElement(models, TAG_ID)
+            tag_model = et.SubElement(models, TAG_MODEL)
+            tag_id = et.SubElement(tag_model, TAG_ID)
             post.global_id.to_xml_element(tag_id)
+            tag_version = et.SubElement(tag_model, TAG_VERSION)
+            tag_version.text = str(post.version)
 
         return et.tostring(response, ENCODING_UNICODE)
 
     @staticmethod
     def _verify_model(content_str, tag_model):
         """
         Verifies all signatures for a single model.
         """
 
         signatures = []
 
         tag_signatures = tag_model.find(TAG_SIGNATURES)
         for tag_signature in tag_signatures:
             signature_type = tag_signature.get(ATTR_TYPE)
             signature_value = tag_signature.get(ATTR_VALUE)
             signature_key = tag_signature.get(ATTR_KEY)
 
             signature = Signature(key_type=signature_type,
                                   key=signature_key,
                                   signature=signature_value)
 
             if not KeyPair.objects.verify(signature, content_str):
                 raise SyncException(EXCEPTION_SIGNATURE.format(content_str))
 
             signatures.append(signature)
 
         return signatures
 
     @staticmethod
     def _attachment_to_xml(tag_attachments, tag_refs, file, hash, url):
         if tag_attachments is not None:
             mimetype = get_file_mimetype(file)
             attachment = et.SubElement(tag_attachments, TAG_ATTACHMENT)
             attachment.set(ATTR_MIMETYPE, mimetype)
             attachment.set(ATTR_ID_TYPE, ID_TYPE_MD5)
             attachment.text = hash
 
         attachment_ref = et.SubElement(tag_refs, TAG_ATTACHMENT_REF)
         attachment_ref.set(ATTR_REF, hash)
         attachment_ref.set(ATTR_URL, url)

boards/models/signature.py

@@ -1,150 +1,150 b''
 import xml.etree.ElementTree as et
 from django.db import models
 from boards.models import KeyPair
 
 
 TAG_MODEL = 'model'
 TAG_REQUEST = 'request'
 TAG_ID = 'id'
 
 TYPE_GET = 'get'
-TYPE_PULL = 'pull'
+TYPE_LIST = 'list'
 
 ATTR_VERSION = 'version'
 ATTR_TYPE = 'type'
 ATTR_NAME = 'name'
 
 ATTR_KEY = 'key'
 ATTR_KEY_TYPE = 'type'
 ATTR_LOCAL_ID = 'local-id'
 
 
 class GlobalIdManager(models.Manager):
     def generate_request_get(self, global_id_list: list):
         """
         Form a get request from a list of ModelId objects.
         """
 
         request = et.Element(TAG_REQUEST)
         request.set(ATTR_TYPE, TYPE_GET)
         request.set(ATTR_VERSION, '1.0')
 
         model = et.SubElement(request, TAG_MODEL)
         model.set(ATTR_VERSION, '1.0')
         model.set(ATTR_NAME, 'post')
 
         for global_id in global_id_list:
             tag_id = et.SubElement(model, TAG_ID)
             global_id.to_xml_element(tag_id)
 
         return et.tostring(request, 'unicode')
 
-    def generate_request_pull(self):
+    def generate_request_list(self):
         """
         Form a pull request from a list of ModelId objects.
         """
 
         request = et.Element(TAG_REQUEST)
-        request.set(ATTR_TYPE, TYPE_PULL)
+        request.set(ATTR_TYPE, TYPE_LIST)
         request.set(ATTR_VERSION, '1.0')
 
         model = et.SubElement(request, TAG_MODEL)
         model.set(ATTR_VERSION, '1.0')
         model.set(ATTR_NAME, 'post')
 
         return et.tostring(request, 'unicode')
 
     def global_id_exists(self, global_id):
         """
         Checks if the same global id already exists in the system.
         """
 
         return self.filter(key=global_id.key,
                            key_type=global_id.key_type,
                            local_id=global_id.local_id).exists()
 
 
 class GlobalId(models.Model):
     """
     Global model ID and cache.
     Key, key type and local ID make a single global identificator of the model.
     Content is an XML cache of the model that can be passed along between nodes
     without manual serialization each time.
     """
     class Meta:
         app_label = 'boards'
 
     objects = GlobalIdManager()
 
     def __init__(self, *args, **kwargs):
         models.Model.__init__(self, *args, **kwargs)
 
         if 'key' in kwargs and 'key_type' in kwargs and 'local_id' in kwargs:
             self.key = kwargs['key']
             self.key_type = kwargs['key_type']
             self.local_id = kwargs['local_id']
 
     key = models.TextField()
     key_type = models.TextField()
     local_id = models.IntegerField()
     content = models.TextField(blank=True, null=True)
 
     def __str__(self):
         return '%s::%s::%d' % (self.key_type, self.key, self.local_id)
 
     def to_xml_element(self, element: et.Element):
         """
         Exports global id to an XML element.
         """
 
         element.set(ATTR_KEY, self.key)
         element.set(ATTR_KEY_TYPE, self.key_type)
         element.set(ATTR_LOCAL_ID, str(self.local_id))
 
     @staticmethod
     def from_xml_element(element: et.Element):
         """
         Parses XML id tag and gets global id from it.
 
         Arguments:
         element -- the XML 'id' element
 
         Returns:
         global_id -- id itself
         exists -- True if the global id was taken from database, False if it
         did not exist and was created.
         """
 
         try:
             return GlobalId.objects.get(key=element.get(ATTR_KEY),
                                         key_type=element.get(ATTR_KEY_TYPE),
                                         local_id=int(element.get(
                                             ATTR_LOCAL_ID))), True
         except GlobalId.DoesNotExist:
             return GlobalId(key=element.get(ATTR_KEY),
                             key_type=element.get(ATTR_KEY_TYPE),
                             local_id=int(element.get(ATTR_LOCAL_ID))), False
 
     def is_local(self):
         """Checks fo the ID is local model's"""
         return KeyPair.objects.filter(
                 key_type=self.key_type, public_key=self.key).exists()
 
 
 class Signature(models.Model):
     class Meta:
         app_label = 'boards'
 
     def __init__(self, *args, **kwargs):
         models.Model.__init__(self, *args, **kwargs)
 
         if 'key' in kwargs and 'key_type' in kwargs and 'signature' in kwargs:
             self.key_type = kwargs['key_type']
             self.key = kwargs['key']
             self.signature = kwargs['signature']
 
     key_type = models.TextField()
     key = models.TextField()
     signature = models.TextField()
 
     global_id = models.ForeignKey('GlobalId')

boards/static/js/thread.js

@@ -1,143 +1,159 b''
 /*
  @licstart  The following is the entire license notice for the
  JavaScript code in this page.
 
 
  Copyright (C) 2013  neko259
 
  The JavaScript code in this page is free software: you can
  redistribute it and/or modify it under the terms of the GNU
  General Public License (GNU GPL) as published by the Free Software
  Foundation, either version 3 of the License, or (at your option)
  any later version.  The code is distributed WITHOUT ANY WARRANTY;
  without even the implied warranty of MERCHANTABILITY or FITNESS
  FOR A PARTICULAR PURPOSE.  See the GNU GPL for more details.
 
  As additional permission under GNU GPL version 3 section 7, you
  may distribute non-source (e.g., minimized or compacted) forms of
  that code without the copy of the GNU GPL normally required by
  section 4, provided you include this license notice and a URL
  through which recipients can access the Corresponding Source.
 
  @licend  The above is the entire license notice
  for the JavaScript code in this page.
  */
 
 var CLOSE_BUTTON = '#form-close-button';
 var REPLY_TO_MSG = '.reply-to-message';
 var REPLY_TO_MSG_ID = '#reply-to-message-id';
 
 var $html = $("html, body");
 
 function moveCaretToEnd(el) {
         if (typeof el.selectionStart == "number") {
             el.selectionStart = el.selectionEnd = el.value.length;
         } else if (typeof el.createTextRange != "undefined") {
             el.focus();
             var range = el.createTextRange();
             range.collapse(false);
             range.select();
         }
 }
 
 function getForm() {
     return $('.post-form-w');
 }
 
 function resetFormPosition() {
     var form = getForm();
     form.insertAfter($('.thread'));
 
     $(CLOSE_BUTTON).hide();
     $(REPLY_TO_MSG).hide();
 }
 
 function showFormAfter(blockToInsertAfter) {
     var form = getForm();
     form.insertAfter(blockToInsertAfter);
 
     $(CLOSE_BUTTON).show();
     form.show();
     $(REPLY_TO_MSG_ID).text(blockToInsertAfter.attr('id'));
     $(REPLY_TO_MSG).show();
 }
 
 function addQuickReply(postId) {
     // If we click "reply" on the same post, it means "cancel"
     if (getForm().prev().attr('id') == postId) {
         resetFormPosition();
     } else {
         var blockToInsert = null;
         var textAreaJq = $('textarea');
         var postLinkRaw = '[post]' + postId + '[/post]'
         var textToAdd = '';
 
         if (postId != null) {
             var post = $('#' + postId);
 
             // If this is not OP, add reflink to the post. If there already is
             // the same reflink, don't add it again.
             if (!post.is(':first-child') && textAreaJq.val().indexOf(postLinkRaw) < 0) {
                 textToAdd += postLinkRaw + '\n';
             }
 
             textAreaJq.val(textAreaJq.val()+ textToAdd);
             blockToInsert = post;
         } else {
             blockToInsert = $('.thread');
         }
         showFormAfter(blockToInsert);
     
         textAreaJq.focus();
 
         var textarea = document.getElementsByTagName('textarea')[0];
         moveCaretToEnd(textarea);
     }
 }
 
 function addQuickQuote() {
+    var textAreaJq = $('textarea');
+
+    var quoteButton = $("#quote-button");
+    var postId = quoteButton.attr('data-post-id');
+    if (postId != null && getForm().prev().attr('id') != postId) {
+        addQuickReply(postId);
+    }
+
     var textToAdd = '';
-    var textAreaJq = $('textarea');
     var selection = window.getSelection().toString();
     if (selection.length == 0) {
-        selection = $("#quote-button").attr('data-text');
+        selection = quoteButton.attr('data-text');
     }
     if (selection.length > 0) {
         textToAdd += '[quote]' + selection + '[/quote]\n';
     }
 
-    textAreaJq.val(textAreaJq.val()+ textToAdd);
+    textAreaJq.val(textAreaJq.val() + textToAdd);
 
     textAreaJq.focus();
 
     var textarea = document.getElementsByTagName('textarea')[0];
     moveCaretToEnd(textarea);
 }
 
 function scrollToBottom() {
     $html.animate({scrollTop: $html.height()}, "fast");
 }
 
 function showQuoteButton() {
     var selection = window.getSelection().getRangeAt(0).getBoundingClientRect();
     var quoteButton = $("#quote-button");
     if (selection.width > 0) {
         // quoteButton.offset({ top: selection.top - selection.height, left: selection.left });
         quoteButton.css({top: selection.top + $(window).scrollTop() - 30, left: selection.left});
         quoteButton.show();
 
         var text = window.getSelection().toString();
         quoteButton.attr('data-text', text);
+
+        var rect = window.getSelection().getRangeAt(0).getBoundingClientRect();
+        var element = $(document.elementFromPoint(rect.x, rect.y));
+        var postParent = element.parents('.post');
+        if (postParent.length > 0) {
+            quoteButton.attr('data-post-id', postParent.attr('id'));
+        } else {
+            quoteButton.attr('data-post-id', null);
+        }
     } else {
         quoteButton.hide();
     }
 }
 
 $(document).ready(function() {
     $('body').on('mouseup', function() {
         showQuoteButton();
     });
     $("#quote-button").click(function() {
         addQuickQuote();
     })
 });
  No newline at end of file

boards/tests/test_keys.py

@@ -1,89 +1,91 b''
 from base64 import b64encode
 import logging
 
 from django.test import TestCase
 from boards.models import KeyPair, GlobalId, Post, Signature
 from boards.models.post.sync import SyncManager
 
 logger = logging.getLogger(__name__)
 
 
 class KeyTest(TestCase):
     def test_create_key(self):
         key = KeyPair.objects.generate_key('ecdsa')
 
         self.assertIsNotNone(key, 'The key was not created.')
 
     def test_validation(self):
         key = KeyPair.objects.generate_key(key_type='ecdsa')
         message = 'msg'
         signature_value = key.sign(message)
 
         signature = Signature(key_type='ecdsa', key=key.public_key,
                               signature=signature_value)
         valid = KeyPair.objects.verify(signature, message)
 
         self.assertTrue(valid, 'Message verification failed.')
 
     def test_primary_constraint(self):
         KeyPair.objects.generate_key(key_type='ecdsa', primary=True)
 
         with self.assertRaises(Exception):
             KeyPair.objects.generate_key(key_type='ecdsa', primary=True)
 
     def test_model_id_save(self):
         model_id = GlobalId(key_type='test', key='test key', local_id='1')
         model_id.save()
 
     def test_request_get(self):
         post = self._create_post_with_key()
 
         request = GlobalId.objects.generate_request_get([post.global_id])
         logger.debug(request)
 
         key = KeyPair.objects.get(primary=True)
         self.assertTrue('<request type="get" version="1.0">'
                         '<model name="post" version="1.0">'
                         '<id key="%s" local-id="1" type="%s" />'
                         '</model>'
                         '</request>' % (
                             key.public_key,
                             key.key_type,
                         ) in request,
                         'Wrong XML generated for the GET request.')
 
     def test_response_get(self):
         post = self._create_post_with_key()
         reply_post = Post.objects.create_post(title='test_title',
                                               text='[post]%d[/post]' % post.id,
                                               thread=post.get_thread())
 
         response = SyncManager.generate_response_get([reply_post])
         logger.debug(response)
 
         key = KeyPair.objects.get(primary=True)
         self.assertTrue('<status>success</status>'
                         '<models>'
                         '<model name="post">'
                         '<content>'
                         '<id key="%s" local-id="%d" type="%s" />'
                         '<title>test_title</title>'
                         '<text>[post]%s[/post]</text>'
                         '<thread><id key="%s" local-id="%d" type="%s" /></thread>'
                         '<pub-time>%s</pub-time>'
+                        '<version>%s</version>'
                         '</content>' % (
                             key.public_key,
                             reply_post.id,
                             key.key_type,
                             str(post.global_id),
                             key.public_key,
                             post.id,
                             key.key_type,
                             str(reply_post.get_pub_time_str()),
+                            post.version,
                         ) in response,
                         'Wrong XML generated for the GET response.')
 
     def _create_post_with_key(self):
         KeyPair.objects.generate_key(primary=True)
 
         return Post.objects.create_post(title='test_title', text='test_text')

boards/tests/test_sync.py

@@ -1,102 +1,106 b''
 from boards.models import KeyPair, Post, Tag
 from boards.models.post.sync import SyncManager
 from boards.tests.mocks import MockRequest
 from boards.views.sync import response_get
 
 __author__ = 'neko259'
 
 
 from django.test import TestCase
 
 
 class SyncTest(TestCase):
     def test_get(self):
         """
         Forms a GET request of a post and checks the response.
         """
 
         key = KeyPair.objects.generate_key(primary=True)
         tag = Tag.objects.create(name='tag1')
         post = Post.objects.create_post(title='test_title',
                                         text='test_text\rline two',
                                         tags=[tag])
 
         request = MockRequest()
         request.body = (
             '<request type="get" version="1.0">'
             '<model name="post" version="1.0">'
             '<id key="%s" local-id="%d" type="%s" />'
             '</model>'
             '</request>' % (post.global_id.key,
                             post.id,
                             post.global_id.key_type)
         )
 
         response = response_get(request).content.decode()
         self.assertTrue(
             '<status>success</status>'
             '<models>'
             '<model name="post">'
             '<content>'
             '<id key="%s" local-id="%d" type="%s" />'
             '<title>%s</title>'
             '<text>%s</text>'
             '<tags><tag>%s</tag></tags>'
             '<pub-time>%s</pub-time>'
+            '<version>%s</version>'
             '</content>' % (
                 post.global_id.key,
                 post.global_id.local_id,
                 post.global_id.key_type,
                 post.title,
                 post.get_sync_text(),
                 post.get_thread().get_tags().first().name,
                 post.get_pub_time_str(),
+                post.version,
             ) in response,
             'Wrong response generated for the GET request.')
 
         post.delete()
         key.delete()
 
         KeyPair.objects.generate_key(primary=True)
 
         SyncManager.parse_response_get(response, None)
         self.assertEqual(1, Post.objects.count(),
                          'Post was not created from XML response.')
 
         parsed_post = Post.objects.first()
         self.assertEqual('tag1',
                          parsed_post.get_thread().get_tags().first().name,
                          'Invalid tag was parsed.')
 
         SyncManager.parse_response_get(response, None)
         self.assertEqual(1, Post.objects.count(),
                          'The same post was imported twice.')
 
         self.assertEqual(1, parsed_post.global_id.signature_set.count(),
                          'Signature was not saved.')
 
         post = parsed_post
 
         # Trying to sync the same once more
         response = response_get(request).content.decode()
 
         self.assertTrue(
             '<status>success</status>'
             '<models>'
             '<model name="post">'
             '<content>'
             '<id key="%s" local-id="%d" type="%s" />'
             '<title>%s</title>'
             '<text>%s</text>'
             '<tags><tag>%s</tag></tags>'
             '<pub-time>%s</pub-time>'
+            '<version>%s</version>'
             '</content>' % (
                 post.global_id.key,
                 post.global_id.local_id,
                 post.global_id.key_type,
                 post.title,
                 post.get_sync_text(),
                 post.get_thread().get_tags().first().name,
                 post.get_pub_time_str(),
+                post.version,
             ) in response,
             'Wrong response generated for the GET request.')

boards/urls.py

@@ -1,99 +1,96 b''
 from django.conf.urls import url
-#from django.views.i18n import javascript_catalog
 
 import neboard
 
 from boards import views
 from boards.rss import AllThreadsFeed, TagThreadsFeed, ThreadPostsFeed
 from boards.views import api, tag_threads, all_threads, \
     settings, all_tags, feed
 from boards.views.authors import AuthorsView
 from boards.views.notifications import NotificationView
-from boards.views.search import BoardSearchView
 from boards.views.static import StaticPageView
 from boards.views.preview import PostPreviewView
-from boards.views.sync import get_post_sync_data, response_get, response_pull
+from boards.views.sync import get_post_sync_data, response_get, response_list
 from boards.views.random import RandomImageView
 from boards.views.tag_gallery import TagGalleryView
 from boards.views.translation import cached_javascript_catalog
 
 
 js_info_dict = {
     'packages': ('boards',),
 }
 
 urlpatterns = [
     # /boards/
     url(r'^$', all_threads.AllThreadsView.as_view(), name='index'),
 
     # /boards/tag/tag_name/
     url(r'^tag/(?P<tag_name>\w+)/$', tag_threads.TagView.as_view(),
         name='tag'),
 
     # /boards/thread/
     url(r'^thread/(?P<post_id>\d+)/$', views.thread.NormalThreadView.as_view(),
         name='thread'),
     url(r'^thread/(?P<post_id>\d+)/mode/gallery/$', views.thread.GalleryThreadView.as_view(),
         name='thread_gallery'),
     url(r'^thread/(?P<post_id>\d+)/mode/tree/$', views.thread.TreeThreadView.as_view(),
         name='thread_tree'),
     # /feed/
     url(r'^feed/$', views.feed.FeedView.as_view(), name='feed'),
 
     url(r'^settings/$', settings.SettingsView.as_view(), name='settings'),
     url(r'^tags/(?P<query>\w+)?/?$', all_tags.AllTagsView.as_view(), name='tags'),
     url(r'^authors/$', AuthorsView.as_view(), name='authors'),
 
     url(r'^banned/$', views.banned.BannedView.as_view(), name='banned'),
     url(r'^staticpage/(?P<name>\w+)/$', StaticPageView.as_view(),
         name='staticpage'),
 
     url(r'^random/$', RandomImageView.as_view(), name='random'),
     url(r'^tag/(?P<tag_name>\w+)/gallery/$', TagGalleryView.as_view(), name='tag_gallery'),
 
     # RSS feeds
     url(r'^rss/$', AllThreadsFeed()),
     url(r'^page/(?P<page>\d+)/rss/$', AllThreadsFeed()),
     url(r'^tag/(?P<tag_name>\w+)/rss/$', TagThreadsFeed()),
     url(r'^tag/(?P<tag_name>\w+)/page/(?P<page>\w+)/rss/$', TagThreadsFeed()),
     url(r'^thread/(?P<post_id>\d+)/rss/$', ThreadPostsFeed()),
 
     # i18n
     url(r'^jsi18n/$', cached_javascript_catalog, js_info_dict,
         name='js_info_dict'),
 
     # API
     url(r'^api/post/(?P<post_id>\d+)/$', api.get_post, name="get_post"),
     url(r'^api/diff_thread/$', api.api_get_threaddiff, name="get_thread_diff"),
     url(r'^api/threads/(?P<count>\w+)/$', api.api_get_threads,
             name='get_threads'),
     url(r'^api/tags/$', api.api_get_tags, name='get_tags'),
     url(r'^api/thread/(?P<opening_post_id>\w+)/$', api.api_get_thread_posts,
             name='get_thread'),
     url(r'^api/add_post/(?P<opening_post_id>\w+)/$', api.api_add_post,
             name='add_post'),
     url(r'^api/notifications/(?P<username>\w+)/$', api.api_get_notifications,
             name='api_notifications'),
     url(r'^api/preview/$', api.api_get_preview, name='preview'),
     url(r'^api/new_posts/$', api.api_get_new_posts, name='new_posts'),
 
     # Sync protocol API
-    url(r'^api/sync/pull/$', response_pull, name='api_sync_pull'),
-    url(r'^api/sync/get/$', response_get, name='api_sync_pull'),
-    # TODO 'get' request
+    url(r'^api/sync/list/$', response_list, name='api_sync_list'),
+    url(r'^api/sync/get/$', response_get, name='api_sync_get'),
 
     # Notifications
     url(r'^notifications/(?P<username>\w+)/$', NotificationView.as_view(), name='notifications'),
     url(r'^notifications/$', NotificationView.as_view(), name='notifications'),
 
     # Post preview
     url(r'^preview/$', PostPreviewView.as_view(), name='preview'),
     url(r'^post_xml/(?P<post_id>\d+)$', get_post_sync_data,
         name='post_sync_data'),
 ]
 
 # Search
 if 'haystack' in neboard.settings.INSTALLED_APPS:
     from boards.views.search import BoardSearchView
     urlpatterns.append(url(r'^search/$', BoardSearchView.as_view(), name='search'))
 

boards/views/sync.py

@@ -1,62 +1,55 b''
 import xml.etree.ElementTree as et
-import xml.dom.minidom
 
 from django.http import HttpResponse, Http404
 from boards.models import GlobalId, Post
 from boards.models.post.sync import SyncManager
 
 
-def response_pull(request):
+def response_list(request):
     request_xml = request.body
 
-    if request_xml is None:
+    if request_xml is None or len(request_xml) == 0:
         return HttpResponse(content='Use the API')
 
-    response_xml = SyncManager.generate_response_pull()
+    response_xml = SyncManager.generate_response_list()
 
     return HttpResponse(content=response_xml)
 
 
 def response_get(request):
     """
     Processes a GET request with post ID list and returns the posts XML list.
     Request should contain an 'xml' post attribute with the actual request XML.
     """
 
     request_xml = request.body
 
-    if request_xml is None:
+    if request_xml is None or len(request_xml) == 0:
         return HttpResponse(content='Use the API')
 
     posts = []
 
     root_tag = et.fromstring(request_xml)
     model_tag = root_tag[0]
     for id_tag in model_tag:
         global_id, exists = GlobalId.from_xml_element(id_tag)
         if exists:
             posts.append(Post.objects.get(global_id=global_id))
 
     response_xml = SyncManager.generate_response_get(posts)
 
     return HttpResponse(content=response_xml)
 
 
 def get_post_sync_data(request, post_id):
     try:
         post = Post.objects.get(id=post_id)
     except Post.DoesNotExist:
         raise Http404()
 
     xml_str = SyncManager.generate_response_get([post])
 
-    xml_repr = xml.dom.minidom.parseString(xml_str)
-    xml_repr = xml_repr.toprettyxml()
-
-    content = '=Global ID=\n%s\n\n=XML=\n%s' \
-              % (post.global_id, xml_repr)
-
     return HttpResponse(
-        content_type='text/plain; charset=utf-8',
-        content=content,
-    )
  No newline at end of file
+        content_type='text/xml; charset=utf-8',
+        content=xml_str,
+    )

docs/api.markdown

@@ -1,81 +1,81 b''
 # INTRO #
 
 The API is provided to query the data from a neaboard server by any client
 application.
 
 Tha data is returned in the json format and got by an http query.
 
 # METHODS #
 
 ## Threads ##
 
     /api/threads/N/?offset=M&tag=O
 
 Get a thread list. You will get ``N`` threads (required parameter) starting from
 ``M``th one (optional parameter, default is 0) with the tag ``O`` (optional parameter,
 threads with any tags are shown by default).
 
 ## Tags ##
 
     /api/tags/
 
 Get all active tag list. Active tag is a tag that has at least 1 active thread
 associated with it.
 
 ## Thread ##
 
     /api/thread/N/
 
 Get all ``N``th thread post. ``N`` is an opening post ID for the thread.
 
 Output format:
 
 * ``posts``: list of posts
 * ``last_update``: last update timestamp
 
 ## Thread diff ##
 
     /api/diff_thread
 
 Parameters:
 
 * ``thread``: thread id
 * ``last_update``: last update timestamp
 * ``last_post``: last added post id
 
-Get the diff of the thread in the `O`` 
+Get the diff of the thread in the ``O`` 
 format. 2 formats are available: ``html`` (used in AJAX thread update) and 
 ``json``. The default format is ``html``. Return list format:
 
 * ``added``: list of added posts
 * ``updated``: list of updated posts
 * ``last_update``: last update timestamp
 
 ## Notifications ##
 
     /api/notifications/<username>/[?last=<id>]
 
 Get user notifications for user starting from the post ID.
 
 * ``username``: name of the notified user
 * ``id``: ID of a last notification post
 
 ## General info ##
 
 In case of incorrect request you can get http error 404.
 
 Response JSON for a post or thread contains:
 
 * ``id``
 * ``title``
 * ``text``
 * ``image`` (if image available)
 * ``image_preview`` (if image available)
 * ``bump_time`` (for threads)
 
 In future, it will also contain:
 
 * tags list (for thread)
 * publishing time
 * bump time
 * reply IDs (if available)

docs/dip-1.markdown

@@ -1,203 +1,210 b''
 # 0 Title #
 
 DIP-1 Common protocol description
 
 # 1 Intro #
 
 This document describes the Data Interchange Protocol (DIP), designed to
 exchange filtered data that can be stored as a graph structure between
 network nodes.
 
 # 2 Purpose #
 
 This protocol will be used to share the models (originally imageboard posts)
 across multiple servers. The main differnce of this protocol is that the node
 can specify what models it wants to get and from whom. The nodes can get
 models from a specific server, or from all except some specific servers. Also
 the models can be filtered by timestamps or tags.
 
 # 3 Protocol description #
 
 The node requests other node's changes list since some time (since epoch if
 this is the start). The other node sends a list of post ids or posts in the
 XML format.
 
 Protocol version is the version of the sync api. Model version is the version
 of data models. If at least one of them is different, the sync cannot be
 performed.
 
 The node signs the data with its keys. The receiving node saves the key at the
 first sync and checks it every time. If the key has changed, the info won't be
 saved from the node (or the node id must be changed). A model can be signed 
 with several keys but at least one of them must be the same as in the global 
 ID to verify the sender.
 
 Each node can have several keys. Nodes can have shared keys to serve as a pool
 (several nodes with the same key).
 
 Each post has an ID in the unique format: key-type::key::local-id
 
 All requests pass a request type, protocol and model versions, and a list of
 optional arguments used for filtering.
 
 Each request has its own version. Version consists of 2 numbers: first is
 incompatible version (1.3 and 2.0 are not compatible and must not be in sync)
 and the second one is minor and compatible (for example, new optional field
 is added which will be igroned by those who don't support it yet).
 
 Post edits and reflinks are not saved to the sync model. The replied post ID 
 can be got from the post text, and reflinks can be computed when loading 
 posts. The edit time is not saved because a foreign post can be 'edited' (new
 replies are added) but the signature must not change (so we can't update the
 content). The inner posts can be edited, and the signature will change then
 but the local-id won't, so the other node can detect that and replace the post
 instead of adding a new one.
 
 ## 3.1 Requests ##
 
 There is no constraint on how the server should calculate the request. The
 server can return any information by any filter and the requesting node is
 responsible for validating it.
 
 The server is required to return the status of request. See 3.2 for details.
 
-### 3.1.1 pull ###
+### 3.1.1 list ###
 
-"pull" request gets the desired model id list by the given filter (e.g. thread, tags,
+"list" request gets the desired model id list by the given filter (e.g. thread, tags,
 author)
 
 Sample request is as follows:
 
     <?xml version="1.1" encoding="UTF-8" ?>
-    <request version="1.0" type="pull">
+    <request version="1.0" type="list">
         <model version="1.0" name="post">
             <timestamp_from>0</timestamp_from>
             <timestamp_to>0</timestamp_to>
             <tags>
                 <tag>tag1</tag>
             </tags>
             <sender>
                 <allow>
                     <key>abcehy3h9t</key>
                     <key>ehoehyoe</key>
                 </allow>
                 <!-- There can be only allow block (all other are denied) or deny block (all other are allowed) -->
             </sender>
         </model>
     </request>
 
 Under the <model> tag there are filters. Filters for the "post" model can
 be found in DIP-2.
 
 Sample response:
 
     <?xml version="1.1" encoding="UTF-8" ?>
     <response>
         <status>success</status>
         <models>
-            <id key="id1" type="ecdsa" local-id="1" />
-            <id key="id1" type="ecdsa" local-id="2" />
-            <id key="id2" type="ecdsa" local-id="1" />
-            <id key="id2" type="ecdsa" local-id="5" />
+            <model>
+                <id key="id1" type="ecdsa" local-id="1">
+                <version>1</version>
+            </model>
+            <model>
+                <id key="id1" type="ecdsa" local-id="2" />
+            </model>
+            <model>
+                <id key="id2" type="ecdsa" local-id="1" />
+                <some-valuable-attr>value</some-valuable-attr>
+            </model>
         </models>
     </response>
 
 ### 3.1.2 get ###
 
 "get" gets models by id list.
 
 Sample request:
 
     <?xml version="1.1" encoding="UTF-8" ?>
     <request version="1.0" type="get">
         <model version="1.0" name="post">
             <id key="id1" type="ecdsa" local-id="1" />
             <id key="id1" type="ecdsa" local-id="2" />
         </model>
     </request>
 
 Id consists of a key, key type and local id. This key is used for signing and
 validating of data in the model content.
 
 Sample response:
 
     <?xml version="1.1" encoding="UTF-8" ?>
     <response>
         <!--
              Valid statuses are 'success' and 'error'.
         -->
         <status>success</status>
         <models>
             <model name="post">
                 <!--
                     Content tag is the data that is signed by signatures and must
                     not be changed for the post from other node.
                 -->
                 <content>
                     <id key="id1" type="ecdsa" local-id="1" />
                     <title>13</title>
                     <text>Thirteen</text>
                     <thread><id key="id1" type="ecdsa" local-id="2" /></thread>
                     <pub-time>12</pub-time>
                     <!--
                         Images are saved as attachments and included in the
                         signature.
                     -->
                     <attachments>
                         <attachment mimetype="image/png" id-type="md5">TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5I</attachment>
                     </attachments>
                 </content>
                 <!--
                     There can be several signatures for one model. At least one
                     signature must be made with the key used in global ID.
                  -->
                 <signatures>
                     <signature key="id1" type="ecdsa" value="dhefhtreh" />
                     <signature key="id45" type="ecdsa" value="dsgfgdhefhtreh" />
                 </signatures>
                 <attachment-refs>
                     <attachment-ref ref="TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5IGhpcyByZWFzb24sIGJ1dCBieSB0"
                         url="/media/images/12345.png" />
                 </attachment-refs>
             </model>
             <model name="post">
                 <content>
                     <id key="id1" type="ecdsa" local-id="id2" />
                     <title>13</title>
                     <text>Thirteen</text>
                     <pub-time>12</pub-time>
                     <edit-time>13</edit-time>
                     <tags>
                         <tag>tag1</tag>
                     </tags>
                 </content>
                 <signatures>
                     <signature key="id2" type="ecdsa" value="dehdfh" />
                 </signatures>
             </model>
         </models>
     </response>
 
 ### 3.1.3 put ###
 
 "put" gives a model to the given node (you have no guarantee the node takes
 it, consider you are just advising the node to take your post). This request
 type is useful in pool where all the nodes try to duplicate all of their data
 across the pool.
 
 ## 3.2 Responses ##
 
 ### 3.2.1 "not supported" ###
 
 If the request if completely not supported, a "not supported" status will be
 returned.
 
 ### 3.2.2 "success" ###
 
 "success" status means the request was processed and the result is returned.
 
 ### 3.2.3 "error" ###
 
 If the server knows for sure that the operation cannot be processed, it sends
 the "error" status. Additional tags describing the error may be <description>
 and <stack>.

docs/dip-2.markdown

@@ -1,30 +1,31 b''
 # 0 Title #
 
 "post" model reference
 
 # 1 Description #
 
 "post" is a model that defines an imageboard message, or post.
 
 # 2 Fields #
 
 # 2.1 Mandatory fields #
 
 * title -- text field.
 * text -- text field.
 * pub-time -- timestamp (TBD: Define format).
+* version -- when post content changes, version should be incremented.
 
 # 2.2 Optional fields #
 
 * attachments -- defines attachments such as images.
 * attachment -- contains and attachment or link to attachment to be downloaded
 manually. Required attributes are mimetype and name.
 
 This field is used for the opening post (thread):
 
 * tags -- text tag name list.
 
 This field is used for non-opening post:
 
 * thread -- ID of a post model that this post is related to.
 * tripcode