Merge remote changes

boards/migrations/0045_post_version.py

@@ -0,0 +1,20 b''
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.5 on 2016-05-11 09:23
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('boards', '0044_globalid_content'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='post',
+            name='version',
+            field=models.IntegerField(default=1),
+        ),
+    ]

boards/admin.py

@@ -1,6 +1,7 b''
 from django.contrib import admin
 from django.utils.translation import ugettext_lazy as _
 from django.core.urlresolvers import reverse
+from django.db.models import F
 from boards.models import Post, Tag, Ban, Thread, Banner, PostImage, KeyPair, GlobalId
 
 
@@ -12,7 +13,8 b' class PostAdmin(admin.ModelAdmin):'
     search_fields = ('id', 'title', 'text', 'poster_ip')
     exclude = ('referenced_posts', 'refmap', 'images', 'global_id')
     readonly_fields = ('poster_ip', 'threads', 'thread', 'linked_images',
-            'attachments', 'uid', 'url', 'pub_time', 'opening', 'linked_global_id')
+            'attachments', 'uid', 'url', 'pub_time', 'opening', 'linked_global_id',
+            'version')
 
     def ban_poster(self, request, queryset):
         bans = 0
@@ -54,6 +56,11 b' class PostAdmin(admin.ModelAdmin):'
                         args=[global_id.id]), str(global_id))
     linked_global_id.allow_tags = True
 
+    def save_model(self, request, obj, form, change):
+        obj.increment_version()
+        obj.save()
+        obj.clear_cache()
+
     actions = ['ban_poster', 'ban_with_hiding']
 
 
@@ -96,6 +103,14 b' class ThreadAdmin(admin.ModelAdmin):'
     def save_related(self, request, form, formsets, change):
         super().save_related(request, form, formsets, change)
         form.instance.refresh_tags()
+
+    def save_model(self, request, obj, form, change):
+        op = obj.get_opening_post()
+        op.increment_version()
+        op.save(update_fields=['version'])
+        obj.save()
+        op.clear_cache()
+
     list_display = ('id', 'op', 'title', 'reply_count', 'status', 'ip',
             'display_tags')
     list_filter = ('bump_time', 'status')

boards/management/commands/invalidate_sync_cache.py

@@ -12,9 +12,11 b' class Command(BaseCommand):'
     @transaction.atomic
     def handle(self, *args, **options):
         count = 0
-        for global_id in GlobalId.objects.all():
+        for global_id in GlobalId.objects.exclude(content__isnull=True).exclude(
+                content=''):
             if global_id.is_local() and global_id.content is not None:
                 global_id.content = None
                 global_id.save()
+                global_id.signature_set.all().delete()
                 count += 1
         print('Invalidated {} caches.'.format(count))

boards/management/commands/sync_with_server.py

@@ -5,7 +5,7 b' import httplib2'
 from django.core.management import BaseCommand
 
 from boards.models import GlobalId
-from boards.models.post.sync import SyncManager
+from boards.models.post.sync import SyncManager, TAG_ID, TAG_VERSION
 
 __author__ = 'neko259'
 
@@ -27,7 +27,7 b' class Command(BaseCommand):'
     def handle(self, *args, **options):
         url = options.get('url')
 
-        pull_url = url + 'api/sync/pull/'
+        list_url = url + 'api/sync/list/'
         get_url = url + 'api/sync/get/'
         file_url = url[:-1]
 
@@ -52,8 +52,8 b' class Command(BaseCommand):'
                 raise Exception('Invalid global ID')
         else:
             h = httplib2.Http()
-            xml = GlobalId.objects.generate_request_pull()
-            response, content = h.request(pull_url, method="POST", body=xml)
+            xml = GlobalId.objects.generate_request_list()
+            response, content = h.request(list_url, method="POST", body=xml)
 
             print(content.decode() + '\n')
 
@@ -64,11 +64,16 b' class Command(BaseCommand):'
 
                 models = root.findall('models')[0]
                 for model in models:
-                    global_id, exists = GlobalId.from_xml_element(model)
-                    if not exists:
-                        print(global_id)
+                    tag_id = model.find(TAG_ID)
+                    global_id, exists = GlobalId.from_xml_element(tag_id)
+                    tag_version = model.find(TAG_VERSION)
+                    if tag_version is not None:
+                        version = int(tag_version.text) or 1
+                    else:
+                        version = 1
+                    if not exists or global_id.post.version < version:
                         ids_to_sync.append(global_id)
-                print()
+                print('Starting sync...')
 
                 if len(ids_to_sync) > 0:
                     limit = options.get('split_query', len(ids_to_sync))

boards/models/post/__init__.py

@@ -11,7 +11,7 b' from boards.utils import datetime_to_epo'
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.urlresolvers import reverse
 from django.db import models
-from django.db.models import TextField, QuerySet
+from django.db.models import TextField, QuerySet, F
 from django.template.defaultfilters import truncatewords, striptags
 from django.template.loader import render_to_string
 
@@ -104,6 +104,7 b' class Post(models.Model, Viewable):'
     tripcode = models.CharField(max_length=50, blank=True, default='')
     opening = models.BooleanField(db_index=True)
     hidden = models.BooleanField(default=False)
+    version = models.IntegerField(default=1)
 
     def __str__(self):
         return 'P#{}/{}'.format(self.id, self.get_title())
@@ -337,8 +338,11 b' class Post(models.Model, Viewable):'
 
         replacements = dict()
         for post_id in REGEX_REPLY.findall(self.get_raw_text()):
-            absolute_post_id = str(Post.objects.get(id=post_id).global_id)
-            replacements[post_id] = absolute_post_id
+            try:
+                absolute_post_id = str(Post.objects.get(id=post_id).global_id)
+                replacements[post_id] = absolute_post_id
+            except Post.DoesNotExist:
+                pass
 
         text = self.get_raw_text() or ''
         for key in replacements:
@@ -379,3 +383,14 b' class Post(models.Model, Viewable):'
 
     def set_hidden(self, hidden):
         self.hidden = hidden
+
+    def increment_version(self):
+        self.version = F('version') + 1
+
+    def clear_cache(self):
+        global_id = self.global_id
+        if global_id is not None and global_id.is_local()\
+                and global_id.content is not None:
+            global_id.content = None
+            global_id.save()
+            global_id.signature_set.all().delete()

boards/models/post/sync.py

@@ -32,6 +32,7 b" TAG_TAG = 'tag'"
 TAG_ATTACHMENT_REFS = 'attachment-refs'
 TAG_ATTACHMENT_REF = 'attachment-ref'
 TAG_TRIPCODE = 'tripcode'
+TAG_VERSION = 'version'
 
 TYPE_GET = 'get'
 
@@ -126,6 +127,8 b' class SyncManager:'
                         SyncManager._attachment_to_xml(
                             attachments_tag, attachment_refs, file.file.file,
                             file.hash, file.file.url)
+                version_tag = et.SubElement(content_tag, TAG_VERSION)
+                version_tag.text = str(post.version)
 
                 global_id.content = et.tostring(content_tag, ENCODING_UNICODE)
                 global_id.save()
@@ -227,11 +230,12 b' class SyncManager:'
                         title=title, text=text, pub_time=pub_time,
                         opening_post=opening_post, tags=tags,
                         global_id=global_id, files=files, tripcode=tripcode)
+                print('Parsed post {}'.format(global_id))
         else:
             raise SyncException(EXCEPTION_NODE.format(tag_status.text))
 
     @staticmethod
-    def generate_response_pull():
+    def generate_response_list():
         response = et.Element(TAG_RESPONSE)
 
         status = et.SubElement(response, TAG_STATUS)
@@ -240,8 +244,11 b' class SyncManager:'
         models = et.SubElement(response, TAG_MODELS)
 
         for post in Post.objects.prefetch_related('global_id').all():
-            tag_id = et.SubElement(models, TAG_ID)
+            tag_model = et.SubElement(models, TAG_MODEL)
+            tag_id = et.SubElement(tag_model, TAG_ID)
             post.global_id.to_xml_element(tag_id)
+            tag_version = et.SubElement(tag_model, TAG_VERSION)
+            tag_version.text = str(post.version)
 
         return et.tostring(response, ENCODING_UNICODE)
 

boards/models/signature.py

@@ -8,7 +8,7 b" TAG_REQUEST = 'request'"
 TAG_ID = 'id'
 
 TYPE_GET = 'get'
-TYPE_PULL = 'pull'
+TYPE_LIST = 'list'
 
 ATTR_VERSION = 'version'
 ATTR_TYPE = 'type'
@@ -39,13 +39,13 b' class GlobalIdManager(models.Manager):'
 
         return et.tostring(request, 'unicode')
 
-    def generate_request_pull(self):
+    def generate_request_list(self):
         """
         Form a pull request from a list of ModelId objects.
         """
 
         request = et.Element(TAG_REQUEST)
-        request.set(ATTR_TYPE, TYPE_PULL)
+        request.set(ATTR_TYPE, TYPE_LIST)
         request.set(ATTR_VERSION, '1.0')
 
         model = et.SubElement(request, TAG_MODEL)

boards/static/js/thread.js

@@ -96,17 +96,24 b' function addQuickReply(postId) {'
 }
 
 function addQuickQuote() {
+    var textAreaJq = $('textarea');
+
+    var quoteButton = $("#quote-button");
+    var postId = quoteButton.attr('data-post-id');
+    if (postId != null && getForm().prev().attr('id') != postId) {
+        addQuickReply(postId);
+    }
+
     var textToAdd = '';
-    var textAreaJq = $('textarea');
     var selection = window.getSelection().toString();
     if (selection.length == 0) {
-        selection = $("#quote-button").attr('data-text');
+        selection = quoteButton.attr('data-text');
     }
     if (selection.length > 0) {
         textToAdd += '[quote]' + selection + '[/quote]\n';
     }
 
-    textAreaJq.val(textAreaJq.val()+ textToAdd);
+    textAreaJq.val(textAreaJq.val() + textToAdd);
 
     textAreaJq.focus();
 
@@ -128,6 +135,15 b' function showQuoteButton() {'
 
         var text = window.getSelection().toString();
         quoteButton.attr('data-text', text);
+
+        var rect = window.getSelection().getRangeAt(0).getBoundingClientRect();
+        var element = $(document.elementFromPoint(rect.x, rect.y));
+        var postParent = element.parents('.post');
+        if (postParent.length > 0) {
+            quoteButton.attr('data-post-id', postParent.attr('id'));
+        } else {
+            quoteButton.attr('data-post-id', null);
+        }
     } else {
         quoteButton.hide();
     }

boards/tests/test_keys.py

@@ -71,6 +71,7 b' class KeyTest(TestCase):'
                         '<text>[post]%s[/post]</text>'
                         '<thread><id key="%s" local-id="%d" type="%s" /></thread>'
                         '<pub-time>%s</pub-time>'
+                        '<version>%s</version>'
                         '</content>' % (
                             key.public_key,
                             reply_post.id,
@@ -80,6 +81,7 b' class KeyTest(TestCase):'
                             post.id,
                             key.key_type,
                             str(reply_post.get_pub_time_str()),
+                            post.version,
                         ) in response,
                         'Wrong XML generated for the GET response.')
 

boards/tests/test_sync.py

@@ -43,6 +43,7 b' class SyncTest(TestCase):'
             '<text>%s</text>'
             '<tags><tag>%s</tag></tags>'
             '<pub-time>%s</pub-time>'
+            '<version>%s</version>'
             '</content>' % (
                 post.global_id.key,
                 post.global_id.local_id,
@@ -51,6 +52,7 b' class SyncTest(TestCase):'
                 post.get_sync_text(),
                 post.get_thread().get_tags().first().name,
                 post.get_pub_time_str(),
+                post.version,
             ) in response,
             'Wrong response generated for the GET request.')
 
@@ -90,6 +92,7 b' class SyncTest(TestCase):'
             '<text>%s</text>'
             '<tags><tag>%s</tag></tags>'
             '<pub-time>%s</pub-time>'
+            '<version>%s</version>'
             '</content>' % (
                 post.global_id.key,
                 post.global_id.local_id,
@@ -98,5 +101,6 b' class SyncTest(TestCase):'
                 post.get_sync_text(),
                 post.get_thread().get_tags().first().name,
                 post.get_pub_time_str(),
+                post.version,
             ) in response,
             'Wrong response generated for the GET request.')

boards/urls.py

@@ -1,5 +1,4 b''
 from django.conf.urls import url
-#from django.views.i18n import javascript_catalog
 
 import neboard
 
@@ -9,10 +8,9 b' from boards.views import api, tag_thread'
     settings, all_tags, feed
 from boards.views.authors import AuthorsView
 from boards.views.notifications import NotificationView
-from boards.views.search import BoardSearchView
 from boards.views.static import StaticPageView
 from boards.views.preview import PostPreviewView
-from boards.views.sync import get_post_sync_data, response_get, response_pull
+from boards.views.sync import get_post_sync_data, response_get, response_list
 from boards.views.random import RandomImageView
 from boards.views.tag_gallery import TagGalleryView
 from boards.views.translation import cached_javascript_catalog
@@ -78,9 +76,8 b' urlpatterns = ['
     url(r'^api/new_posts/$', api.api_get_new_posts, name='new_posts'),
 
     # Sync protocol API
-    url(r'^api/sync/pull/$', response_pull, name='api_sync_pull'),
-    url(r'^api/sync/get/$', response_get, name='api_sync_pull'),
-    # TODO 'get' request
+    url(r'^api/sync/list/$', response_list, name='api_sync_list'),
+    url(r'^api/sync/get/$', response_get, name='api_sync_get'),
 
     # Notifications
     url(r'^notifications/(?P<username>\w+)/$', NotificationView.as_view(), name='notifications'),

boards/views/sync.py

@@ -1,18 +1,17 b''
 import xml.etree.ElementTree as et
-import xml.dom.minidom
 
 from django.http import HttpResponse, Http404
 from boards.models import GlobalId, Post
 from boards.models.post.sync import SyncManager
 
 
-def response_pull(request):
+def response_list(request):
     request_xml = request.body
 
-    if request_xml is None:
+    if request_xml is None or len(request_xml) == 0:
         return HttpResponse(content='Use the API')
 
-    response_xml = SyncManager.generate_response_pull()
+    response_xml = SyncManager.generate_response_list()
 
     return HttpResponse(content=response_xml)
 
@@ -25,7 +24,7 b' def response_get(request):'
 
     request_xml = request.body
 
-    if request_xml is None:
+    if request_xml is None or len(request_xml) == 0:
         return HttpResponse(content='Use the API')
 
     posts = []
@@ -50,13 +49,7 b' def get_post_sync_data(request, post_id)'
 
     xml_str = SyncManager.generate_response_get([post])
 
-    xml_repr = xml.dom.minidom.parseString(xml_str)
-    xml_repr = xml_repr.toprettyxml()
-
-    content = '=Global ID=\n%s\n\n=XML=\n%s' \
-              % (post.global_id, xml_repr)
-
     return HttpResponse(
-        content_type='text/plain; charset=utf-8',
-        content=content,
-    )
  No newline at end of file
+        content_type='text/xml; charset=utf-8',
+        content=xml_str,
+    )

docs/api.markdown

@@ -43,7 +43,7 b' Parameters:'
 * ``last_update``: last update timestamp
 * ``last_post``: last added post id
 
-Get the diff of the thread in the `O`` 
+Get the diff of the thread in the ``O`` 
 format. 2 formats are available: ``html`` (used in AJAX thread update) and 
 ``json``. The default format is ``html``. Return list format:
 

docs/dip-1.markdown

@@ -61,15 +61,15 b' responsible for validating it.'
 
 The server is required to return the status of request. See 3.2 for details.
 
-### 3.1.1 pull ###
+### 3.1.1 list ###
 
-"pull" request gets the desired model id list by the given filter (e.g. thread, tags,
+"list" request gets the desired model id list by the given filter (e.g. thread, tags,
 author)
 
 Sample request is as follows:
 
     <?xml version="1.1" encoding="UTF-8" ?>
-    <request version="1.0" type="pull">
+    <request version="1.0" type="list">
         <model version="1.0" name="post">
             <timestamp_from>0</timestamp_from>
             <timestamp_to>0</timestamp_to>
@@ -95,10 +95,17 b' Sample response:'
     <response>
         <status>success</status>
         <models>
-            <id key="id1" type="ecdsa" local-id="1" />
-            <id key="id1" type="ecdsa" local-id="2" />
-            <id key="id2" type="ecdsa" local-id="1" />
-            <id key="id2" type="ecdsa" local-id="5" />
+            <model>
+                <id key="id1" type="ecdsa" local-id="1">
+                <version>1</version>
+            </model>
+            <model>
+                <id key="id1" type="ecdsa" local-id="2" />
+            </model>
+            <model>
+                <id key="id2" type="ecdsa" local-id="1" />
+                <some-valuable-attr>value</some-valuable-attr>
+            </model>
         </models>
     </response>
 

docs/dip-2.markdown

@@ -13,6 +13,7 b''
 * title -- text field.
 * text -- text field.
 * pub-time -- timestamp (TBD: Define format).
+* version -- when post content changes, version should be incremented.
 
 # 2.2 Optional fields #