Merge tip

boards/abstracts/sync_filters.py

@@ -0,0 +1,31 b''
+import xml.etree.ElementTree as et
+
+from boards.models import Post
+
+TAG_THREAD = 'thread'
+
+
+class PostFilter:
+    def __init__(self, content=None):
+        self.content = content
+
+    def filter(self, posts):
+        return posts
+
+    def add_filter(self, model_tag, value):
+        return model_tag
+
+
+class ThreadFilter(PostFilter):
+    def filter(self, posts):
+        op_id = self.content.text
+
+        op = Post.objects.filter(opening=True, id=op_id).first()
+        if op:
+            return posts.filter(thread=op.get_thread())
+        else:
+            return posts.none()
+
+    def add_filter(self, model_tag, value):
+        thread_tag = et.SubElement(model_tag, TAG_THREAD)
+        thread_tag.text = str(value)

boards/migrations/0056_auto_20170123_1620.py

@@ -0,0 +1,24 b''
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.5 on 2017-01-23 14:20
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('boards', '0055_auto_20161229_1132'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='attachment',
+            options={'ordering': ('id',)},
+        ),
+        migrations.AlterField(
+            model_name='post',
+            name='uid',
+            field=models.TextField(),
+        ),
+    ]

boards/config/default_settings.ini

@@ -18,6 +18,7 b' PostingDelay = 30'
 Autoban = false
 DefaultTag = test
 MaxFileCount = 1
+AdditionalSpoilerSpaces = false
 
 [Messages]
 # Thread bumplimit

boards/management/commands/sync_with_server.py

@@ -24,6 +24,8 b' class Command(BaseCommand):'
         parser.add_argument('--split-query', type=int, default=1,
                             help='Split GET query into separate by the given'
                                  ' number of posts in one')
+        parser.add_argument('--thread', type=int,
+                            help='Get posts of one specific thread')
 
     def handle(self, *args, **options):
         logger = logging.getLogger('boards.sync')
@@ -45,7 +47,7 b' class Command(BaseCommand):'
                 global_id = GlobalId(key_type=key_type, key=key,
                                      local_id=local_id)
 
-                xml = GlobalId.objects.generate_request_get([global_id])
+                xml = SyncManager.generate_request_get([global_id])
                 h = httplib2.Http()
                 response, content = h.request(get_url, method="POST", body=xml)
 
@@ -55,7 +57,8 b' class Command(BaseCommand):'
         else:
             logger.info('Running LIST request...')
             h = httplib2.Http()
-            xml = GlobalId.objects.generate_request_list()
+            xml = SyncManager.generate_request_list(
+                opening_post=options.get('thread'))
             response, content = h.request(list_url, method="POST", body=xml)
             logger.info('Processing response...')
 
@@ -83,7 +86,7 b' class Command(BaseCommand):'
                 if len(ids_to_sync) > 0:
                     limit = options.get('split_query', len(ids_to_sync))
                     for offset in range(0, len(ids_to_sync), limit):
-                        xml = GlobalId.objects.generate_request_get(ids_to_sync[offset:offset+limit])
+                        xml = SyncManager.generate_request_get(ids_to_sync[offset:offset + limit])
                         h = httplib2.Http()
                         logger.info('Running GET request...')
                         response, content = h.request(get_url, method="POST", body=xml)

boards/mdx_neboard.py

@@ -10,6 +10,7 b' from django.core.exceptions import Objec'
 from django.core.urlresolvers import reverse
 
 import boards
+from boards import settings
 
 
 __author__ = 'neko259'
@@ -24,6 +25,7 b' LINE_BREAK_HTML = \'<div class="br"></div'
 SPOILER_SPACE = '&nbsp;'
 
 MAX_SPOILER_MULTIPLIER = 2
+MAX_SPOILER_SPACE_COUNT = 20
 
 
 class TextFormatter():
@@ -202,11 +204,15 b' def render_tag(tag_name, value, options,'
 
 
 def render_spoiler(tag_name, value, options, parent, context):
-    text_len = len(value)
-    space_count = random.randint(0, text_len * MAX_SPOILER_MULTIPLIER)
-    side_spaces = SPOILER_SPACE * (space_count // 2)
-    return '<span class="spoiler">{}{}{}</span>'.format(side_spaces, value,
-            side_spaces)
+    if settings.get_bool('Forms', 'AdditionalSpoilerSpaces'):
+        text_len = len(value)
+        space_count = min(random.randint(0, text_len * MAX_SPOILER_MULTIPLIER),
+                MAX_SPOILER_SPACE_COUNT)
+        side_spaces = SPOILER_SPACE * (space_count // 2)
+    else:
+        side_spaces = ''
+    return '<span class="spoiler">{}{}{}</span>'.format(side_spaces,
+            value, side_spaces)
 
 
 formatters = [

boards/models/attachment/downloaders.py

@@ -30,15 +30,15 b' class Downloader:'
         return True
 
     @staticmethod
-    def download(url: str):
+    def download(url: str, validate):
         # Verify content headers
         response_head = requests.head(url, verify=False)
         content_type = response_head.headers[HEADER_CONTENT_TYPE].split(';')[0]
-        if content_type in TYPE_URL_ONLY:
+        if validate and content_type in TYPE_URL_ONLY:
             return None
 
         length_header = response_head.headers.get(HEADER_CONTENT_LENGTH)
-        if length_header:
+        if validate and length_header:
             length = int(length_header)
             validate_file_size(length)
         # Get the actual content into memory
@@ -63,7 +63,7 b' class Downloader:'
 
 class YouTubeDownloader(Downloader):
     @staticmethod
-    def download(url: str):
+    def download(url: str, validate):
         yt = YouTube()
         yt.from_url(url)
         videos = yt.filter(YOUTUBE_VIDEO_FORMAT)
@@ -82,7 +82,7 b' class NothingDownloader(Downloader):'
         return REGEX_MAGNET.match(url)
 
     @staticmethod
-    def download(url: str):
+    def download(url: str, validate):
         return None
 
 
@@ -93,9 +93,9 b' DOWNLOADERS = ('
 )
 
 
-def download(url):
+def download(url, validate=True):
     for downloader in DOWNLOADERS:
         if downloader.handles(url):
-            return downloader.download(url)
+            return downloader.download(url, validate=validate)
     raise Exception('No downloader supports this URL.')
 

boards/models/post/__init__.py

@@ -90,7 +90,7 b' class Post(models.Model, Viewable):'
     thread = models.ForeignKey('Thread', db_index=True, related_name='replies')
 
     url = models.TextField()
-    uid = models.TextField(db_index=True)
+    uid = models.TextField()
 
     # Global ID with author key. If the message was downloaded from another
     # server, this indicates the server.

boards/models/post/sync.py

@@ -1,9 +1,13 b''
 import xml.etree.ElementTree as et
 import logging
+from xml.etree import ElementTree
 
 from boards.abstracts.exceptions import SyncException
+from boards.abstracts.sync_filters import ThreadFilter
 from boards.models import KeyPair, GlobalId, Signature, Post, Tag
 from boards.models.attachment.downloaders import download
+from boards.models.signature import TAG_REQUEST, ATTR_TYPE, TYPE_GET, \
+    ATTR_VERSION, TAG_MODEL, ATTR_NAME, TAG_ID, TYPE_LIST
 from boards.utils import get_file_mimetype, get_file_hash
 from django.db import transaction
 from django import forms
@@ -238,7 +242,7 b' class SyncManager:'
                         TAG_ATTACHMENT_REF, attachment.text))
                     url = tag_ref.get(ATTR_URL)
                     try:
-                        attached_file = download(hostname + url)
+                        attached_file = download(hostname + url, validate=False)
 
                         if attached_file is None:
                             raise SyncException(EXCEPTION_DOWNLOAD)
@@ -269,7 +273,7 b' class SyncManager:'
                 logger.debug('Parsed new post {}'.format(global_id))
 
     @staticmethod
-    def generate_response_list():
+    def generate_response_list(filters):
         response = et.Element(TAG_RESPONSE)
 
         status = et.SubElement(response, TAG_STATUS)
@@ -277,7 +281,11 b' class SyncManager:'
 
         models = et.SubElement(response, TAG_MODELS)
 
-        for post in Post.objects.prefetch_related('global_id').all():
+        posts = Post.objects.prefetch_related('global_id')
+        for post_filter in filters:
+            posts = post_filter.filter(posts)
+
+        for post in posts:
             tag_model = et.SubElement(models, TAG_MODEL)
             tag_id = et.SubElement(tag_model, TAG_ID)
             post.global_id.to_xml_element(tag_id)
@@ -334,4 +342,4 b' class SyncManager:'
         if tag_refs is not None:
             attachment_ref = et.SubElement(tag_refs, TAG_ATTACHMENT_REF)
             attachment_ref.set(ATTR_REF, attachment.hash)
-            attachment_ref.set(ATTR_URL, attachment.file.url)
+            attachment_ref.set(ATTR_URL, attachment.file.url)	

boards/models/signature.py

@@ -1,8 +1,9 b''
 import xml.etree.ElementTree as et
+
 from django.db import models
+
 from boards.models import KeyPair
 
-
 TAG_MODEL = 'model'
 TAG_REQUEST = 'request'
 TAG_ID = 'id'
@@ -20,40 +21,6 b" ATTR_LOCAL_ID = 'local-id'"
 
 
 class GlobalIdManager(models.Manager):
-    def generate_request_get(self, global_id_list: list):
-        """
-        Form a get request from a list of ModelId objects.
-        """
-
-        request = et.Element(TAG_REQUEST)
-        request.set(ATTR_TYPE, TYPE_GET)
-        request.set(ATTR_VERSION, '1.0')
-
-        model = et.SubElement(request, TAG_MODEL)
-        model.set(ATTR_VERSION, '1.0')
-        model.set(ATTR_NAME, 'post')
-
-        for global_id in global_id_list:
-            tag_id = et.SubElement(model, TAG_ID)
-            global_id.to_xml_element(tag_id)
-
-        return et.tostring(request, 'unicode')
-
-    def generate_request_list(self):
-        """
-        Form a pull request from a list of ModelId objects.
-        """
-
-        request = et.Element(TAG_REQUEST)
-        request.set(ATTR_TYPE, TYPE_LIST)
-        request.set(ATTR_VERSION, '1.0')
-
-        model = et.SubElement(request, TAG_MODEL)
-        model.set(ATTR_VERSION, '1.0')
-        model.set(ATTR_NAME, 'post')
-
-        return et.tostring(request, 'unicode')
-
     def global_id_exists(self, global_id):
         """
         Checks if the same global id already exists in the system.

boards/signals.py

@@ -23,47 +23,50 b' THUMB_SIZES = ((200, 150),)'
 
 @receiver(post_save, sender=Post)
 def connect_replies(instance, **kwargs):
-    for reply_number in re.finditer(REGEX_REPLY, instance.get_raw_text()):
-        post_id = reply_number.group(1)
+    if not kwargs['update_fields']:
+        for reply_number in re.finditer(REGEX_REPLY, instance.get_raw_text()):
+            post_id = reply_number.group(1)
 
-        try:
-            referenced_post = Post.objects.get(id=post_id)
+            try:
+                referenced_post = Post.objects.get(id=post_id)
 
-            if not referenced_post.referenced_posts.filter(
-                    id=instance.id).exists():
-                referenced_post.referenced_posts.add(instance)
-                referenced_post.last_edit_time = instance.pub_time
-                referenced_post.build_refmap()
-                referenced_post.save(update_fields=['refmap', 'last_edit_time'])
-        except Post.DoesNotExist:
-            pass
+                if not referenced_post.referenced_posts.filter(
+                        id=instance.id).exists():
+                    referenced_post.referenced_posts.add(instance)
+                    referenced_post.last_edit_time = instance.pub_time
+                    referenced_post.build_refmap()
+                    referenced_post.save(update_fields=['refmap', 'last_edit_time'])
+            except Post.DoesNotExist:
+                pass
 
 
 @receiver(post_save, sender=Post)
 @receiver(post_import_deps, sender=Post)
 def connect_global_replies(instance, **kwargs):
-    for reply_number in re.finditer(REGEX_GLOBAL_REPLY, instance.get_raw_text()):
-        key_type = reply_number.group(1)
-        key = reply_number.group(2)
-        local_id = reply_number.group(3)
+    if not kwargs['update_fields']:
+        for reply_number in re.finditer(REGEX_GLOBAL_REPLY, instance.get_raw_text()):
+            key_type = reply_number.group(1)
+            key = reply_number.group(2)
+            local_id = reply_number.group(3)
 
-        try:
-            global_id = GlobalId.objects.get(key_type=key_type, key=key,
-                    local_id=local_id)
-            referenced_post = Post.objects.get(global_id=global_id)
-            referenced_post.referenced_posts.add(instance)
-            referenced_post.last_edit_time = instance.pub_time
-            referenced_post.build_refmap()
-            referenced_post.save(update_fields=['refmap', 'last_edit_time'])
-        except (GlobalId.DoesNotExist, Post.DoesNotExist):
-            pass
+            try:
+                global_id = GlobalId.objects.get(key_type=key_type, key=key,
+                        local_id=local_id)
+                referenced_post = Post.objects.get(global_id=global_id)
+                referenced_post.referenced_posts.add(instance)
+                referenced_post.last_edit_time = instance.pub_time
+                referenced_post.build_refmap()
+                referenced_post.save(update_fields=['refmap', 'last_edit_time'])
+            except (GlobalId.DoesNotExist, Post.DoesNotExist):
+                pass
 
 
 @receiver(post_save, sender=Post)
 def connect_notifications(instance, **kwargs):
-    for reply_number in re.finditer(REGEX_NOTIFICATION, instance.get_raw_text()):
-        user_name = reply_number.group(1).lower()
-        Notification.objects.get_or_create(name=user_name, post=instance)
+    if not kwargs['update_fields']:
+        for reply_number in re.finditer(REGEX_NOTIFICATION, instance.get_raw_text()):
+            user_name = reply_number.group(1).lower()
+            Notification.objects.get_or_create(name=user_name, post=instance)
 
 
 @receiver(pre_save, sender=Post)

boards/static/js/thread.js

@@ -57,40 +57,35 b' function showFormAfter(blockToInsertAfte'
 }
 
 function addQuickReply(postId) {
-    // If we click "reply" on the same post, it means "cancel"
-    if (getForm().prev().attr('id') == postId) {
-        resetFormPosition();
-    } else {
-        var blockToInsert = null;
-        var textAreaJq = getPostTextarea();
-        var postLinkRaw = '[post]' + postId + '[/post]'
-        var textToAdd = '';
+    var blockToInsert = null;
+    var textAreaJq = getPostTextarea();
+    var postLinkRaw = '[post]' + postId + '[/post]'
+    var textToAdd = '';
 
-        if (postId != null) {
-            var post = $('#' + postId);
+    if (postId != null) {
+        var post = $('#' + postId);
 
-            // If this is not OP, add reflink to the post. If there already is
-            // the same reflink, don't add it again.
-            var postText = textAreaJq.val();
-            if (!post.is(':first-child') && postText.indexOf(postLinkRaw) < 0) {
-                // Insert line break if none is present.
-                if (postText.length > 0 && !postText.endsWith('\n') && !postText.endsWith('\r')) {
-                    textToAdd += '\n';
-                }
-                textToAdd += postLinkRaw + '\n';
+        // If this is not OP, add reflink to the post. If there already is
+        // the same reflink, don't add it again.
+        var postText = textAreaJq.val();
+        if (!post.is(':first-child') && postText.indexOf(postLinkRaw) < 0) {
+            // Insert line break if none is present.
+            if (postText.length > 0 && !postText.endsWith('\n') && !postText.endsWith('\r')) {
+                textToAdd += '\n';
             }
+            textToAdd += postLinkRaw + '\n';
+        }
 
-            textAreaJq.val(textAreaJq.val()+ textToAdd);
-            blockToInsert = post;
-        } else {
-            blockToInsert = $('.thread');
-        }
-        showFormAfter(blockToInsert);
-    
-        textAreaJq.focus();
+        textAreaJq.val(textAreaJq.val()+ textToAdd);
+        blockToInsert = post;
+    } else {
+        blockToInsert = $('.thread');
+    }
+    showFormAfter(blockToInsert);
 
-        moveCaretToEnd(textAreaJq);
-    }
+    textAreaJq.focus();
+
+    moveCaretToEnd(textAreaJq);
 }
 
 function addQuickQuote() {
@@ -98,7 +93,7 b' function addQuickQuote() {'
 
     var quoteButton = $("#quote-button");
     var postId = quoteButton.attr('data-post-id');
-    if (postId != null && getForm().prev().attr('id') != postId) {
+    if (postId != null) {
         addQuickReply(postId);
     }
 

boards/tests/test_keys.py

@@ -1,4 +1,3 b''
-from base64 import b64encode
 import logging
 
 from django.test import TestCase
@@ -38,7 +37,7 b' class KeyTest(TestCase):'
     def test_request_get(self):
         post = self._create_post_with_key()
 
-        request = GlobalId.objects.generate_request_get([post.global_id])
+        request = SyncManager.generate_request_get([post.global_id])
         logger.debug(request)
 
         key = KeyPair.objects.get(primary=True)

boards/tests/test_sync.py

@@ -3,7 +3,7 b' from django.test import TestCase'
 from boards.models import KeyPair, Post, Tag
 from boards.models.post.sync import SyncManager
 from boards.tests.mocks import MockRequest
-from boards.views.sync import response_get
+from boards.views.sync import response_get, response_list
 
 __author__ = 'neko259'
 
@@ -103,3 +103,112 b' class SyncTest(TestCase):'
                 post.version,
             ) in response,
             'Wrong response generated for the GET request.')
+
+    def test_list_all(self):
+        key = KeyPair.objects.generate_key(primary=True)
+        tag = Tag.objects.create(name='tag1')
+        post = Post.objects.create_post(title='test_title',
+                                        text='test_text\rline two',
+                                        tags=[tag])
+        post2 = Post.objects.create_post(title='test title 2',
+                                         text='test text 2',
+                                         tags=[tag])
+
+        request_all = MockRequest()
+        request_all.body = (
+            '<request type="list" version="1.0">'
+            '<model name="post" version="1.0">'
+            '</model>'
+            '</request>'
+        )
+
+        response_all = response_list(request_all).content.decode()
+        self.assertTrue(
+            '<status>success</status>'
+            '<models>'
+            '<model>'
+            '<id key="{}" local-id="{}" type="{}" />'
+            '<version>{}</version>'
+            '</model>'
+            '<model>'
+            '<id key="{}" local-id="{}" type="{}" />'
+            '<version>{}</version>'
+            '</model>'
+            '</models>'.format(
+                post.global_id.key,
+                post.global_id.local_id,
+                post.global_id.key_type,
+                post.version,
+                post2.global_id.key,
+                post2.global_id.local_id,
+                post2.global_id.key_type,
+                post2.version,
+            ) in response_all,
+            'Wrong response generated for the LIST request for all posts.')
+
+    def test_list_existing_thread(self):
+        key = KeyPair.objects.generate_key(primary=True)
+        tag = Tag.objects.create(name='tag1')
+        post = Post.objects.create_post(title='test_title',
+                                        text='test_text\rline two',
+                                        tags=[tag])
+        post2 = Post.objects.create_post(title='test title 2',
+                                         text='test text 2',
+                                         tags=[tag])
+
+        request_thread = MockRequest()
+        request_thread.body = (
+            '<request type="list" version="1.0">'
+            '<model name="post" version="1.0">'
+            '<thread>{}</thread>'
+            '</model>'
+            '</request>'.format(
+                post.id,
+            )
+        )
+
+        response_thread = response_list(request_thread).content.decode()
+        self.assertTrue(
+            '<status>success</status>'
+            '<models>'
+            '<model>'
+            '<id key="{}" local-id="{}" type="{}" />'
+            '<version>{}</version>'
+            '</model>'
+            '</models>'.format(
+                post.global_id.key,
+                post.global_id.local_id,
+                post.global_id.key_type,
+                post.version,
+            ) in response_thread,
+            'Wrong response generated for the LIST request for posts of '
+            'existing thread.')
+
+    def test_list_non_existing_thread(self):
+        key = KeyPair.objects.generate_key(primary=True)
+        tag = Tag.objects.create(name='tag1')
+        post = Post.objects.create_post(title='test_title',
+                                        text='test_text\rline two',
+                                        tags=[tag])
+        post2 = Post.objects.create_post(title='test title 2',
+                                         text='test text 2',
+                                         tags=[tag])
+
+        request_thread = MockRequest()
+        request_thread.body = (
+            '<request type="list" version="1.0">'
+            '<model name="post" version="1.0">'
+            '<thread>{}</thread>'
+            '</model>'
+            '</request>'.format(
+                0,
+            )
+        )
+
+        response_thread = response_list(request_thread).content.decode()
+        self.assertTrue(
+            '<status>success</status>'
+            '<models />'
+            in response_thread,
+            'Wrong response generated for the LIST request for posts of '
+            'non-existing thread.')

boards/views/sync.py

@@ -1,17 +1,41 b''
+import logging
+
 import xml.etree.ElementTree as et
 
 from django.http import HttpResponse, Http404
+
+from boards.abstracts.sync_filters import ThreadFilter, TAG_THREAD
 from boards.models import GlobalId, Post
 from boards.models.post.sync import SyncManager
 
 
+logger = logging.getLogger('boards.sync')
+
+
+FILTERS = {
+        TAG_THREAD: ThreadFilter,
+}
+
+
 def response_list(request):
     request_xml = request.body
 
+    filters = []
+
     if request_xml is None or len(request_xml) == 0:
         return HttpResponse(content='Use the API')
+    else:
+        root_tag = et.fromstring(request_xml)
+        model_tag = root_tag[0]
 
-    response_xml = SyncManager.generate_response_list()
+        for tag_filter in model_tag:
+            filter_name = tag_filter.tag
+            model_filter = FILTERS.get(filter_name)(tag_filter)
+            if not model_filter:
+                logger.warning('Unavailable filter: {}'.format(filter_name))
+            filters.append(model_filter)
+
+    response_xml = SyncManager.generate_response_list(filters)
 
     return HttpResponse(content=response_xml)
 

readme.markdown

@@ -33,11 +33,11 b' 3. Setup a database in `neboard/settings'
 
 Depending on configured database and search engine, you need to install corresponding dependencies manually.
 
-Default database is *sqlite*, default search engine is *simple*.
+Default database is *sqlite*. If you want to change the database backend, refer to the django documentation for the correct settings. Please note that sqlite accepts only one connection at a time, so you won't be able to run 2 servers or a server and a sync at the same time.
 
 4. Setup SECRET_KEY to a secret value in `neboard/settings.py
 5. Run `./manage.py migrate` to apply all migrations
-6. Apply config changes to `boards/config/config.ini`. You can see the default settings in `boards/config/default_config.ini`
+6. Apply config changes to `boards/config/settings.ini`. You can see the default settings in `boards/config/default_config.ini`(do not delete or overwrite it).
 7. If you want to use decetral engine, run `./manage.py generate_keypair` to generate keys
 
 # RUNNING #