# HG changeset patch # User Bohdan Horbeshko # Date 2017-01-24 22:08:44 # Node ID 1765cedca23327d793a340a31f64e4ab54ecb544 # Parent 7afa09d0bd6c32d52b3ce9c445824f980b596e79 # Parent 772d4b193ca2096565b16271211d03fc0c0b6a4a Merge tip diff --git a/boards/abstracts/sync_filters.py b/boards/abstracts/sync_filters.py new file mode 100644 --- /dev/null +++ b/boards/abstracts/sync_filters.py @@ -0,0 +1,31 @@ +import xml.etree.ElementTree as et + +from boards.models import Post + +TAG_THREAD = 'thread' + + +class PostFilter: + def __init__(self, content=None): + self.content = content + + def filter(self, posts): + return posts + + def add_filter(self, model_tag, value): + return model_tag + + +class ThreadFilter(PostFilter): + def filter(self, posts): + op_id = self.content.text + + op = Post.objects.filter(opening=True, id=op_id).first() + if op: + return posts.filter(thread=op.get_thread()) + else: + return posts.none() + + def add_filter(self, model_tag, value): + thread_tag = et.SubElement(model_tag, TAG_THREAD) + thread_tag.text = str(value) diff --git a/boards/config/default_settings.ini b/boards/config/default_settings.ini --- a/boards/config/default_settings.ini +++ b/boards/config/default_settings.ini @@ -18,6 +18,7 @@ PostingDelay = 30 Autoban = false DefaultTag = test MaxFileCount = 1 +AdditionalSpoilerSpaces = false [Messages] # Thread bumplimit diff --git a/boards/management/commands/sync_with_server.py b/boards/management/commands/sync_with_server.py --- a/boards/management/commands/sync_with_server.py +++ b/boards/management/commands/sync_with_server.py @@ -24,6 +24,8 @@ class Command(BaseCommand): parser.add_argument('--split-query', type=int, default=1, help='Split GET query into separate by the given' ' number of posts in one') + parser.add_argument('--thread', type=int, + help='Get posts of one specific thread') def handle(self, *args, **options): logger = logging.getLogger('boards.sync') @@ -45,7 +47,7 @@ class Command(BaseCommand): global_id = GlobalId(key_type=key_type, key=key, local_id=local_id) - xml = GlobalId.objects.generate_request_get([global_id]) + xml = SyncManager.generate_request_get([global_id]) h = httplib2.Http() response, content = h.request(get_url, method="POST", body=xml) @@ -55,7 +57,8 @@ class Command(BaseCommand): else: logger.info('Running LIST request...') h = httplib2.Http() - xml = GlobalId.objects.generate_request_list() + xml = SyncManager.generate_request_list( + opening_post=options.get('thread')) response, content = h.request(list_url, method="POST", body=xml) logger.info('Processing response...') @@ -83,7 +86,7 @@ class Command(BaseCommand): if len(ids_to_sync) > 0: limit = options.get('split_query', len(ids_to_sync)) for offset in range(0, len(ids_to_sync), limit): - xml = GlobalId.objects.generate_request_get(ids_to_sync[offset:offset+limit]) + xml = SyncManager.generate_request_get(ids_to_sync[offset:offset + limit]) h = httplib2.Http() logger.info('Running GET request...') response, content = h.request(get_url, method="POST", body=xml) diff --git a/boards/mdx_neboard.py b/boards/mdx_neboard.py --- a/boards/mdx_neboard.py +++ b/boards/mdx_neboard.py @@ -10,6 +10,7 @@ from django.core.exceptions import Objec from django.core.urlresolvers import reverse import boards +from boards import settings __author__ = 'neko259' @@ -24,6 +25,7 @@ LINE_BREAK_HTML = '
{}{}{}'.format(side_spaces, value, - side_spaces) + if settings.get_bool('Forms', 'AdditionalSpoilerSpaces'): + text_len = len(value) + space_count = min(random.randint(0, text_len * MAX_SPOILER_MULTIPLIER), + MAX_SPOILER_SPACE_COUNT) + side_spaces = SPOILER_SPACE * (space_count // 2) + else: + side_spaces = '' + return '{}{}{}'.format(side_spaces, + value, side_spaces) formatters = [ diff --git a/boards/migrations/0056_auto_20170123_1620.py b/boards/migrations/0056_auto_20170123_1620.py new file mode 100644 --- /dev/null +++ b/boards/migrations/0056_auto_20170123_1620.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2017-01-23 14:20 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('boards', '0055_auto_20161229_1132'), + ] + + operations = [ + migrations.AlterModelOptions( + name='attachment', + options={'ordering': ('id',)}, + ), + migrations.AlterField( + model_name='post', + name='uid', + field=models.TextField(), + ), + ] diff --git a/boards/models/attachment/downloaders.py b/boards/models/attachment/downloaders.py --- a/boards/models/attachment/downloaders.py +++ b/boards/models/attachment/downloaders.py @@ -30,15 +30,15 @@ class Downloader: return True @staticmethod - def download(url: str): + def download(url: str, validate): # Verify content headers response_head = requests.head(url, verify=False) content_type = response_head.headers[HEADER_CONTENT_TYPE].split(';')[0] - if content_type in TYPE_URL_ONLY: + if validate and content_type in TYPE_URL_ONLY: return None length_header = response_head.headers.get(HEADER_CONTENT_LENGTH) - if length_header: + if validate and length_header: length = int(length_header) validate_file_size(length) # Get the actual content into memory @@ -63,7 +63,7 @@ class Downloader: class YouTubeDownloader(Downloader): @staticmethod - def download(url: str): + def download(url: str, validate): yt = YouTube() yt.from_url(url) videos = yt.filter(YOUTUBE_VIDEO_FORMAT) @@ -82,7 +82,7 @@ class NothingDownloader(Downloader): return REGEX_MAGNET.match(url) @staticmethod - def download(url: str): + def download(url: str, validate): return None @@ -93,9 +93,9 @@ DOWNLOADERS = ( ) -def download(url): +def download(url, validate=True): for downloader in DOWNLOADERS: if downloader.handles(url): - return downloader.download(url) + return downloader.download(url, validate=validate) raise Exception('No downloader supports this URL.') diff --git a/boards/models/post/__init__.py b/boards/models/post/__init__.py --- a/boards/models/post/__init__.py +++ b/boards/models/post/__init__.py @@ -90,7 +90,7 @@ class Post(models.Model, Viewable): thread = models.ForeignKey('Thread', db_index=True, related_name='replies') url = models.TextField() - uid = models.TextField(db_index=True) + uid = models.TextField() # Global ID with author key. If the message was downloaded from another # server, this indicates the server. diff --git a/boards/models/post/sync.py b/boards/models/post/sync.py --- a/boards/models/post/sync.py +++ b/boards/models/post/sync.py @@ -1,9 +1,13 @@ import xml.etree.ElementTree as et import logging +from xml.etree import ElementTree from boards.abstracts.exceptions import SyncException +from boards.abstracts.sync_filters import ThreadFilter from boards.models import KeyPair, GlobalId, Signature, Post, Tag from boards.models.attachment.downloaders import download +from boards.models.signature import TAG_REQUEST, ATTR_TYPE, TYPE_GET, \ + ATTR_VERSION, TAG_MODEL, ATTR_NAME, TAG_ID, TYPE_LIST from boards.utils import get_file_mimetype, get_file_hash from django.db import transaction from django import forms @@ -238,7 +242,7 @@ class SyncManager: TAG_ATTACHMENT_REF, attachment.text)) url = tag_ref.get(ATTR_URL) try: - attached_file = download(hostname + url) + attached_file = download(hostname + url, validate=False) if attached_file is None: raise SyncException(EXCEPTION_DOWNLOAD) @@ -269,7 +273,7 @@ class SyncManager: logger.debug('Parsed new post {}'.format(global_id)) @staticmethod - def generate_response_list(): + def generate_response_list(filters): response = et.Element(TAG_RESPONSE) status = et.SubElement(response, TAG_STATUS) @@ -277,7 +281,11 @@ class SyncManager: models = et.SubElement(response, TAG_MODELS) - for post in Post.objects.prefetch_related('global_id').all(): + posts = Post.objects.prefetch_related('global_id') + for post_filter in filters: + posts = post_filter.filter(posts) + + for post in posts: tag_model = et.SubElement(models, TAG_MODEL) tag_id = et.SubElement(tag_model, TAG_ID) post.global_id.to_xml_element(tag_id) @@ -334,4 +342,4 @@ class SyncManager: if tag_refs is not None: attachment_ref = et.SubElement(tag_refs, TAG_ATTACHMENT_REF) attachment_ref.set(ATTR_REF, attachment.hash) - attachment_ref.set(ATTR_URL, attachment.file.url) + attachment_ref.set(ATTR_URL, attachment.file.url) diff --git a/boards/models/signature.py b/boards/models/signature.py --- a/boards/models/signature.py +++ b/boards/models/signature.py @@ -1,8 +1,9 @@ import xml.etree.ElementTree as et + from django.db import models + from boards.models import KeyPair - TAG_MODEL = 'model' TAG_REQUEST = 'request' TAG_ID = 'id' @@ -20,40 +21,6 @@ ATTR_LOCAL_ID = 'local-id' class GlobalIdManager(models.Manager): - def generate_request_get(self, global_id_list: list): - """ - Form a get request from a list of ModelId objects. - """ - - request = et.Element(TAG_REQUEST) - request.set(ATTR_TYPE, TYPE_GET) - request.set(ATTR_VERSION, '1.0') - - model = et.SubElement(request, TAG_MODEL) - model.set(ATTR_VERSION, '1.0') - model.set(ATTR_NAME, 'post') - - for global_id in global_id_list: - tag_id = et.SubElement(model, TAG_ID) - global_id.to_xml_element(tag_id) - - return et.tostring(request, 'unicode') - - def generate_request_list(self): - """ - Form a pull request from a list of ModelId objects. - """ - - request = et.Element(TAG_REQUEST) - request.set(ATTR_TYPE, TYPE_LIST) - request.set(ATTR_VERSION, '1.0') - - model = et.SubElement(request, TAG_MODEL) - model.set(ATTR_VERSION, '1.0') - model.set(ATTR_NAME, 'post') - - return et.tostring(request, 'unicode') - def global_id_exists(self, global_id): """ Checks if the same global id already exists in the system. diff --git a/boards/signals.py b/boards/signals.py --- a/boards/signals.py +++ b/boards/signals.py @@ -23,47 +23,50 @@ THUMB_SIZES = ((200, 150),) @receiver(post_save, sender=Post) def connect_replies(instance, **kwargs): - for reply_number in re.finditer(REGEX_REPLY, instance.get_raw_text()): - post_id = reply_number.group(1) + if not kwargs['update_fields']: + for reply_number in re.finditer(REGEX_REPLY, instance.get_raw_text()): + post_id = reply_number.group(1) - try: - referenced_post = Post.objects.get(id=post_id) + try: + referenced_post = Post.objects.get(id=post_id) - if not referenced_post.referenced_posts.filter( - id=instance.id).exists(): - referenced_post.referenced_posts.add(instance) - referenced_post.last_edit_time = instance.pub_time - referenced_post.build_refmap() - referenced_post.save(update_fields=['refmap', 'last_edit_time']) - except Post.DoesNotExist: - pass + if not referenced_post.referenced_posts.filter( + id=instance.id).exists(): + referenced_post.referenced_posts.add(instance) + referenced_post.last_edit_time = instance.pub_time + referenced_post.build_refmap() + referenced_post.save(update_fields=['refmap', 'last_edit_time']) + except Post.DoesNotExist: + pass @receiver(post_save, sender=Post) @receiver(post_import_deps, sender=Post) def connect_global_replies(instance, **kwargs): - for reply_number in re.finditer(REGEX_GLOBAL_REPLY, instance.get_raw_text()): - key_type = reply_number.group(1) - key = reply_number.group(2) - local_id = reply_number.group(3) + if not kwargs['update_fields']: + for reply_number in re.finditer(REGEX_GLOBAL_REPLY, instance.get_raw_text()): + key_type = reply_number.group(1) + key = reply_number.group(2) + local_id = reply_number.group(3) - try: - global_id = GlobalId.objects.get(key_type=key_type, key=key, - local_id=local_id) - referenced_post = Post.objects.get(global_id=global_id) - referenced_post.referenced_posts.add(instance) - referenced_post.last_edit_time = instance.pub_time - referenced_post.build_refmap() - referenced_post.save(update_fields=['refmap', 'last_edit_time']) - except (GlobalId.DoesNotExist, Post.DoesNotExist): - pass + try: + global_id = GlobalId.objects.get(key_type=key_type, key=key, + local_id=local_id) + referenced_post = Post.objects.get(global_id=global_id) + referenced_post.referenced_posts.add(instance) + referenced_post.last_edit_time = instance.pub_time + referenced_post.build_refmap() + referenced_post.save(update_fields=['refmap', 'last_edit_time']) + except (GlobalId.DoesNotExist, Post.DoesNotExist): + pass @receiver(post_save, sender=Post) def connect_notifications(instance, **kwargs): - for reply_number in re.finditer(REGEX_NOTIFICATION, instance.get_raw_text()): - user_name = reply_number.group(1).lower() - Notification.objects.get_or_create(name=user_name, post=instance) + if not kwargs['update_fields']: + for reply_number in re.finditer(REGEX_NOTIFICATION, instance.get_raw_text()): + user_name = reply_number.group(1).lower() + Notification.objects.get_or_create(name=user_name, post=instance) @receiver(pre_save, sender=Post) diff --git a/boards/static/js/thread.js b/boards/static/js/thread.js --- a/boards/static/js/thread.js +++ b/boards/static/js/thread.js @@ -57,40 +57,35 @@ function showFormAfter(blockToInsertAfte } function addQuickReply(postId) { - // If we click "reply" on the same post, it means "cancel" - if (getForm().prev().attr('id') == postId) { - resetFormPosition(); - } else { - var blockToInsert = null; - var textAreaJq = getPostTextarea(); - var postLinkRaw = '[post]' + postId + '[/post]' - var textToAdd = ''; + var blockToInsert = null; + var textAreaJq = getPostTextarea(); + var postLinkRaw = '[post]' + postId + '[/post]' + var textToAdd = ''; - if (postId != null) { - var post = $('#' + postId); + if (postId != null) { + var post = $('#' + postId); - // If this is not OP, add reflink to the post. If there already is - // the same reflink, don't add it again. - var postText = textAreaJq.val(); - if (!post.is(':first-child') && postText.indexOf(postLinkRaw) < 0) { - // Insert line break if none is present. - if (postText.length > 0 && !postText.endsWith('\n') && !postText.endsWith('\r')) { - textToAdd += '\n'; - } - textToAdd += postLinkRaw + '\n'; + // If this is not OP, add reflink to the post. If there already is + // the same reflink, don't add it again. + var postText = textAreaJq.val(); + if (!post.is(':first-child') && postText.indexOf(postLinkRaw) < 0) { + // Insert line break if none is present. + if (postText.length > 0 && !postText.endsWith('\n') && !postText.endsWith('\r')) { + textToAdd += '\n'; } + textToAdd += postLinkRaw + '\n'; + } - textAreaJq.val(textAreaJq.val()+ textToAdd); - blockToInsert = post; - } else { - blockToInsert = $('.thread'); - } - showFormAfter(blockToInsert); - - textAreaJq.focus(); + textAreaJq.val(textAreaJq.val()+ textToAdd); + blockToInsert = post; + } else { + blockToInsert = $('.thread'); + } + showFormAfter(blockToInsert); - moveCaretToEnd(textAreaJq); - } + textAreaJq.focus(); + + moveCaretToEnd(textAreaJq); } function addQuickQuote() { @@ -98,7 +93,7 @@ function addQuickQuote() { var quoteButton = $("#quote-button"); var postId = quoteButton.attr('data-post-id'); - if (postId != null && getForm().prev().attr('id') != postId) { + if (postId != null) { addQuickReply(postId); } diff --git a/boards/tests/test_keys.py b/boards/tests/test_keys.py --- a/boards/tests/test_keys.py +++ b/boards/tests/test_keys.py @@ -1,4 +1,3 @@ -from base64 import b64encode import logging from django.test import TestCase @@ -38,7 +37,7 @@ class KeyTest(TestCase): def test_request_get(self): post = self._create_post_with_key() - request = GlobalId.objects.generate_request_get([post.global_id]) + request = SyncManager.generate_request_get([post.global_id]) logger.debug(request) key = KeyPair.objects.get(primary=True) diff --git a/boards/tests/test_sync.py b/boards/tests/test_sync.py --- a/boards/tests/test_sync.py +++ b/boards/tests/test_sync.py @@ -3,7 +3,7 @@ from django.test import TestCase from boards.models import KeyPair, Post, Tag from boards.models.post.sync import SyncManager from boards.tests.mocks import MockRequest -from boards.views.sync import response_get +from boards.views.sync import response_get, response_list __author__ = 'neko259' @@ -103,3 +103,112 @@ class SyncTest(TestCase): post.version, ) in response, 'Wrong response generated for the GET request.') + + def test_list_all(self): + key = KeyPair.objects.generate_key(primary=True) + tag = Tag.objects.create(name='tag1') + post = Post.objects.create_post(title='test_title', + text='test_text\rline two', + tags=[tag]) + post2 = Post.objects.create_post(title='test title 2', + text='test text 2', + tags=[tag]) + + request_all = MockRequest() + request_all.body = ( + '' + '' + '' + '' + ) + + response_all = response_list(request_all).content.decode() + self.assertTrue( + 'success' + '' + '' + '' + '{}' + '' + '' + '' + '{}' + '' + ''.format( + post.global_id.key, + post.global_id.local_id, + post.global_id.key_type, + post.version, + post2.global_id.key, + post2.global_id.local_id, + post2.global_id.key_type, + post2.version, + ) in response_all, + 'Wrong response generated for the LIST request for all posts.') + + def test_list_existing_thread(self): + key = KeyPair.objects.generate_key(primary=True) + tag = Tag.objects.create(name='tag1') + post = Post.objects.create_post(title='test_title', + text='test_text\rline two', + tags=[tag]) + post2 = Post.objects.create_post(title='test title 2', + text='test text 2', + tags=[tag]) + + request_thread = MockRequest() + request_thread.body = ( + '' + '' + '{}' + '' + ''.format( + post.id, + ) + ) + + response_thread = response_list(request_thread).content.decode() + self.assertTrue( + 'success' + '' + '' + '' + '{}' + '' + ''.format( + post.global_id.key, + post.global_id.local_id, + post.global_id.key_type, + post.version, + ) in response_thread, + 'Wrong response generated for the LIST request for posts of ' + 'existing thread.') + + def test_list_non_existing_thread(self): + key = KeyPair.objects.generate_key(primary=True) + tag = Tag.objects.create(name='tag1') + post = Post.objects.create_post(title='test_title', + text='test_text\rline two', + tags=[tag]) + post2 = Post.objects.create_post(title='test title 2', + text='test text 2', + tags=[tag]) + + request_thread = MockRequest() + request_thread.body = ( + '' + '' + '{}' + '' + ''.format( + 0, + ) + ) + + response_thread = response_list(request_thread).content.decode() + self.assertTrue( + 'success' + '' + in response_thread, + 'Wrong response generated for the LIST request for posts of ' + 'non-existing thread.') diff --git a/boards/views/sync.py b/boards/views/sync.py --- a/boards/views/sync.py +++ b/boards/views/sync.py @@ -1,17 +1,41 @@ +import logging + import xml.etree.ElementTree as et from django.http import HttpResponse, Http404 + +from boards.abstracts.sync_filters import ThreadFilter, TAG_THREAD from boards.models import GlobalId, Post from boards.models.post.sync import SyncManager +logger = logging.getLogger('boards.sync') + + +FILTERS = { + TAG_THREAD: ThreadFilter, +} + + def response_list(request): request_xml = request.body + filters = [] + if request_xml is None or len(request_xml) == 0: return HttpResponse(content='Use the API') + else: + root_tag = et.fromstring(request_xml) + model_tag = root_tag[0] - response_xml = SyncManager.generate_response_list() + for tag_filter in model_tag: + filter_name = tag_filter.tag + model_filter = FILTERS.get(filter_name)(tag_filter) + if not model_filter: + logger.warning('Unavailable filter: {}'.format(filter_name)) + filters.append(model_filter) + + response_xml = SyncManager.generate_response_list(filters) return HttpResponse(content=response_xml) diff --git a/readme.markdown b/readme.markdown --- a/readme.markdown +++ b/readme.markdown @@ -33,11 +33,11 @@ 3. Setup a database in `neboard/settings Depending on configured database and search engine, you need to install corresponding dependencies manually. -Default database is *sqlite*, default search engine is *simple*. +Default database is *sqlite*. If you want to change the database backend, refer to the django documentation for the correct settings. Please note that sqlite accepts only one connection at a time, so you won't be able to run 2 servers or a server and a sync at the same time. 4. Setup SECRET_KEY to a secret value in `neboard/settings.py 5. Run `./manage.py migrate` to apply all migrations -6. Apply config changes to `boards/config/config.ini`. You can see the default settings in `boards/config/default_config.ini` +6. Apply config changes to `boards/config/settings.ini`. You can see the default settings in `boards/config/default_config.ini`(do not delete or overwrite it). 7. If you want to use decetral engine, run `./manage.py generate_keypair` to generate keys # RUNNING #