diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -51,3 +51,4 @@ 19785af352684884f94566785ba1b13b3ddc5216 757b4ada4ca121db4296b13ec0df491645db8fd0 3.5.0 3da1a2d02072eec5419956265dcd8c7f47155c12 4.0.0 da8f0f9d5099ee8b22aa317b91beb06543011f1d 4.1.0 +9619ecc0f79b705c94b3ac873896809eaf7779a6 4.1.1 diff --git a/boards/abstracts/sync_filters.py b/boards/abstracts/sync_filters.py --- a/boards/abstracts/sync_filters.py +++ b/boards/abstracts/sync_filters.py @@ -1,8 +1,11 @@ import xml.etree.ElementTree as et -from boards.models import Post +from boards.models import Post, Tag TAG_THREAD = 'thread' +TAG_TAGS = 'tags' +TAG_TAG = 'tag' +TAG_TIME_FROM = 'timestamp_from' class PostFilter: @@ -29,3 +32,35 @@ class ThreadFilter(PostFilter): def add_filter(self, model_tag, value): thread_tag = et.SubElement(model_tag, TAG_THREAD) thread_tag.text = str(value) + + +class TagsFilter(PostFilter): + def filter(self, posts): + tags = [] + for tag_tag in self.content: + try: + tags.append(Tag.objects.get(name=tag_tag.text)) + except Tag.DoesNotExist: + pass + + if tags: + return posts.filter(thread__tags__in=tags) + else: + return posts.none() + + def add_filter(self, model_tag, value): + tags_tag = et.SubElement(model_tag, TAG_TAGS) + for tag_name in value: + tag_tag = et.SubElement(tags_tag, TAG_TAG) + tag_tag.text = tag_name + + +class TimestampFromFilter(PostFilter): + def filter(self, posts): + from_time = self.content.text + return posts.filter(pub_time__gt=from_time) + + def add_filter(self, model_tag, value): + tags_from_time = et.SubElement(model_tag, TAG_TIME_FROM) + tags_from_time.text = value + diff --git a/boards/config/default_settings.ini b/boards/config/default_settings.ini --- a/boards/config/default_settings.ini +++ b/boards/config/default_settings.ini @@ -1,5 +1,5 @@ [Version] -Version = 4.1.0 2017 +Version = 4.1.1 2017 SiteName = Neboard DEV [Cache] diff --git a/boards/management/commands/sync_with_server.py b/boards/management/commands/sync_with_server.py --- a/boards/management/commands/sync_with_server.py +++ b/boards/management/commands/sync_with_server.py @@ -26,6 +26,10 @@ class Command(BaseCommand): ' number of posts in one') parser.add_argument('--thread', type=int, help='Get posts of one specific thread') + parser.add_argument('--tags', type=str, + help='Get posts of the tags, comma-separated') + parser.add_argument('--time-from', type=str, + help='Get posts from the given timestamp') def handle(self, *args, **options): logger = logging.getLogger('boards.sync') @@ -57,9 +61,19 @@ class Command(BaseCommand): else: logger.info('Running LIST request...') h = httplib2.Http() + + tags = [] + tags_str = options.get('tags') + if tags_str: + tags = tags_str.split(',') + xml = SyncManager.generate_request_list( - opening_post=options.get('thread')) + opening_post=options.get('thread'), tags=tags, + timestamp_from=options.get('time_from')).encode() response, content = h.request(list_url, method="POST", body=xml) + if response.status != 200: + raise Exception('Server returned error {}'.format(response.status)) + logger.info('Processing response...') root = ET.fromstring(content) @@ -93,6 +107,8 @@ class Command(BaseCommand): logger.info('Processing response...') SyncManager.parse_response_get(content, file_url) + + logger.info('Sync completed successfully') else: logger.info('Nothing to get, everything synced') else: diff --git a/boards/models/post/sync.py b/boards/models/post/sync.py --- a/boards/models/post/sync.py +++ b/boards/models/post/sync.py @@ -3,7 +3,8 @@ import logging from xml.etree import ElementTree from boards.abstracts.exceptions import SyncException -from boards.abstracts.sync_filters import ThreadFilter +from boards.abstracts.sync_filters import ThreadFilter, TagsFilter,\ + TimestampFromFilter from boards.models import KeyPair, GlobalId, Signature, Post, Tag from boards.models.attachment.downloaders import download from boards.models.signature import TAG_REQUEST, ATTR_TYPE, TYPE_GET, \ @@ -17,6 +18,7 @@ EXCEPTION_DOWNLOAD = 'File was not downl EXCEPTION_HASH = 'File hash does not match attachment hash.' EXCEPTION_SIGNATURE = 'Invalid model signature for {}.' EXCEPTION_AUTHOR_SIGNATURE = 'Model {} has no author signature.' +EXCEPTION_THREAD = 'No thread exists for post {}' ENCODING_UNICODE = 'unicode' TAG_MODEL = 'model' @@ -193,7 +195,7 @@ class SyncManager: version = int(tag_content.find(TAG_VERSION).text) is_old = exists and global_id.post.version < version if exists and not is_old: - print('Post with same ID exists and is up to date.') + logger.debug('Post {} exists and is up to date.'.format(global_id)) else: global_id.content = content_str global_id.save() @@ -218,7 +220,7 @@ class SyncManager: if exists: opening_post = Post.objects.get(global_id=op_global_id) else: - logger.debug('No thread exists for post {}'.format(global_id)) + raise Exception(EXCEPTION_THREAD.format(global_id)) else: opening_post = None tag_tags = tag_content.find(TAG_TAGS) @@ -339,7 +341,51 @@ class SyncManager: attachment_tag.set(ATTR_ID_TYPE, ID_TYPE_URL) attachment_tag.text = attachment.url - if tag_refs is not None: + if tag_refs is not None and attachment.is_internal(): attachment_ref = et.SubElement(tag_refs, TAG_ATTACHMENT_REF) attachment_ref.set(ATTR_REF, attachment.hash) - attachment_ref.set(ATTR_URL, attachment.file.url) + attachment_ref.set(ATTR_URL, attachment.file.url) + + @staticmethod + def generate_request_get(global_id_list: list): + """ + Form a get request from a list of ModelId objects. + """ + + request = et.Element(TAG_REQUEST) + request.set(ATTR_TYPE, TYPE_GET) + request.set(ATTR_VERSION, '1.0') + + model = et.SubElement(request, TAG_MODEL) + model.set(ATTR_VERSION, '1.0') + model.set(ATTR_NAME, 'post') + + for global_id in global_id_list: + tag_id = et.SubElement(model, TAG_ID) + global_id.to_xml_element(tag_id) + + return et.tostring(request, 'unicode') + + @staticmethod + def generate_request_list(opening_post=None, tags=list(), + timestamp_from=None): + """ + Form a pull request from a list of ModelId objects. + """ + + request = et.Element(TAG_REQUEST) + request.set(ATTR_TYPE, TYPE_LIST) + request.set(ATTR_VERSION, '1.0') + + model = et.SubElement(request, TAG_MODEL) + model.set(ATTR_VERSION, '1.0') + model.set(ATTR_NAME, 'post') + + if opening_post: + ThreadFilter().add_filter(model, opening_post) + if tags: + TagsFilter().add_filter(model, tags) + if timestamp_from: + TimestampFromFilter().add_filter(model, timestamp_from) + + return et.tostring(request, 'unicode') diff --git a/boards/tests/test_sync.py b/boards/tests/test_sync.py --- a/boards/tests/test_sync.py +++ b/boards/tests/test_sync.py @@ -212,3 +212,42 @@ class SyncTest(TestCase): in response_thread, 'Wrong response generated for the LIST request for posts of ' 'non-existing thread.') + + def test_list_pub_time(self): + key = KeyPair.objects.generate_key(primary=True) + tag = Tag.objects.create(name='tag1') + post = Post.objects.create_post(title='test_title', + text='test_text\rline two', + tags=[tag]) + post2 = Post.objects.create_post(title='test title 2', + text='test text 2', + tags=[tag]) + + request_thread = MockRequest() + request_thread.body = ( + '' + '' + '{}' + '' + ''.format( + post.pub_time, + ) + ) + + response_thread = response_list(request_thread).content.decode() + self.assertTrue( + 'success' + '' + '' + '' + '{}' + '' + ''.format( + post2.global_id.key, + post2.global_id.local_id, + post2.global_id.key_type, + post2.version, + ) in response_thread, + 'Wrong response generated for the LIST request for posts of ' + 'existing thread.') + diff --git a/boards/utils.py b/boards/utils.py --- a/boards/utils.py +++ b/boards/utils.py @@ -117,7 +117,7 @@ def get_file_hash(file) -> str: def validate_file_size(size: int): max_size = boards.settings.get_int('Forms', 'MaxFileSize') - if size > max_size: + if max_size > 0 and size > max_size: raise forms.ValidationError( _('File must be less than %s but is %s.') % (filesizeformat(max_size), filesizeformat(size))) diff --git a/boards/views/landing.py b/boards/views/landing.py --- a/boards/views/landing.py +++ b/boards/views/landing.py @@ -29,10 +29,14 @@ class LandingView(BaseBoardView): Tag.objects.filter(required=True)) today = datetime.now() - timedelta(1) - max_landing_threads = settings.get_int('View', 'MaxFavoriteThreads') ops = Post.objects.filter(thread__replies__pub_time__gt=today, opening=True, thread__status=STATUS_ACTIVE)\ .annotate(today_post_count=Count('thread__replies'))\ - .order_by('-pub_time')[:max_landing_threads] + .order_by('-pub_time') + + max_landing_threads = settings.get_int('View', 'MaxFavoriteThreads') + if max_landing_threads > 0: + ops = ops[:max_landing_threads] + params[PARAM_LATEST_THREADS] = ops params[PARAM_IMAGES] = Attachment.objects.get_random_images( diff --git a/boards/views/search.py b/boards/views/search.py --- a/boards/views/search.py +++ b/boards/views/search.py @@ -1,5 +1,6 @@ from django.shortcuts import render from django.views.generic import View +from django.db.models import Q from boards.abstracts.paginator import get_paginator from boards.forms import SearchForm, PlainErrorList @@ -32,8 +33,8 @@ class BoardSearchView(View): if form.is_valid(): query = form.cleaned_data[FORM_QUERY] if len(query) >= MIN_QUERY_LENGTH: - results = Post.objects.filter(text__icontains=query)\ - .order_by('-id') + results = Post.objects.filter(Q(text__icontains=query) | + Q(title__icontains=query)).order_by('-id') paginator = get_paginator(results, RESULTS_PER_PAGE) page = int(request.GET.get(REQUEST_PAGE, '1')) diff --git a/boards/views/sync.py b/boards/views/sync.py --- a/boards/views/sync.py +++ b/boards/views/sync.py @@ -4,7 +4,9 @@ import xml.etree.ElementTree as et from django.http import HttpResponse, Http404 -from boards.abstracts.sync_filters import ThreadFilter, TAG_THREAD +from boards.abstracts.sync_filters import ThreadFilter, TagsFilter,\ + TimestampFromFilter,\ + TAG_THREAD, TAG_TAGS, TAG_TIME_FROM from boards.models import GlobalId, Post from boards.models.post.sync import SyncManager @@ -14,6 +16,8 @@ logger = logging.getLogger('boards.sync' FILTERS = { TAG_THREAD: ThreadFilter, + TAG_TAGS: TagsFilter, + TAG_TIME_FROM: TimestampFromFilter, } @@ -30,10 +34,10 @@ def response_list(request): for tag_filter in model_tag: filter_name = tag_filter.tag - model_filter = FILTERS.get(filter_name)(tag_filter) + model_filter = FILTERS.get(filter_name) if not model_filter: logger.warning('Unavailable filter: {}'.format(filter_name)) - filters.append(model_filter) + filters.append(model_filter(tag_filter)) response_xml = SyncManager.generate_response_list(filters)