New backend for fav threads. Now only last post ids are saved, no thread ids

boards/abstracts/settingsmanager.py

@@ -1,240 +1,246
-from boards import settings
+import boards
 from boards.models import Tag, TagAlias, Attachment
 from boards.models.attachment import AttachmentSticker
-from boards.models.thread import FAV_THREAD_NO_UPDATES
 from boards.models.user import UserSettings
-from boards.settings import SECTION_VIEW
 
 MAX_TRIPCODE_COLLISIONS = 50
 
 __author__ = 'neko259'
 
 SESSION_SETTING = 'setting'
 
 SETTING_THEME = 'theme'
 SETTING_FAVORITE_TAGS = 'favorite_tags'
 SETTING_FAVORITE_THREADS = 'favorite_threads'
 SETTING_HIDDEN_TAGS = 'hidden_tags'
 SETTING_USERNAME = 'username'
 SETTING_LAST_NOTIFICATION_ID = 'last_notification'
 SETTING_IMAGE_VIEWER = 'image_viewer'
 SETTING_IMAGES = 'images_aliases'
 SETTING_ONLY_FAVORITES = 'only_favorites'
+SETTING_LAST_POSTS = 'last_posts'
 
 DEFAULT_THEME = 'md'
 
 
 class SettingsManager:
     """
     Base settings manager class. get_setting and set_setting methods should
     be overriden.
     """
     def __init__(self):
         pass
 
     def get_theme(self) -> str:
         theme = self.get_setting(SETTING_THEME)
         if not theme:
             theme = DEFAULT_THEME
             self.set_setting(SETTING_THEME, theme)
 
         return theme
 
     def set_theme(self, theme):
         self.set_setting(SETTING_THEME, theme)
 
     def get_setting(self, setting, default=None):
         pass
 
     def set_setting(self, setting, value):
         pass
 
     def get_fav_tags(self) -> list:
         tag_names = self.get_setting(SETTING_FAVORITE_TAGS)
         tags = []
         if tag_names:
             tags = list(Tag.objects.filter(aliases__in=TagAlias.objects
                 .filter_localized(parent__aliases__name__in=tag_names))
                 .order_by('aliases__name'))
         return tags
 
     def add_fav_tag(self, tag):
         tags = self.get_setting(SETTING_FAVORITE_TAGS)
         if not tags:
             tags = [tag.get_name()]
         else:
             if not tag.get_name() in tags:
                 tags.append(tag.get_name())
 
         tags.sort()
         self.set_setting(SETTING_FAVORITE_TAGS, tags)
 
     def del_fav_tag(self, tag):
         tags = self.get_setting(SETTING_FAVORITE_TAGS)
         if tag.get_name() in tags:
             tags.remove(tag.get_name())
             self.set_setting(SETTING_FAVORITE_TAGS, tags)
 
     def get_hidden_tags(self) -> list:
         tag_names = self.get_setting(SETTING_HIDDEN_TAGS)
         tags = []
         if tag_names:
             tags = list(Tag.objects.filter(aliases__in=TagAlias.objects
                 .filter_localized(parent__aliases__name__in=tag_names))
                 .order_by('aliases__name'))
 
         return tags
 
     def add_hidden_tag(self, tag):
         tags = self.get_setting(SETTING_HIDDEN_TAGS)
         if not tags:
             tags = [tag.get_name()]
         else:
             if not tag.get_name() in tags:
                 tags.append(tag.get_name())
 
         tags.sort()
         self.set_setting(SETTING_HIDDEN_TAGS, tags)
 
     def del_hidden_tag(self, tag):
         tags = self.get_setting(SETTING_HIDDEN_TAGS)
         if tag.get_name() in tags:
             tags.remove(tag.get_name())
             self.set_setting(SETTING_HIDDEN_TAGS, tags)
 
-    def get_fav_threads(self) -> dict:
-        return self.get_setting(SETTING_FAVORITE_THREADS, default=dict())
-
     def add_or_read_fav_thread(self, opening_post):
-        threads = self.get_fav_threads()
+        last_post_ids = self.get_setting(SETTING_LAST_POSTS)
+        if not last_post_ids:
+            last_post_ids = []
 
-        max_fav_threads = settings.get_int(SECTION_VIEW, 'MaxFavoriteThreads')
-        if (str(opening_post.id) in threads) or (len(threads) < max_fav_threads):
-            thread = opening_post.get_thread()
-            # Don't check for new posts if the thread is archived already
-            if thread.is_archived():
-                last_id = FAV_THREAD_NO_UPDATES
-            else:
-                last_id = thread.get_replies().last().id
-            threads[str(opening_post.id)] = last_id
-            self.set_setting(SETTING_FAVORITE_THREADS, threads)
+        self.del_fav_thread(opening_post)
+        
+        last_post_id = opening_post.get_thread().get_replies().last().id
+        last_post_ids.append(last_post_id)
+
+        self.set_setting(SETTING_LAST_POSTS, last_post_ids)
 
     def del_fav_thread(self, opening_post):
-        threads = self.get_fav_threads()
-        if self.thread_is_fav(opening_post):
-            del threads[str(opening_post.id)]
-            self.set_setting(SETTING_FAVORITE_THREADS, threads)
+        last_posts_ids = self.get_setting(SETTING_LAST_POSTS)
+
+        for post in self.get_last_posts():
+            if post.get_thread() == opening_post.get_thread():
+                last_posts_ids.remove(post.id)
+
+        self.set_setting(SETTING_LAST_POSTS, last_posts_ids)
 
     def thread_is_fav(self, opening_post):
-        return str(opening_post.id) in self.get_fav_threads()
+        for post in self.get_last_posts():
+            if post.get_thread() == opening_post.get_thread():
+                return True
+        return False
 
     def get_notification_usernames(self):
         names = set()
         name_list = self.get_setting(SETTING_USERNAME)
         if name_list is not None:
             name_list = name_list.strip()
             if len(name_list) > 0:
                 names = name_list.lower().split(',')
                 names = set(name.strip() for name in names)
         return names
 
     def get_attachment_by_alias(self, alias):
         images = self.get_setting(SETTING_IMAGES)
         if images and alias in images:
             try:
                 return Attachment.objects.get(id=images.get(alias))
             except Attachment.DoesNotExist:
                 self.remove_attachment_alias(alias)
 
     def add_attachment_alias(self, alias, attachment):
         images = self.get_setting(SETTING_IMAGES)
         if images is None:
             images = dict()
         images[alias] = attachment.id
         self.set_setting(SETTING_IMAGES, images)
 
     def remove_attachment_alias(self, alias):
         images = self.get_setting(SETTING_IMAGES)
         del images[alias]
         self.set_setting(SETTING_IMAGES, images)
 
     def get_stickers(self):
         images = self.get_setting(SETTING_IMAGES)
         stickers = []
         if images:
             for key, value in images.items():
                 try:
                     attachment = Attachment.objects.get(id=value)
                     stickers.append(AttachmentSticker(name=key, attachment=attachment))
                 except Attachment.DoesNotExist:
                     self.remove_attachment_alias(key)
         return stickers
 
     def tag_is_fav(self, tag):
         fav_tag_names = self.get_setting(SETTING_FAVORITE_TAGS)
         return fav_tag_names is not None and tag.get_name() in fav_tag_names
 
     def tag_is_hidden(self, tag):
         hidden_tag_names = self.get_setting(SETTING_HIDDEN_TAGS)
         return hidden_tag_names is not None and tag.get_name() in hidden_tag_names
 
+    def get_last_posts(self):
+        post_ids = self.get_setting(SETTING_LAST_POSTS) or []
+        return [boards.models.Post.objects.get(id=post_id) for post_id in post_ids]
+
 
 class SessionSettingsManager(SettingsManager):
     """
     Session-based settings manager. All settings are saved to the user's
     session.
     """
     def __init__(self, session):
         SettingsManager.__init__(self)
         self.session = session
 
     def get_setting(self, setting, default=None):
         if setting in self.session:
             return self.session[setting]
         else:
             self.set_setting(setting, default)
             return default
 
     def set_setting(self, setting, value):
         self.session[setting] = value
 
 
 class DatabaseSettingsManager(SessionSettingsManager):
     def __init__(self, session):
         super().__init__(session)
+        if not session.session_key:
+            session.save()
         self.settings, created = UserSettings.objects.get_or_create(session_key=session.session_key)
 
     def add_fav_tag(self, tag):
         self.settings.fav_tags.add(tag)
 
     def del_fav_tag(self, tag):
         self.settings.fav_tags.remove(tag)
 
     def get_fav_tags(self) -> list:
         return self.settings.fav_tags.filter(
             aliases__in=TagAlias.objects.filter_localized())\
             .order_by('aliases__name')
 
     def get_hidden_tags(self) -> list:
         return self.settings.hidden_tags.all()
 
     def add_hidden_tag(self, tag):
         self.settings.hidden_tags.add(tag)
         
     def del_hidden_tag(self, tag):
         self.settings.hidden_tags.remove(tag)
 
     def tag_is_fav(self, tag):
         return self.settings.fav_tags.filter(id=tag.id).exists()
 
     def tag_is_hidden(self, tag):
         return self.settings.hidden_tags.filter(id=tag.id).exists()
 
 
 def get_settings_manager(request) -> SettingsManager:
     """
     Get settings manager based on the request object. Currently only
     session-based manager is supported. In the future, cookie-based or
     database-based managers could be implemented.
     """
     return DatabaseSettingsManager(request.session)

boards/context_processors.py

@@ -1,87 +1,83
 from boards.abstracts.settingsmanager import get_settings_manager, \
     SETTING_LAST_NOTIFICATION_ID, SETTING_IMAGE_VIEWER, SETTING_ONLY_FAVORITES
 from boards.models import Banner
 from boards.models.user import Notification
 from boards import settings
 from boards.models import Post, Tag, Thread
 from boards.settings import SECTION_FORMS, SECTION_VIEW, SECTION_VERSION
 
 THEME_CSS = 'css/{}/base_page.css'
 
 CONTEXT_SITE_NAME = 'site_name'
 CONTEXT_VERSION = 'version'
 CONTEXT_THEME_CSS = 'theme_css'
 CONTEXT_THEME = 'theme'
 CONTEXT_PPD = 'posts_per_day'
 CONTEXT_USER = 'user'
 CONTEXT_NEW_NOTIFICATIONS_COUNT = 'new_notifications_count'
 CONTEXT_USERNAMES = 'usernames'
 CONTEXT_TAGS_STR = 'tags_str'
 CONTEXT_IMAGE_VIEWER = 'image_viewer'
 CONTEXT_HAS_FAV_THREADS = 'has_fav_threads'
 CONTEXT_POW_DIFFICULTY = 'pow_difficulty'
 CONTEXT_NEW_POST_COUNT = 'new_post_count'
 CONTEXT_BANNERS = 'banners'
 CONTEXT_ONLY_FAVORITES = 'only_favorites'
 
 
 def get_notifications(context, settings_manager):
     usernames = settings_manager.get_notification_usernames()
     new_notifications_count = 0
     if usernames:
         last_notification_id = settings_manager.get_setting(
             SETTING_LAST_NOTIFICATION_ID)
 
         new_notifications_count = Notification.objects.get_notification_posts(
                 usernames=usernames, last=last_notification_id).only('id').count()
     context[CONTEXT_NEW_NOTIFICATIONS_COUNT] = new_notifications_count
     context[CONTEXT_USERNAMES] = usernames
 
 
 def get_new_post_count(context, settings_manager):
-    fav_threads = settings_manager.get_fav_threads()
-    if fav_threads:
-        fav_thread_ops = Post.objects.filter(id__in=fav_threads.keys()) \
-            .order_by('-pub_time').only('thread_id', 'pub_time')
-        ops = [{'op': op, 'last_id': fav_threads[str(op.id)]} for op in fav_thread_ops]
-        count = Thread.objects.get_new_post_count(ops)
-        if count > 0:
-            context[CONTEXT_NEW_POST_COUNT] = '(+{})'.format(count)
+    last_posts = settings_manager.get_last_posts()
+    count = Thread.objects.get_new_post_count(last_posts)
+    if count > 0:
+        context[CONTEXT_NEW_POST_COUNT] = '(+{})'.format(count)
 
 
 def user_and_ui_processor(request):
     context = dict()
 
     context[CONTEXT_PPD] = float(Post.objects.get_posts_per_day())
 
     settings_manager = get_settings_manager(request)
     fav_tags = settings_manager.get_fav_tags()
 
     context[CONTEXT_TAGS_STR] = Tag.objects.get_tag_url_list(fav_tags)
     theme = settings_manager.get_theme()
     context[CONTEXT_THEME] = theme
 
     # TODO Use static here
     context[CONTEXT_THEME_CSS] = THEME_CSS.format(theme)
 
     context[CONTEXT_VERSION] = settings.get(SECTION_VERSION, 'Version')
     context[CONTEXT_SITE_NAME] = settings.get(SECTION_VERSION, 'SiteName')
 
     if settings.get_bool(SECTION_FORMS, 'LimitFirstPosting'):
         context[CONTEXT_POW_DIFFICULTY] = settings.get_int(SECTION_FORMS, 'PowDifficulty')
 
     context[CONTEXT_IMAGE_VIEWER] = settings_manager.get_setting(
         SETTING_IMAGE_VIEWER,
         default=settings.get(SECTION_VIEW, 'DefaultImageViewer'))
 
     context[CONTEXT_HAS_FAV_THREADS] =\
-        len(settings_manager.get_fav_threads()) > 0
+        len(settings_manager.get_last_posts()) > 0
 
     context[CONTEXT_BANNERS] = Banner.objects.order_by('-id')
     context[CONTEXT_ONLY_FAVORITES] = settings_manager.get_setting(
             SETTING_ONLY_FAVORITES, default=False)
 
     get_notifications(context, settings_manager)
     get_new_post_count(context, settings_manager)
 
     return context

boards/models/thread.py

@@ -1,326 +1,324
 import logging
 from datetime import timedelta
 
 from django.db import models, transaction
 from django.db.models import Count, Sum, QuerySet, Q
 from django.utils import timezone
 
 import boards
 from boards import settings
 from boards.models import STATUS_BUMPLIMIT, STATUS_ACTIVE, STATUS_ARCHIVE
 from boards.models.attachment import FILE_TYPES_IMAGE
 from boards.models.post import Post
 from boards.models.tag import Tag, TagAlias
 from boards.settings import SECTION_VIEW
 from boards.utils import cached_result, datetime_to_epoch
 
 FAV_THREAD_NO_UPDATES = -1
 
 
 __author__ = 'neko259'
 
 
 logger = logging.getLogger(__name__)
 
 
 WS_NOTIFICATION_TYPE_NEW_POST = 'new_post'
 WS_NOTIFICATION_TYPE = 'notification_type'
 
 WS_CHANNEL_THREAD = "thread:"
 
 STATUS_CHOICES = (
         (STATUS_ACTIVE, STATUS_ACTIVE),
         (STATUS_BUMPLIMIT, STATUS_BUMPLIMIT),
         (STATUS_ARCHIVE, STATUS_ARCHIVE),
 )
 
 
 class ThreadManager(models.Manager):
     def process_old_threads(self):
         """
          Preserves maximum thread count. If there are too many threads,
          archive or delete the old ones.
          """
         old_time_delta = settings.get_int('Messages', 'ThreadArchiveDays')
         old_time = timezone.now() - timedelta(days=old_time_delta)
         old_ops = Post.objects.filter(opening=True, pub_time__lte=old_time).exclude(thread__status=STATUS_ARCHIVE)
 
         for op in old_ops:
             thread = op.get_thread()
             if settings.get_bool('Storage', 'ArchiveThreads'):
                 self._archive_thread(thread)
             else:
                 thread.delete()
             logger.info('Processed old thread {}'.format(thread))
 
 
     def _archive_thread(self, thread):
         thread.status = STATUS_ARCHIVE
         thread.last_edit_time = timezone.now()
         thread.update_posts_time()
         thread.save(update_fields=['last_edit_time', 'status'])
 
-    def get_new_posts(self, datas):
+    def get_new_posts(self, last_posts):
         query = None
-        # TODO Use classes instead of dicts
-        for data in datas:
-            if data['last_id'] != FAV_THREAD_NO_UPDATES:
-                q = (Q(id=data['op'].get_thread_id())
-                     & Q(replies__id__gt=data['last_id']))
+        for post in last_posts:
+            if not post.get_thread().is_archived():
+                q = Q(id=post.thread_id) & Q(replies__id__gt=post.id)
                 if query is None:
                     query = q
                 else:
                     query = query | q
         if query is not None:
             return self.filter(query).annotate(
                 new_post_count=Count('replies'))
 
-    def get_new_post_count(self, datas):
-        new_posts = self.get_new_posts(datas)
+    def get_new_post_count(self, last_posts):
+        new_posts = self.get_new_posts(last_posts)
         return new_posts.aggregate(total_count=Count('replies'))\
                 ['total_count'] if new_posts else 0
 
 
 def get_thread_max_posts():
     return settings.get_int('Messages', 'MaxPostsPerThread')
 
 
 class Thread(models.Model):
     objects = ThreadManager()
 
     class Meta:
         app_label = 'boards'
 
     tags = models.ManyToManyField('Tag', related_name='thread_tags')
     bump_time = models.DateTimeField(db_index=True)
     last_edit_time = models.DateTimeField()
     max_posts = models.IntegerField(default=get_thread_max_posts)
     status = models.CharField(max_length=50, default=STATUS_ACTIVE,
             choices=STATUS_CHOICES, db_index=True)
     monochrome = models.BooleanField(default=False)
     stickerpack = models.BooleanField(default=False)
 
     def get_tags(self) -> QuerySet:
         """
         Gets a sorted tag list.
         """
 
         return self.tags.filter(aliases__in=TagAlias.objects.filter_localized(parent__thread_tags=self)).order_by('aliases__name')
 
     def bump(self):
         """
         Bumps (moves to up) thread if possible.
         """
 
         if self.can_bump():
             self.bump_time = self.last_edit_time
 
             self.update_bump_status()
 
             logger.info('Bumped thread %d' % self.id)
 
     def has_post_limit(self) -> bool:
         return self.max_posts > 0
 
     def update_bump_status(self, exclude_posts=None):
         if self.has_post_limit() and self.get_reply_count() >= self.max_posts:
             self.status = STATUS_BUMPLIMIT
             self.update_posts_time(exclude_posts=exclude_posts)
 
     def _get_cache_key(self):
         return [datetime_to_epoch(self.last_edit_time)]
 
     @cached_result(key_method=_get_cache_key)
     def get_reply_count(self) -> int:
         return self.get_replies().count()
 
     @cached_result(key_method=_get_cache_key)
     def get_images_count(self) -> int:
         return self.get_replies().filter(
             attachments__mimetype__in=FILE_TYPES_IMAGE)\
             .annotate(images_count=Count(
             'attachments')).aggregate(Sum('images_count'))['images_count__sum'] or 0
 
     @cached_result(key_method=_get_cache_key)
     def get_attachment_count(self) -> int:
         return self.get_replies().annotate(attachment_count=Count('attachments'))\
                 .aggregate(Sum('attachment_count'))['attachment_count__sum'] or 0
 
     def can_bump(self) -> bool:
         """
         Checks if the thread can be bumped by replying to it.
         """
 
         return self.get_status() == STATUS_ACTIVE
 
     def get_last_replies(self) -> QuerySet:
         """
         Gets several last replies, not including opening post
         """
 
         last_replies_count = settings.get_int(SECTION_VIEW, 'LastRepliesCount')
 
         if last_replies_count > 0:
             reply_count = self.get_reply_count()
 
             if reply_count > 0:
                 reply_count_to_show = min(last_replies_count,
                                           reply_count - 1)
                 replies = self.get_replies()
                 last_replies = replies[reply_count - reply_count_to_show:]
 
                 return last_replies
 
     def get_skipped_replies_count(self) -> int:
         """
         Gets number of posts between opening post and last replies.
         """
         reply_count = self.get_reply_count()
         last_replies_count = min(settings.get_int(SECTION_VIEW, 'LastRepliesCount'),
                                  reply_count - 1)
         return reply_count - last_replies_count - 1
 
     # TODO Remove argument, it is not used
     def get_replies(self, view_fields_only=True) -> QuerySet:
         """
         Gets sorted thread posts
         """
         query = self.replies.order_by('pub_time').prefetch_related(
             'attachments')
         return query
 
     def get_viewable_replies(self) -> QuerySet:
         """
         Gets replies with only fields that are used for viewing.
         """
         return self.get_replies().defer('text', 'last_edit_time')
 
     def get_top_level_replies(self) -> QuerySet:
         return self.get_replies().exclude(refposts__threads__in=[self])
 
     def get_replies_with_images(self, view_fields_only=False) -> QuerySet:
         """
         Gets replies that have at least one image attached
         """
         return self.get_replies(view_fields_only).filter(
             attachments__mimetype__in=FILE_TYPES_IMAGE).annotate(images_count=Count(
             'attachments')).filter(images_count__gt=0)
 
     def get_opening_post(self, only_id=False) -> Post:
         """
         Gets the first post of the thread
         """
 
         query = self.get_replies().filter(opening=True)
         if only_id:
             query = query.only('id')
         opening_post = query.first()
 
         return opening_post
 
     @cached_result()
     def get_opening_post_id(self) -> int:
         """
         Gets ID of the first thread post.
         """
 
         return self.get_opening_post(only_id=True).id
 
     def get_pub_time(self):
         """
         Gets opening post's pub time because thread does not have its own one.
         """
 
         return self.get_opening_post().pub_time
 
     def __str__(self):
         return 'T#{}/{}'.format(self.id, self.get_opening_post())
 
     def get_tag_url_list(self) -> list:
         return boards.models.Tag.objects.get_tag_url_list(self.get_tags().all())
 
     def update_posts_time(self, exclude_posts=None):
         last_edit_time = self.last_edit_time
 
         for post in self.replies.all():
             if exclude_posts is None or post not in exclude_posts:
                 # Manual update is required because uids are generated on save
                 post.last_edit_time = last_edit_time
                 post.save(update_fields=['last_edit_time'])
 
     def get_absolute_url(self):
         return self.get_opening_post().get_absolute_url()
 
     def get_required_tags(self):
         return self.get_tags().filter(required=True)
 
     def get_sections_str(self):
         return Tag.objects.get_tag_url_list(self.get_required_tags())
 
     def get_replies_newer(self, post_id):
         return self.get_replies().filter(id__gt=post_id)
 
     def is_archived(self):
         return self.get_status() == STATUS_ARCHIVE
 
     def is_bumplimit(self):
         return self.get_status() == STATUS_BUMPLIMIT
 
     def get_status(self):
         return self.status
 
     def is_monochrome(self):
         return self.monochrome
 
     def is_stickerpack(self):
         return self.stickerpack
 
     # If tags have parent, add them to the tag list
     @transaction.atomic
     def refresh_tags(self):
         for tag in self.get_tags().all():
             parents = tag.get_all_parents()
             if len(parents) > 0:
                 self.tags.add(*parents)
 
     def get_reply_tree(self):
         replies = self.get_replies().prefetch_related('refposts')
         tree = []
         for reply in replies:
             parents = reply.refposts.all()
 
             found_parent = False
             searching_for_index = False
 
             if len(parents) > 0:
                 index = 0
                 parent_depth = 0
 
                 indexes_to_insert = []
 
                 for depth, element in tree:
                     index += 1
 
                     # If this element is next after parent on the same level,
                     # insert child before it
                     if searching_for_index and depth <= parent_depth:
                         indexes_to_insert.append((index - 1, parent_depth))
                         searching_for_index = False
 
                     if element in parents:
                         found_parent = True
                         searching_for_index = True
                         parent_depth = depth
 
             if not found_parent:
                 tree.append((0, reply))
             else:
                 if searching_for_index:
                     tree.append((parent_depth + 1, reply))
 
                 offset = 0
                 for last_index, parent_depth in indexes_to_insert:
                     tree.insert(last_index + offset, (parent_depth + 1, reply))
                     offset += 1
 
         return tree

boards/views/api.py

@@ -1,331 +1,329
 import json
 import logging
 
 from django.core import serializers
 from django.db import transaction
 from django.db.models import Q
 from django.http import HttpResponse, HttpResponseBadRequest
 from django.shortcuts import get_object_or_404
 from django.views.decorators.csrf import csrf_protect
 
 from boards.abstracts.settingsmanager import get_settings_manager
 from boards.forms import PostForm, PlainErrorList, ThreadForm
 from boards.mdx_neboard import Parser
 from boards.models import Post, Thread, Tag, TagAlias
 from boards.models.attachment import AttachmentSticker
 from boards.models.thread import STATUS_ARCHIVE
 from boards.models.user import Notification
 from boards.utils import datetime_to_epoch
 
 __author__ = 'neko259'
 
 PARAMETER_TRUNCATED = 'truncated'
 PARAMETER_TAG = 'tag'
 PARAMETER_OFFSET = 'offset'
 PARAMETER_DIFF_TYPE = 'type'
 PARAMETER_POST = 'post'
 PARAMETER_UPDATED = 'updated'
 PARAMETER_LAST_UPDATE = 'last_update'
 PARAMETER_THREAD = 'thread'
 PARAMETER_UIDS = 'uids'
 PARAMETER_SUBSCRIBED = 'subscribed'
 
 DIFF_TYPE_HTML = 'html'
 DIFF_TYPE_JSON = 'json'
 
 STATUS_OK = 'ok'
 STATUS_ERROR = 'error'
 
 logger = logging.getLogger(__name__)
 
 
 @transaction.atomic
 def api_get_threaddiff(request):
     """
     Gets posts that were changed or added since time
     """
 
     thread_id = request.POST.get(PARAMETER_THREAD)
     uids_str = request.POST.get(PARAMETER_UIDS)
 
     if not thread_id or not uids_str:
         return HttpResponse(content='Invalid request.')
 
     uids = uids_str.strip().split(' ')
 
     opening_post = get_object_or_404(Post, id=thread_id)
     thread = opening_post.get_thread()
 
     json_data = {
         PARAMETER_UPDATED: [],
         PARAMETER_LAST_UPDATE: None,  # TODO Maybe this can be removed already?
     }
     posts = Post.objects.filter(thread=thread).exclude(uid__in=uids)
 
     diff_type = request.GET.get(PARAMETER_DIFF_TYPE, DIFF_TYPE_HTML)
 
     for post in posts:
         json_data[PARAMETER_UPDATED].append(post.get_post_data(
             format_type=diff_type, request=request))
     json_data[PARAMETER_LAST_UPDATE] = str(thread.last_edit_time)
 
     settings_manager = get_settings_manager(request)
     json_data[PARAMETER_SUBSCRIBED] = str(settings_manager.thread_is_fav(opening_post))
 
     # If the tag is favorite, update the counter
     settings_manager = get_settings_manager(request)
     favorite = settings_manager.thread_is_fav(opening_post)
     if favorite:
         settings_manager.add_or_read_fav_thread(opening_post)
 
     return HttpResponse(content=json.dumps(json_data))
 
 
 @csrf_protect
 def api_add_post(request, opening_post_id=None):
     """
     Adds a post and return the JSON response for it
     """
 
     if opening_post_id:
         opening_post = get_object_or_404(Post, id=opening_post_id)
     else:
         opening_post = None
 
     status = STATUS_OK
     errors = []
 
     post = None
     if request.method == 'POST':
         if opening_post:
             form = PostForm(request.POST, request.FILES, error_class=PlainErrorList)
         else:
             form = ThreadForm(request.POST, request.FILES, error_class=PlainErrorList)
 
         form.session = request.session
 
         if form.need_to_ban:
             # Ban user because he is suspected to be a bot
             # _ban_current_user(request)
             status = STATUS_ERROR
         if form.is_valid():
             post = Post.objects.create_from_form(request, form, opening_post,
                                                  html_response=False)
             if not post:
                 status = STATUS_ERROR
             else:
                 logger.info('Added post #%d via api.' % post.id)
         else:
             status = STATUS_ERROR
             errors = form.as_json_errors()
     else:
         status = STATUS_ERROR
 
     response = {
         'status': status,
         'errors': errors,
     }
 
     if post:
         response['post_id'] = post.id
         if not opening_post:
             # FIXME For now we include URL only for threads to navigate to them.
             # This needs to become something universal, just not yet sure how.
             response['url'] = post.get_absolute_url()
 
     return HttpResponse(content=json.dumps(response))
 
 
 def get_post(request, post_id):
     """
     Gets the html of a post. Used for popups. Post can be truncated if used
     in threads list with 'truncated' get parameter.
     """
 
     post = get_object_or_404(Post, id=post_id)
     truncated = PARAMETER_TRUNCATED in request.GET
 
     return HttpResponse(content=post.get_view(truncated=truncated, need_op_data=True))
 
 
 def api_get_threads(request, count):
     """
     Gets the JSON thread opening posts list.
     Parameters that can be used for filtering:
     tag, offset (from which thread to get results)
     """
 
     if PARAMETER_TAG in request.GET:
         tag_name = request.GET[PARAMETER_TAG]
         if tag_name is not None:
             tag = get_object_or_404(Tag, name=tag_name)
             threads = tag.get_threads().exclude(status=STATUS_ARCHIVE)
     else: 
         threads = Thread.objects.exclude(status=STATUS_ARCHIVE)
 
     if PARAMETER_OFFSET in request.GET:
         offset = request.GET[PARAMETER_OFFSET]
         offset = int(offset) if offset is not None else 0
     else:
         offset = 0
 
     threads = threads.order_by('-bump_time')
     threads = threads[offset:offset + int(count)]
 
     opening_posts = []
     for thread in threads:
         opening_post = thread.get_opening_post()
 
         # TODO Add tags, replies and images count
         post_data = opening_post.get_post_data(include_last_update=True)
         post_data['status'] = thread.get_status()
 
         opening_posts.append(post_data)
 
     return HttpResponse(content=json.dumps(opening_posts))
 
 
 # TODO Test this
 def api_get_tags(request):
     """
     Gets all tags or user tags.
     """
 
     # TODO Get favorite tags for the given user ID
 
     tags = TagAlias.objects.all()
 
     term = request.GET.get('term')
     if term is not None:
         tags = tags.filter(name__contains=term)
 
     tag_names = [tag.name for tag in tags]
 
     return HttpResponse(content=json.dumps(tag_names))
 
 
 def api_get_stickers(request):
     term = request.GET.get('term')
     if not term:
         return HttpResponseBadRequest()
     
     global_stickers = AttachmentSticker.objects.filter(Q(name__icontains=term) | Q(stickerpack__name__icontains=term))
     local_stickers = [sticker for sticker in get_settings_manager(request).get_stickers() if term in sticker.name]
     stickers = list(global_stickers) + local_stickers
 
     image_dict = [{'thumb': sticker.attachment.get_thumb_url(),
                    'alias': str(sticker)}
                   for sticker in stickers]
 
     return HttpResponse(content=json.dumps(image_dict))
 
 
 # TODO The result can be cached by the thread last update time
 # TODO Test this
 def api_get_thread_posts(request, opening_post_id):
     """
     Gets the JSON array of thread posts
     """
 
     opening_post = get_object_or_404(Post, id=opening_post_id)
     thread = opening_post.get_thread()
     posts = thread.get_replies()
 
     json_data = {
         'posts': [],
         'last_update': None,
     }
     json_post_list = []
 
     for post in posts:
         json_post_list.append(post.get_post_data())
     json_data['last_update'] = datetime_to_epoch(thread.last_edit_time)
     json_data['posts'] = json_post_list
 
     return HttpResponse(content=json.dumps(json_data))
 
 
 def api_get_notifications(request, username):
     last_notification_id_str = request.GET.get('last', None)
     last_id = int(last_notification_id_str) if last_notification_id_str is not None else None
 
     posts = Notification.objects.get_notification_posts(usernames=[username],
                                                         last=last_id)
 
     json_post_list = []
     for post in posts:
         json_post_list.append(post.get_post_data())
     return HttpResponse(content=json.dumps(json_post_list))
 
 
 def api_get_post(request, post_id):
     """
     Gets the JSON of a post. This can be
     used as and API for external clients.
     """
 
     post = get_object_or_404(Post, id=post_id)
 
     json = serializers.serialize("json", [post], fields=(
         "pub_time", "_text_rendered", "title", "text", "image",
         "image_width", "image_height", "replies", "tags"
     ))
 
     return HttpResponse(content=json)
 
 
 def api_get_preview(request):
     raw_text = request.POST['raw_text']
 
     parser = Parser()
     return HttpResponse(content=parser.parse(parser.preparse(raw_text)))
 
 
 def api_get_new_posts(request):
     """
     Gets favorite threads and unread posts count.
     """
     posts = list()
 
     include_posts = 'include_posts' in request.GET
 
     settings_manager = get_settings_manager(request)
-    fav_threads = settings_manager.get_fav_threads()
-    fav_thread_ops = Post.objects.filter(id__in=fav_threads.keys())\
-        .order_by('-pub_time').prefetch_related('thread')
 
-    ops = [{'op': op, 'last_id': fav_threads[str(op.id)]} for op in fav_thread_ops]
+    last_posts = settings_manager.get_last_posts()
     if include_posts:
-        new_post_threads = Thread.objects.get_new_posts(ops)
+        new_post_threads = Thread.objects.get_new_posts(last_posts)
         if new_post_threads:
             thread_ids = {thread.id: thread for thread in new_post_threads}
         else:
             thread_ids = dict()
 
-        for op in fav_thread_ops:
+        for post in last_posts:
             fav_thread_dict = dict()
 
-            op_thread = op.get_thread()
-            if op_thread.id in thread_ids:
-                thread = thread_ids[op_thread.id]
+            thread = post.get_thread()
+            op = thread.get_opening_post()
+            if thread.id in thread_ids:
+                thread = thread_ids[thread.id]
                 new_post_count = thread.new_post_count
                 fav_thread_dict['newest_post_link'] = thread.get_replies()\
-                        .filter(id__gt=fav_threads[str(op.id)])\
+                        .filter(id__gt=post.id)\
                         .first().get_absolute_url(thread=thread)
             else:
                 new_post_count = 0
             fav_thread_dict['new_post_count'] = new_post_count
 
             fav_thread_dict['id'] = op.id
 
             fav_thread_dict['post_url'] = op.get_link_view()
             fav_thread_dict['title'] = op.title
 
             posts.append(fav_thread_dict)
     else:
         fav_thread_dict = dict()
         fav_thread_dict['new_post_count'] = \
-            Thread.objects.get_new_post_count(ops)
+            Thread.objects.get_new_post_count(last_posts)
         posts.append(fav_thread_dict)
 
     return HttpResponse(content=json.dumps(posts))

boards/views/feed.py

@@ -1,132 +1,132
 from django.urls import reverse
 from django.shortcuts import render
 
 from boards import settings
 from boards.abstracts.constants import PARAM_PAGE
 from boards.abstracts.paginator import get_paginator
 from boards.abstracts.settingsmanager import get_settings_manager
 from boards.models import Post
 from boards.settings import SECTION_VIEW
 from boards.views.base import BaseBoardView
 from boards.views.mixins import PaginatedMixin
 
 POSTS_PER_PAGE = settings.get_int(SECTION_VIEW, 'PostsPerPage')
 
 PARAMETER_POSTS = 'posts'
 PARAMETER_QUERIES = 'queries'
 
 TEMPLATE = 'boards/feed.html'
 DEFAULT_PAGE = 1
 
 
 class FeedFilter:
     @staticmethod
     def get_filtered_posts(request, posts):
         return posts
 
     @staticmethod
     def get_query(request):
         return None
 
 
 class TripcodeFilter(FeedFilter):
     @staticmethod
     def get_filtered_posts(request, posts):
         filtered_posts = posts
         tripcode = request.GET.get('tripcode', None)
         if tripcode:
             filtered_posts = filtered_posts.filter(tripcode=tripcode)
         return filtered_posts
 
     @staticmethod
     def get_query(request):
         tripcode = request.GET.get('tripcode', None)
         if tripcode:
             return 'Tripcode: {}'.format(tripcode)
 
 
 class FavoritesFilter(FeedFilter):
     @staticmethod
     def get_filtered_posts(request, posts):
         filtered_posts = posts
 
         favorites = 'favorites' in request.GET
         if favorites:
             settings_manager = get_settings_manager(request)
-            fav_thread_ops = Post.objects.filter(id__in=settings_manager.get_fav_threads().keys())
-            fav_threads = [op.get_thread() for op in fav_thread_ops]
-            filtered_posts = filtered_posts.filter(thread__in=fav_threads)
+            last_posts = settings_manager.get_last_posts()
+            threads = [post.get_thread() for post in last_posts]
+            filtered_posts = filtered_posts.filter(thread__in=threads)
         return filtered_posts
 
 
 class IpFilter(FeedFilter):
     @staticmethod
     def get_filtered_posts(request, posts):
         filtered_posts = posts
 
         ip = request.GET.get('ip', None)
         if ip and request.user.has_perm('post_delete'):
             filtered_posts = filtered_posts.filter(poster_ip=ip)
         return filtered_posts
 
     @staticmethod
     def get_query(request):
         ip = request.GET.get('ip', None)
         if ip:
             return 'IP: {}'.format(ip)
 
 
 class ImageFilter(FeedFilter):
     @staticmethod
     def get_filtered_posts(request, posts):
         filtered_posts = posts
 
         image = request.GET.get('image', None)
         if image:
             filtered_posts = filtered_posts.filter(attachments__file=image)
         return filtered_posts
 
     @staticmethod
     def get_query(request):
         image = request.GET.get('image', None)
         if image:
             return 'File: {}'.format(image)
 
 
 class FeedView(PaginatedMixin, BaseBoardView):
     filters = (
             TripcodeFilter,
             FavoritesFilter,
             IpFilter,
             ImageFilter,
     )
 
     def get(self, request):
         page = request.GET.get(PARAM_PAGE, DEFAULT_PAGE)
 
         params = self.get_context_data(request=request)
 
         settings_manager = get_settings_manager(request)
 
         posts = Post.objects.exclude(
             thread__tags__in=settings_manager.get_hidden_tags()).order_by(
             '-pub_time').prefetch_related('attachments', 'thread')
         queries = []
         for filter in self.filters:
             posts = filter.get_filtered_posts(request, posts)
             query = filter.get_query(request)
             if query:
                 queries.append(query)
         params[PARAMETER_QUERIES] = queries
 
         paginator = get_paginator(posts, POSTS_PER_PAGE)
         paginator.current_page = int(page)
 
         params[PARAMETER_POSTS] = paginator.page(page).object_list
 
         paginator.set_url(reverse('feed'), request.GET.dict())
 
         params.update(self.get_page_context(paginator, page))
 
         return render(request, TEMPLATE, params)