Added django management command to clean old users. Cleaned up failing tests. Fixed error when trying to get a non-existent post

boards/management/__init__.py

@@ -0,0 +1,1 b''
+__author__ = 'vurdalak'

boards/management/commands/__init__.py

@@ -0,0 +1,1 b''
+__author__ = 'vurdalak'

boards/management/commands/cleanusers.py

@@ -0,0 +1,29 b''
+from datetime import datetime, timedelta
+from django.core.management import BaseCommand
+from django.db import transaction
+from django.db.models import Count
+from boards.models import User, Post
+
+__author__ = 'neko259'
+
+OLD_USER_AGE_DAYS = 90
+
+
+class Command(BaseCommand):
+    help = 'Removes empty users (that don\'t have posts or tags'
+
+    @transaction.atomic
+    def handle(self, *args, **options):
+        old_registration_date = datetime.now().date() - timedelta(
+            OLD_USER_AGE_DAYS)
+
+        old_users = User.objects.annotate(tags_count=Count('fav_tags')).filter(
+            tags_count=0).filter(registration_time__lt=old_registration_date)
+        deleted_users = 0
+        for user in old_users:
+            if not Post.objects.filter(user=user).exists():
+                self.stdout.write('Deleting user %s' % user.user_id)
+                user.delete()
+                deleted_users += 1
+
+        self.stdout.write('Deleted %d users' % deleted_users)
  No newline at end of file

boards/tests.py

@@ -1,260 +1,259 b''
 # coding=utf-8
 import time
 import logging
 from django.core.paginator import Paginator
 
 from django.test import TestCase
 from django.test.client import Client
 from django.core.urlresolvers import reverse, NoReverseMatch
 
 from boards.models import Post, Tag, Thread
 from boards import urls
 from neboard import settings
 
 PAGE_404 = 'boards/404.html'
 
 TEST_TEXT = 'test text'
 
 NEW_THREAD_PAGE = '/'
 THREAD_PAGE_ONE = '/thread/1/'
 THREAD_PAGE = '/thread/'
 TAG_PAGE = '/tag/'
 HTTP_CODE_REDIRECT = 302
 HTTP_CODE_OK = 200
 HTTP_CODE_NOT_FOUND = 404
 
 logger = logging.getLogger(__name__)
 
 
 class PostTests(TestCase):
 
     def _create_post(self):
         return Post.objects.create_post(title='title',
                                         text='text')
 
     def test_post_add(self):
         """Test adding post"""
 
         post = self._create_post()
 
         self.assertIsNotNone(post, 'No post was created')
 
     def test_delete_post(self):
         """Test post deletion"""
 
         post = self._create_post()
         post_id = post.id
 
         Post.objects.delete_post(post)
 
         self.assertFalse(Post.objects.filter(id=post_id).exists())
 
     def test_delete_thread(self):
         """Test thread deletion"""
 
         opening_post = self._create_post()
         thread = opening_post.get_thread()
         reply = Post.objects.create_post("", "", thread=thread)
 
         thread.delete_with_posts()
 
         self.assertFalse(Post.objects.filter(id=reply.id).exists())
 
     def test_post_to_thread(self):
         """Test adding post to a thread"""
 
         op = self._create_post()
         post = Post.objects.create_post("", "", thread=op.get_thread())
 
         self.assertIsNotNone(post, 'Reply to thread wasn\'t created')
         self.assertEqual(op.get_thread().last_edit_time, post.pub_time,
                          'Post\'s create time doesn\'t match thread last edit'
                          ' time')
 
     def test_delete_posts_by_ip(self):
         """Test deleting posts with the given ip"""
 
         post = self._create_post()
         post_id = post.id
 
         Post.objects.delete_posts_by_ip('0.0.0.0')
 
         self.assertFalse(Post.objects.filter(id=post_id).exists())
 
     def test_get_thread(self):
         """Test getting all posts of a thread"""
 
         opening_post = self._create_post()
 
         for i in range(0, 2):
             Post.objects.create_post('title', 'text',
                                      thread=opening_post.get_thread())
 
         thread = opening_post.get_thread()
 
         self.assertEqual(3, thread.replies.count())
 
     def test_create_post_with_tag(self):
         """Test adding tag to post"""
 
         tag = Tag.objects.create(name='test_tag')
         post = Post.objects.create_post(title='title', text='text', tags=[tag])
 
         thread = post.get_thread()
         self.assertIsNotNone(post, 'Post not created')
         self.assertTrue(tag in thread.tags.all(), 'Tag not added to thread')
         self.assertTrue(thread in tag.threads.all(), 'Thread not added to tag')
 
     def test_thread_max_count(self):
         """Test deletion of old posts when the max thread count is reached"""
 
         for i in range(settings.MAX_THREAD_COUNT + 1):
             self._create_post()
 
         self.assertEqual(settings.MAX_THREAD_COUNT,
                          len(Thread.objects.filter(archived=False)))
 
     def test_pages(self):
         """Test that the thread list is properly split into pages"""
 
         for i in range(settings.MAX_THREAD_COUNT):
             self._create_post()
 
         all_threads = Thread.objects.filter(archived=False)
 
         paginator = Paginator(Thread.objects.filter(archived=False),
                               settings.THREADS_PER_PAGE)
         posts_in_second_page = paginator.page(2).object_list
         first_post = posts_in_second_page[0]
 
         self.assertEqual(all_threads[settings.THREADS_PER_PAGE].id,
                          first_post.id)
 
     def test_linked_tag(self):
         """Test adding a linked tag"""
 
         linked_tag = Tag.objects.create(name=u'tag1')
         tag = Tag.objects.create(name=u'tag2', linked=linked_tag)
 
         post = Post.objects.create_post("", "", tags=[tag])
 
         self.assertTrue(linked_tag in post.get_thread().tags.all(),
                         'Linked tag was not added')
 
 
 class PagesTest(TestCase):
 
     def test_404(self):
         """Test receiving error 404 when opening a non-existent page"""
 
         tag_name = u'test_tag'
         tag = Tag.objects.create(name=tag_name)
         client = Client()
 
         Post.objects.create_post('title', TEST_TEXT, tags=[tag])
 
         existing_post_id = Post.objects.all()[0].id
         response_existing = client.get(THREAD_PAGE + str(existing_post_id) +
                                        '/')
         self.assertEqual(HTTP_CODE_OK, response_existing.status_code,
                          u'Cannot open existing thread')
 
         response_not_existing = client.get(THREAD_PAGE + str(
             existing_post_id + 1) + '/')
-        self.assertEqual(PAGE_404,
-                         response_not_existing.templates[0].name,
+        self.assertEqual(PAGE_404, response_not_existing.templates[0].name,
                          u'Not existing thread is opened')
 
         response_existing = client.get(TAG_PAGE + tag_name + '/')
         self.assertEqual(HTTP_CODE_OK,
                          response_existing.status_code,
                          u'Cannot open existing tag')
 
         response_not_existing = client.get(TAG_PAGE + u'not_tag' + '/')
         self.assertEqual(PAGE_404,
                          response_not_existing.templates[0].name,
                          u'Not existing tag is opened')
 
         reply_id = Post.objects.create_post('', TEST_TEXT,
                                             thread=Post.objects.all()[0]
-                                            .thread)
+                                            .get_thread())
         response_not_existing = client.get(THREAD_PAGE + str(
             reply_id) + '/')
         self.assertEqual(PAGE_404,
                          response_not_existing.templates[0].name,
                          u'Reply is opened as a thread')
 
 
 class FormTest(TestCase):
     def test_post_validation(self):
         # Disable captcha for the test
         captcha_enabled = settings.ENABLE_CAPTCHA
         settings.ENABLE_CAPTCHA = False
 
         client = Client()
 
         valid_tags = u'tag1 tag_2 тег_3'
         invalid_tags = u'$%_356 ---'
 
         response = client.post(NEW_THREAD_PAGE, {'title': 'test title',
                                                  'text': TEST_TEXT,
                                                  'tags': valid_tags})
         self.assertEqual(response.status_code, HTTP_CODE_REDIRECT,
                          msg='Posting new message failed: got code ' +
                              str(response.status_code))
 
         self.assertEqual(1, Post.objects.count(),
                          msg='No posts were created')
 
         client.post(NEW_THREAD_PAGE, {'text': TEST_TEXT,
                                       'tags': invalid_tags})
         self.assertEqual(1, Post.objects.count(), msg='The validation passed '
                                                       'where it should fail')
 
         # Change posting delay so we don't have to wait for 30 seconds or more
         old_posting_delay = settings.POSTING_DELAY
         # Wait fot the posting delay or we won't be able to post
         settings.POSTING_DELAY = 1
         time.sleep(settings.POSTING_DELAY + 1)
         response = client.post(THREAD_PAGE_ONE, {'text': TEST_TEXT,
                                                  'tags': valid_tags})
         self.assertEqual(HTTP_CODE_REDIRECT, response.status_code,
                          msg=u'Posting new message failed: got code ' +
                              str(response.status_code))
         # Restore posting delay
         settings.POSTING_DELAY = old_posting_delay
 
         self.assertEqual(2, Post.objects.count(),
                          msg=u'No posts were created')
 
         # Restore captcha setting
         settings.ENABLE_CAPTCHA = captcha_enabled
 
 
 class ViewTest(TestCase):
 
     def test_all_views(self):
         '''
         Try opening all views defined in ulrs.py that don't need additional
         parameters
         '''
 
         client = Client()
         for url in urls.urlpatterns:
             try:
                 view_name = url.name
                 logger.debug('Testing view %s' % view_name)
 
                 try:
                     response = client.get(reverse(view_name))
 
                     self.assertEqual(HTTP_CODE_OK, response.status_code,
                             '%s view not opened' % view_name)
                 except NoReverseMatch:
                     # This view just needs additional arguments
                     pass
                 except Exception, e:
                     self.fail('Got exception %s at %s view' % (e, view_name))
             except AttributeError:
                 # This is normal, some views do not have names
                 pass

boards/views/api.py

@@ -1,248 +1,245 b''
 from datetime import datetime
 import json
 import logging
 from django.db import transaction
 from django.http import HttpResponse
 from django.shortcuts import get_object_or_404, render
 from django.template import RequestContext
 from django.utils import timezone
 from django.core import serializers
 
 from boards.forms import PostForm, PlainErrorList
 from boards.models import Post, Thread, Tag
 from boards.utils import datetime_to_epoch
 from boards.views.thread import ThreadView
 
 __author__ = 'neko259'
 
 PARAMETER_TRUNCATED = 'truncated'
 PARAMETER_TAG = 'tag'
 PARAMETER_OFFSET = 'offset'
 PARAMETER_DIFF_TYPE = 'type'
 
 DIFF_TYPE_HTML = 'html'
 DIFF_TYPE_JSON = 'json'
 
 STATUS_OK = 'ok'
 STATUS_ERROR = 'error'
 
 logger = logging.getLogger(__name__)
 
 
 @transaction.atomic
 def api_get_threaddiff(request, thread_id, last_update_time):
     """
     Gets posts that were changed or added since time
     """
 
     thread = get_object_or_404(Post, id=thread_id).get_thread()
 
-    logger.info('Getting thread #%s diff since %s' % (thread_id,
-        last_update_time))
-
     filter_time = datetime.fromtimestamp(float(last_update_time) / 1000000,
                                          timezone.get_current_timezone())
 
     json_data = {
         'added': [],
         'updated': [],
         'last_update': None,
     }
     added_posts = Post.objects.filter(thread_new=thread,
                                       pub_time__gt=filter_time) \
         .order_by('pub_time')
     updated_posts = Post.objects.filter(thread_new=thread,
                                         pub_time__lte=filter_time,
                                         last_edit_time__gt=filter_time)
 
     diff_type = DIFF_TYPE_HTML
     if PARAMETER_DIFF_TYPE in request.GET:
         diff_type = request.GET[PARAMETER_DIFF_TYPE]
 
     for post in added_posts:
         json_data['added'].append(_get_post_data(post.id, diff_type, request))
     for post in updated_posts:
         json_data['updated'].append(_get_post_data(post.id, diff_type, request))
     json_data['last_update'] = datetime_to_epoch(thread.last_edit_time)
 
     return HttpResponse(content=json.dumps(json_data))
 
 
 def api_add_post(request, opening_post_id):
     """
     Adds a post and return the JSON response for it
     """
 
     opening_post = get_object_or_404(Post, id=opening_post_id)
 
     logger.info('Adding post via api...')
 
     status = STATUS_OK
     errors = []
 
     if request.method == 'POST':
         form = PostForm(request.POST, request.FILES, error_class=PlainErrorList)
         form.session = request.session
 
         if form.need_to_ban:
             # Ban user because he is suspected to be a bot
             # _ban_current_user(request)
             status = STATUS_ERROR
         if form.is_valid():
             post = ThreadView().new_post(request, form, opening_post,
                                            html_response=False)
             if not post:
                 status = STATUS_ERROR
             else:
                 logger.info('Added post #%d via api.' % post.id)
         else:
             status = STATUS_ERROR
             errors = form.as_json_errors()
 
     response = {
         'status': status,
         'errors': errors,
     }
 
     return HttpResponse(content=json.dumps(response))
 
 
 def get_post(request, post_id):
     """
     Gets the html of a post. Used for popups. Post can be truncated if used
     in threads list with 'truncated' get parameter.
     """
 
     logger.info('Getting post #%s' % post_id)
 
     post = get_object_or_404(Post, id=post_id)
 
     context = RequestContext(request)
     context['post'] = post
     if PARAMETER_TRUNCATED in request.GET:
         context[PARAMETER_TRUNCATED] = True
 
     return render(request, 'boards/api_post.html', context)
 
 
 # TODO Test this
 def api_get_threads(request, count):
     """
     Gets the JSON thread opening posts list.
     Parameters that can be used for filtering:
     tag, offset (from which thread to get results)
     """
 
     if PARAMETER_TAG in request.GET:
         tag_name = request.GET[PARAMETER_TAG]
         if tag_name is not None:
             tag = get_object_or_404(Tag, name=tag_name)
             threads = tag.threads.filter(archived=False)
     else: 
         threads = Thread.objects.filter(archived=False)
 
     if PARAMETER_OFFSET in request.GET:
         offset = request.GET[PARAMETER_OFFSET]
         offset = int(offset) if offset is not None else 0
     else:
         offset = 0
 
     threads = threads.order_by('-bump_time')
     threads = threads[offset:offset + int(count)]
 
     opening_posts = []
     for thread in threads:
         opening_post = thread.get_opening_post()
 
         # TODO Add tags, replies and images count
         opening_posts.append(_get_post_data(opening_post.id,
             include_last_update=True))
 
     return HttpResponse(content=json.dumps(opening_posts))
 
 
 # TODO Test this
 def api_get_tags(request):
     """
     Gets all tags or user tags.
     """
 
     # TODO Get favorite tags for the given user ID
 
     tags = Tag.objects.get_not_empty_tags()
     tag_names = []
     for tag in tags:
         tag_names.append(tag.name)
 
     return HttpResponse(content=json.dumps(tag_names))
 
 
 # TODO The result can be cached by the thread last update time
 # TODO Test this
 def api_get_thread_posts(request, opening_post_id):
     """
     Gets the JSON array of thread posts
     """
 
     opening_post = get_object_or_404(Post, id=opening_post_id)
     thread = opening_post.get_thread()
     posts = thread.get_replies()
 
     json_data = {
         'posts': [],
         'last_update': None,
     }
     json_post_list = []
 
     for post in posts:
         json_post_list.append(_get_post_data(post.id))
     json_data['last_update'] = datetime_to_epoch(thread.last_edit_time)
     json_data['posts'] = json_post_list
 
     return HttpResponse(content=json.dumps(json_data))
 
 
 def api_get_post(request, post_id):
     """
     Gets the JSON of a post. This can be
     used as and API for external clients.
     """
 
     post = get_object_or_404(Post, id=post_id)
 
     json = serializers.serialize("json", [post], fields=(
         "pub_time", "_text_rendered", "title", "text", "image",
         "image_width", "image_height", "replies", "tags"
     ))
 
     return HttpResponse(content=json)
 
 
 def get_tag_popularity(request, tag_name):
     tag = get_object_or_404(Tag, name=tag_name)
 
     json_data = []
     json_data['popularity'] = tag.get_popularity()
 
     return HttpResponse(content=json.dumps(json_data))
 
 
 # TODO Add pub time and replies
 def _get_post_data(post_id, format_type=DIFF_TYPE_JSON, request=None,
         include_last_update=False):
     if format_type == DIFF_TYPE_HTML:
         return get_post(request, post_id).content.strip()
     elif format_type == DIFF_TYPE_JSON:
         post = get_object_or_404(Post, id=post_id)
         post_json = {
             'id': post.id,
             'title': post.title,
             'text': post.text.rendered,
         }
         if post.image:
             post_json['image'] = post.image.url
             post_json['image_preview'] = post.image.url_200x150
         if include_last_update:
             post_json['bump_time'] = datetime_to_epoch(
                     post.thread_new.bump_time)
         return post_json

boards/views/base.py

@@ -1,127 +1,107 b''
 from datetime import datetime, timedelta
 import hashlib
 from django.db import transaction
 from django.db.models import Count
 from django.template import RequestContext
 from django.utils import timezone
 from django.views.generic import View
 from boards import utils
 from boards.models import User, Post
 from boards.models.post import SETTING_MODERATE
 from boards.models.user import RANK_USER, Ban
 import neboard
 
 BAN_REASON_SPAM = 'Autoban: spam bot'
 
-OLD_USER_AGE_DAYS = 90
-
 PARAMETER_FORM = 'form'
 
 
 class BaseBoardView(View):
 
     def get_context_data(self, **kwargs):
         request = kwargs['request']
         context = self._default_context(request)
 
         context['version'] = neboard.settings.VERSION
         context['site_name'] = neboard.settings.SITE_NAME
 
         return context
 
     def _default_context(self, request):
         """Create context with default values that are used in most views"""
 
         context = RequestContext(request)
 
         user = self._get_user(request)
         context['user'] = user
         context['tags'] = user.fav_tags.all()
         context['posts_per_day'] = float(Post.objects.get_posts_per_day())
 
         theme = self._get_theme(request, user)
         context['theme'] = theme
         context['theme_css'] = 'css/' + theme + '/base_page.css'
 
         # This shows the moderator panel
         moderate = user.get_setting(SETTING_MODERATE)
         if moderate == 'True':
             context['moderator'] = user.is_moderator()
         else:
             context['moderator'] = False
 
         return context
 
     def _get_user(self, request):
         """
         Get current user from the session. If the user does not exist, create
         a new one.
         """
 
         session = request.session
         if not 'user_id' in session:
             request.session.save()
 
             md5 = hashlib.md5()
             md5.update(session.session_key)
             new_id = md5.hexdigest()
 
             while User.objects.filter(user_id=new_id).exists():
                 md5.update(str(timezone.now()))
                 new_id = md5.hexdigest()
 
             time_now = timezone.now()
             user = User.objects.create(user_id=new_id, rank=RANK_USER,
                                        registration_time=time_now)
 
-            # TODO Move this to manage.py commands
-            #self._delete_old_users()
-
             session['user_id'] = user.id
         else:
             user = User.objects.select_related('fav_tags').get(
                 id=session['user_id'])
 
         return user
 
     def _get_theme(self, request, user=None):
         """
         Get user's CSS theme
         """
 
         if not user:
             user = self._get_user(request)
         theme = user.get_setting('theme')
         if not theme:
             theme = neboard.settings.DEFAULT_THEME
 
         return theme
 
-    def _delete_old_users(self):
-        """
-        Delete users with no favorite tags and posted messages. These can be spam
-        bots or just old user accounts
-        """
-
-        old_registration_date = datetime.now().date() - timedelta(
-            OLD_USER_AGE_DAYS)
-
-        for user in User.objects.annotate(tags_count=Count('fav_tags')).filter(
-                tags_count=0).filter(
-                registration_time__lt=old_registration_date):
-            if not Post.objects.filter(user=user).exists():
-                user.delete()
-
     @transaction.atomic
     def _ban_current_user(self, request):
         """
         Add current user to the IP ban list
         """
 
         ip = utils.get_client_ip(request)
         ban, created = Ban.objects.get_or_create(ip=ip)
         if created:
             ban.can_read = False
             ban.reason = BAN_REASON_SPAM
         ban.save()
 

boards/views/thread.py

@@ -1,129 +1,132 b''
 import string
 from django.core.urlresolvers import reverse
 from django.db import transaction
 from django.http import Http404
 from django.shortcuts import get_object_or_404, render, redirect
 from django.views.generic.edit import FormMixin
 from boards import utils
 from boards.forms import PostForm, PlainErrorList
 from boards.models import Post, Ban, Tag
 from boards.views.banned import BannedView
 from boards.views.base import BaseBoardView, PARAMETER_FORM
 from boards.views.posting_mixin import PostMixin
 import neboard
 
 MODE_GALLERY = 'gallery'
 MODE_NORMAL = 'normal'
 
 PARAMETER_MAX_REPLIES = 'max_replies'
 PARAMETER_THREAD = 'thread'
 PARAMETER_BUMPABLE = 'bumpable'
 
 
 class ThreadView(BaseBoardView, PostMixin, FormMixin):
 
     def get(self, request, post_id, mode=MODE_NORMAL, form=None):
+        try:
         opening_post = Post.objects.filter(id=post_id).only('thread_new')[0]
+        except IndexError:
+            raise Http404
 
         # If this is not OP, don't show it as it is
         if not opening_post or not opening_post.is_opening():
             raise Http404
 
         if not form:
             form = PostForm(error_class=PlainErrorList)
 
         thread_to_show = opening_post.get_thread()
 
         context = self.get_context_data(request=request)
 
         context[PARAMETER_FORM] = form
         context["last_update"] = utils.datetime_to_epoch(
             thread_to_show.last_edit_time)
         context[PARAMETER_THREAD] = thread_to_show
         context[PARAMETER_MAX_REPLIES] = neboard.settings.MAX_POSTS_PER_THREAD
 
         if MODE_NORMAL == mode:
             context[PARAMETER_BUMPABLE] = thread_to_show.can_bump()
             if context[PARAMETER_BUMPABLE]:
                 context['posts_left'] = neboard.settings.MAX_POSTS_PER_THREAD \
                         - thread_to_show.get_reply_count()
                 context['bumplimit_progress'] = str(
                     float(context['posts_left']) /
                     neboard.settings.MAX_POSTS_PER_THREAD * 100)
 
             context['opening_post'] = opening_post
 
             document = 'boards/thread.html'
         elif MODE_GALLERY == mode:
             posts = thread_to_show.get_replies(view_fields_only=True)
             context['posts'] = posts.filter(image_width__gt=0)
 
             document = 'boards/thread_gallery.html'
         else:
             raise Http404
 
         return render(request, document, context)
 
     def post(self, request, post_id, mode=MODE_NORMAL):
         opening_post = get_object_or_404(Post, id=post_id)
 
         # If this is not OP, don't show it as it is
         if not opening_post.is_opening():
             raise Http404
 
         if not opening_post.get_thread().archived:
             form = PostForm(request.POST, request.FILES,
                             error_class=PlainErrorList)
             form.session = request.session
 
             if form.is_valid():
                 return self.new_post(request, form, opening_post)
             if form.need_to_ban:
                 # Ban user because he is suspected to be a bot
                 self._ban_current_user(request)
 
             return self.get(request, post_id, mode, form)
 
     @transaction.atomic
     def new_post(self, request, form, opening_post=None, html_response=True):
         """Add a new post (in thread or as a reply)."""
 
         ip = utils.get_client_ip(request)
         is_banned = Ban.objects.filter(ip=ip).exists()
 
         if is_banned:
             if html_response:
                 return redirect(BannedView().as_view())
             else:
                 return None
 
         data = form.cleaned_data
 
         title = data['title']
         text = data['text']
 
         text = self._remove_invalid_links(text)
 
         if 'image' in data.keys():
             image = data['image']
         else:
             image = None
 
         tags = []
 
         post_thread = opening_post.get_thread()
 
         post = Post.objects.create_post(title=title, text=text, ip=ip,
                                         thread=post_thread, image=image,
                                         tags=tags,
                                         user=self._get_user(request))
 
         thread_to_show = (opening_post.id if opening_post else post.id)
 
         if html_response:
             if opening_post:
                 return redirect(reverse(
                     'thread',
                     kwargs={'post_id': thread_to_show}) + '#' + str(post.id))
         else:
             return post