Moved text parser and preparser to a separate module (BB-64)

boards/mdx_neboard.py

@@ -1,210 +1,233 b''
 # coding=utf-8
 
 import re
 import bbcode
+
+from urllib.parse import unquote
+
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.urlresolvers import reverse
 
 import boards
 
 
 __author__ = 'neko259'
 
 
 REFLINK_PATTERN = re.compile(r'^\d+$')
 MULTI_NEWLINES_PATTERN = re.compile(r'(\r?\n){2,}')
 ONE_NEWLINE = '\n'
+REGEX_URL = re.compile(r'https?\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(/\S*)?')
 
 
 class TextFormatter():
     """
     An interface for formatter that can be used in the text format panel
     """
 
     def __init__(self):
         pass
 
     name = ''
 
     # Left and right tags for the button preview
     preview_left = ''
     preview_right = ''
 
     # Left and right characters for the textarea input
     format_left = ''
     format_right = ''
 
 
 class AutolinkPattern():
     def handleMatch(self, m):
         link_element = etree.Element('a')
         href = m.group(2)
         link_element.set('href', href)
         link_element.text = href
 
         return link_element
 
 
 class QuotePattern(TextFormatter):
     name = 'q'
     preview_left = '<span class="multiquote">'
     preview_right = '</span>'
 
     format_left = '[quote]'
     format_right = '[/quote]'
 
 
 class SpoilerPattern(TextFormatter):
     name = 'spoiler'
     preview_left = '<span class="spoiler">'
     preview_right = '</span>'
 
     format_left = '[spoiler]'
     format_right = '[/spoiler]'
 
     def handleMatch(self, m):
         quote_element = etree.Element('span')
         quote_element.set('class', 'spoiler')
         quote_element.text = m.group(2)
 
         return quote_element
 
 
 class CommentPattern(TextFormatter):
     name = ''
     preview_left = '<span class="comment">// '
     preview_right = '</span>'
 
     format_left = '[comment]'
     format_right = '[/comment]'
 
 
 # TODO Use <s> tag here
 class StrikeThroughPattern(TextFormatter):
     name = 's'
     preview_left = '<span class="strikethrough">'
     preview_right = '</span>'
 
     format_left = '[s]'
     format_right = '[/s]'
 
 
 class ItalicPattern(TextFormatter):
     name = 'i'
     preview_left = '<i>'
     preview_right = '</i>'
 
     format_left = '[i]'
     format_right = '[/i]'
 
 
 class BoldPattern(TextFormatter):
     name = 'b'
     preview_left = '<b>'
     preview_right = '</b>'
 
     format_left = '[b]'
     format_right = '[/b]'
 
 
 class CodePattern(TextFormatter):
     name = 'code'
     preview_left = '<code>'
     preview_right = '</code>'
 
     format_left = '[code]'
     format_right = '[/code]'
 
 
 def render_reflink(tag_name, value, options, parent, context):
     result = '>>%s' % value
 
     if REFLINK_PATTERN.match(value):
         post_id = int(value)
 
         try:
             post = boards.models.Post.objects.get(id=post_id)
 
             result = '<a href="%s">&gt;&gt;%s</a>' % (post.get_url(), post_id)
         except ObjectDoesNotExist:
             pass
 
     return result
 
 
 def render_multithread(tag_name, value, options, parent, context):
     result = '>>>%s' % value
 
     if REFLINK_PATTERN.match(value):
         post_id = int(value)
 
         try:
             post = boards.models.Post.objects.get(id=post_id)
 
             if post.is_opening():
                 result = '<a href="%s">&gt;&gt;&gt;%s</a>' % (post.get_url(), post_id)
         except ObjectDoesNotExist:
             pass
 
     return result
 
 
 def render_quote(tag_name, value, options, parent, context):
     source = ''
     if 'source' in options:
         source = options['source']
 
     if source:
         result = '<div class="multiquote"><div class="quote-header">%s</div><div class="quote-text">%s</div></div>' % (source, value)
     else:
         result = '<div class="multiquote"><div class="quote-text">%s</div></div>' % value
 
     return result
 
 
 def render_notification(tag_name, value, options, parent, content):
     username = value.lower()
 
     return '<a href="{}" class="user-cast">@{}</a>'.format(
         reverse('notifications', kwargs={'username': username}), username)
 
 
-def preparse_text(text):
-    """
-    Performs manual parsing before the bbcode parser is used.
-    """
-
-    return MULTI_NEWLINES_PATTERN.sub(ONE_NEWLINE, text)
-
-
-def bbcode_extended(markup):
-    # The newline hack is added because br's margin does not work in all
-    # browsers except firefox, when the div's does.
-    parser = bbcode.Parser(newline='<div class="br"></div>')
-    parser.add_formatter('post', render_reflink, strip=True)
-    parser.add_formatter('thread', render_multithread, strip=True)
-    parser.add_formatter('quote', render_quote, strip=True)
-    parser.add_formatter('user', render_notification, strip=True)
-    parser.add_simple_formatter('comment',
-                                '<span class="comment">//%(value)s</span>')
-    parser.add_simple_formatter('spoiler',
-                                '<span class="spoiler">%(value)s</span>')
-    parser.add_simple_formatter('s',
-                                '<span class="strikethrough">%(value)s</span>')
-    # TODO Why not use built-in tag?
-    parser.add_simple_formatter('code',
-                                '<pre><code>%(value)s</pre></code>',
-                                render_embedded=False)
-
-    text = preparse_text(markup)
-    return parser.format(text)
-
 formatters = [
     QuotePattern,
     SpoilerPattern,
     ItalicPattern,
     BoldPattern,
     CommentPattern,
     StrikeThroughPattern,
     CodePattern,
 ]
+
+
+PREPARSE_PATTERNS = {
+    r'>>>(\d+)': r'[thread]\1[/thread]',  # Multi-thread post ">>>123"
+    r'(?<!>)>>(\d+)': r'[post]\1[/post]',  # Reflink ">>123"
+    r'^>([^>].+)': r'[quote]\1[/quote]',  # Quote ">text"
+    r'^//(.+)': r'[comment]\1[/comment]',  # Comment "//text"
+    r'\B@(\w+)': r'[user]\1[/user]',  # User notification "@user"
+}
+
+
+class Parser:
+    def __init__(self):
+        # The newline hack is added because br's margin does not work in all
+        # browsers except firefox, when the div's does.
+        self.parser = bbcode.Parser(newline='<div class="br"></div>')
+
+        self.parser.add_formatter('post', render_reflink, strip=True)
+        self.parser.add_formatter('thread', render_multithread, strip=True)
+        self.parser.add_formatter('quote', render_quote, strip=True)
+        self.parser.add_formatter('user', render_notification, strip=True)
+        self.parser.add_simple_formatter(
+            'comment', '<span class="comment">//%(value)s</span>')
+        self.parser.add_simple_formatter(
+            'spoiler', '<span class="spoiler">%(value)s</span>')
+        self.parser.add_simple_formatter(
+            's', '<span class="strikethrough">%(value)s</span>')
+        # TODO Why not use built-in tag?
+        self.parser.add_simple_formatter('code',
+                                         '<pre><code>%(value)s</pre></code>',
+                                         render_embedded=False)
+
+    def preparse(self, text):
+        """
+        Performs manual parsing before the bbcode parser is used.
+        Preparsed text is saved as raw and the text before preparsing is lost.
+        """
+        new_text = MULTI_NEWLINES_PATTERN.sub(ONE_NEWLINE, text)
+
+        for key, value in PREPARSE_PATTERNS.items():
+            new_text = re.sub(key, value, new_text, flags=re.MULTILINE)
+
+        for link in REGEX_URL.findall(text):
+            new_text = new_text.replace(link, unquote(link))
+
+        return new_text
+
+    def parse(self, text):
+        return self.parser.format(text)
  No newline at end of file

boards/models/post.py

@@ -1,476 +1,450 b''
 from datetime import datetime, timedelta, date
 from datetime import time as dtime
 import logging
 import re
 
-from urllib.parse import unquote
-
 from adjacent import Client
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.urlresolvers import reverse
 from django.db import models, transaction
 from django.db.models import TextField
 from django.template.loader import render_to_string
 from django.utils import timezone
 
 from boards import settings
-from boards.mdx_neboard import bbcode_extended
+from boards.mdx_neboard import Parser
 from boards.models import PostImage
 from boards.models.base import Viewable
 from boards.utils import datetime_to_epoch, cached_result
 from boards.models.user import Notification
 import boards.models.thread
 
 
 WS_NOTIFICATION_TYPE_NEW_POST = 'new_post'
 WS_NOTIFICATION_TYPE = 'notification_type'
 
 WS_CHANNEL_THREAD = "thread:"
 
 APP_LABEL_BOARDS = 'boards'
 
 POSTS_PER_DAY_RANGE = 7
 
 BAN_REASON_AUTO = 'Auto'
 
 IMAGE_THUMB_SIZE = (200, 150)
 
 TITLE_MAX_LENGTH = 200
 
 # TODO This should be removed
 NO_IP = '0.0.0.0'
 
 # TODO Real user agent should be saved instead of this
 UNKNOWN_UA = ''
 
 REGEX_REPLY = re.compile(r'\[post\](\d+)\[/post\]')
 REGEX_MULTI_THREAD = re.compile(r'\[thread\](\d+)\[/thread\]')
-REGEX_URL = re.compile(r'https?\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(/\S*)?')
 REGEX_NOTIFICATION = re.compile(r'\[user\](\w+)\[/user\]')
 
 PARAMETER_TRUNCATED = 'truncated'
 PARAMETER_TAG = 'tag'
 PARAMETER_OFFSET = 'offset'
 PARAMETER_DIFF_TYPE = 'type'
 PARAMETER_BUMPABLE = 'bumpable'
 PARAMETER_THREAD = 'thread'
 PARAMETER_IS_OPENING = 'is_opening'
 PARAMETER_MODERATOR = 'moderator'
 PARAMETER_POST = 'post'
 PARAMETER_OP_ID = 'opening_post_id'
 PARAMETER_NEED_OPEN_LINK = 'need_open_link'
 PARAMETER_REPLY_LINK = 'reply_link'
 
 DIFF_TYPE_HTML = 'html'
 DIFF_TYPE_JSON = 'json'
 
-PREPARSE_PATTERNS = {
-    r'>>>(\d+)': r'[thread]\1[/thread]',  # Multi-thread post ">>>123"
-    r'(?<!>)>>(\d+)': r'[post]\1[/post]',  # Reflink ">>123"
-    r'^>([^>].+)': r'[quote]\1[/quote]',  # Quote ">text"
-    r'^//(.+)': r'[comment]\1[/comment]',  # Comment "//text"
-    r'\B@(\w+)': r'[user]\1[/user]',  # User notification "@user"
-}
-
 
 class PostManager(models.Manager):
     @transaction.atomic
     def create_post(self, title: str, text: str, image=None, thread=None,
                     ip=NO_IP, tags: list=None):
         """
         Creates new post
         """
 
         if not tags:
             tags = []
 
         posting_time = timezone.now()
         if not thread:
             thread = boards.models.thread.Thread.objects.create(
                 bump_time=posting_time, last_edit_time=posting_time)
             new_thread = True
         else:
             new_thread = False
 
-        pre_text = self._preparse_text(text)
+        pre_text = Parser().preparse(text)
 
         post = self.create(title=title,
                            text=pre_text,
                            pub_time=posting_time,
                            poster_ip=ip,
                            thread=thread,
                            poster_user_agent=UNKNOWN_UA,  # TODO Get UA at
                            # last!
                            last_edit_time=posting_time)
         post.threads.add(thread)
 
         logger = logging.getLogger('boards.post.create')
 
         logger.info('Created post {} by {}'.format(
             post, post.poster_ip))
 
         if image:
             post.images.add(PostImage.objects.create_with_hash(image))
 
         list(map(thread.add_tag, tags))
 
         if new_thread:
             boards.models.thread.Thread.objects.process_oldest_threads()
         else:
             thread.last_edit_time = posting_time
             thread.bump()
             thread.save()
 
         post.connect_replies()
         post.connect_threads()
         post.connect_notifications()
 
         return post
 
     def delete_posts_by_ip(self, ip):
         """
         Deletes all posts of the author with same IP
         """
 
         posts = self.filter(poster_ip=ip)
         for post in posts:
             post.delete()
 
     @cached_result
     def get_posts_per_day(self):
         """
         Gets average count of posts per day for the last 7 days
         """
 
         day_end = date.today()
         day_start = day_end - timedelta(POSTS_PER_DAY_RANGE)
 
         day_time_start = timezone.make_aware(datetime.combine(
             day_start, dtime()), timezone.get_current_timezone())
         day_time_end = timezone.make_aware(datetime.combine(
             day_end, dtime()), timezone.get_current_timezone())
 
         posts_per_period = float(self.filter(
             pub_time__lte=day_time_end,
             pub_time__gte=day_time_start).count())
 
         ppd = posts_per_period / POSTS_PER_DAY_RANGE
 
         return ppd
     
-    # TODO Make a separate parser module and move preparser there
-    def _preparse_text(self, text: str) -> str:
-        """
-        Preparses text to change patterns like '>>' to a proper bbcode
-        tags.
-        """
-
-        for key, value in PREPARSE_PATTERNS.items():
-            text = re.sub(key, value, text, flags=re.MULTILINE)
-
-        for link in REGEX_URL.findall(text):
-            text = text.replace(link, unquote(link))
-
-        return text
-
 
 class Post(models.Model, Viewable):
     """A post is a message."""
 
     objects = PostManager()
 
     class Meta:
         app_label = APP_LABEL_BOARDS
         ordering = ('id',)
 
     title = models.CharField(max_length=TITLE_MAX_LENGTH)
     pub_time = models.DateTimeField()
     text = TextField(blank=True, null=True)
     _text_rendered = TextField(blank=True, null=True, editable=False)
 
     images = models.ManyToManyField(PostImage, null=True, blank=True,
                                     related_name='ip+', db_index=True)
 
     poster_ip = models.GenericIPAddressField()
     poster_user_agent = models.TextField()
 
     last_edit_time = models.DateTimeField()
 
     referenced_posts = models.ManyToManyField('Post', symmetrical=False,
                                               null=True,
                                               blank=True, related_name='rfp+',
                                               db_index=True)
     refmap = models.TextField(null=True, blank=True)
     threads = models.ManyToManyField('Thread', db_index=True)
     thread = models.ForeignKey('Thread', db_index=True, related_name='pt+')
 
     def __str__(self):
         return 'P#{}/{}'.format(self.id, self.title)
 
     def get_title(self) -> str:
         """
         Gets original post title or part of its text.
         """
 
         title = self.title
         if not title:
             title = self.get_text()
 
         return title
 
     def build_refmap(self) -> None:
         """
         Builds a replies map string from replies list. This is a cache to stop
         the server from recalculating the map on every post show.
         """
 
         post_urls = ['<a href="{}">&gt;&gt;{}</a>'.format(
             refpost.get_url(), refpost.id) for refpost in self.referenced_posts.all()]
 
         self.refmap = ', '.join(post_urls)
 
     def get_sorted_referenced_posts(self):
         return self.refmap
 
     def is_referenced(self) -> bool:
         return self.refmap and len(self.refmap) > 0
 
     def is_opening(self) -> bool:
         """
         Checks if this is an opening post or just a reply.
         """
 
         return self.get_thread().get_opening_post_id() == self.id
 
     @cached_result
     def get_url(self):
         """
         Gets full url to the post.
         """
 
         thread = self.get_thread()
 
         opening_id = thread.get_opening_post_id()
 
         if self.id != opening_id:
             link = reverse('thread', kwargs={
                 'post_id': opening_id}) + '#' + str(self.id)
         else:
             link = reverse('thread', kwargs={'post_id': self.id})
 
         return link
 
     def get_thread(self):
         return self.thread
 
     def get_threads(self):
         """
         Gets post's thread.
         """
 
         return self.threads
 
     def get_referenced_posts(self):
         return self.referenced_posts.only('id', 'threads')
 
     def get_view(self, moderator=False, need_open_link=False,
                  truncated=False, *args, **kwargs):
         """
         Renders post's HTML view. Some of the post params can be passed over
         kwargs for the means of caching (if we view the thread, some params
         are same for every post and don't need to be computed over and over.
         """
 
         thread = self.get_thread()
         is_opening = kwargs.get(PARAMETER_IS_OPENING, self.is_opening())
         can_bump = kwargs.get(PARAMETER_BUMPABLE, thread.can_bump())
 
         if is_opening:
             opening_post_id = self.id
         else:
             opening_post_id = thread.get_opening_post_id()
 
         return render_to_string('boards/post.html', {
             PARAMETER_POST: self,
             PARAMETER_MODERATOR: moderator,
             PARAMETER_IS_OPENING: is_opening,
             PARAMETER_THREAD: thread,
             PARAMETER_BUMPABLE: can_bump,
             PARAMETER_NEED_OPEN_LINK: need_open_link,
             PARAMETER_TRUNCATED: truncated,
             PARAMETER_OP_ID: opening_post_id,
         })
 
     def get_search_view(self, *args, **kwargs):
         return self.get_view(args, kwargs)
 
     def get_first_image(self) -> PostImage:
         return self.images.earliest('id')
 
     def delete(self, using=None):
         """
         Deletes all post images and the post itself.
         """
 
         for image in self.images.all():
             image_refs_count = Post.objects.filter(images__in=[image]).count()
             if image_refs_count == 1:
                 image.delete()
 
         thread = self.get_thread()
         thread.last_edit_time = timezone.now()
         thread.save()
 
         super(Post, self).delete(using)
 
         logging.getLogger('boards.post.delete').info(
             'Deleted post {}'.format(self))
 
     def get_post_data(self, format_type=DIFF_TYPE_JSON, request=None,
                       include_last_update=False):
         """
         Gets post HTML or JSON data that can be rendered on a page or used by
         API.
         """
 
         if format_type == DIFF_TYPE_HTML:
             params = dict()
             params['post'] = self
             if PARAMETER_TRUNCATED in request.GET:
                 params[PARAMETER_TRUNCATED] = True
             else:
                 params[PARAMETER_REPLY_LINK] = True
 
             return render_to_string('boards/api_post.html', params)
         elif format_type == DIFF_TYPE_JSON:
             post_json = {
                 'id': self.id,
                 'title': self.title,
                 'text': self._text_rendered,
                 }
             if self.images.exists():
                 post_image = self.get_first_image()
                 post_json['image'] = post_image.image.url
                 post_json['image_preview'] = post_image.image.url_200x150
             if include_last_update:
                 post_json['bump_time'] = datetime_to_epoch(
                     self.get_thread().bump_time)
             return post_json
 
     def send_to_websocket(self, request, recursive=True):
         """
         Sends post HTML data to the thread web socket.
         """
 
         if not settings.WEBSOCKETS_ENABLED:
             return
 
         client = Client()
 
         logger = logging.getLogger('boards.post.websocket')
 
         thread_ids = list()
         for thread in self.get_threads().all():
             thread_ids.append(thread.id)
 
             channel_name = WS_CHANNEL_THREAD + str(thread.get_opening_post_id())
             client.publish(channel_name, {
                 WS_NOTIFICATION_TYPE: WS_NOTIFICATION_TYPE_NEW_POST,
             })
             client.send()
 
             logger.info('Sent notification from post #{} to channel {}'.format(
                 self.id, channel_name))
 
         if recursive:
             for reply_number in re.finditer(REGEX_REPLY, self.get_raw_text()):
                 post_id = reply_number.group(1)
 
                 try:
                     ref_post = Post.objects.get(id=post_id)
 
                     if ref_post.get_threads().exclude(id__in=thread_ids).exists():
                         # If post is in this thread, its thread was already notified.
                         # Otherwise, notify its thread separately.
                         ref_post.send_to_websocket(request, recursive=False)
                 except ObjectDoesNotExist:
                     pass
 
     def save(self, force_insert=False, force_update=False, using=None,
              update_fields=None):
-        self._text_rendered = bbcode_extended(self.get_raw_text())
+        self._text_rendered = Parser().parse(self.get_raw_text())
 
         super().save(force_insert, force_update, using, update_fields)
 
     def get_text(self) -> str:
         return self._text_rendered
 
     def get_raw_text(self) -> str:
         return self.text
 
     def get_absolute_id(self) -> str:
         """
         If the post has many threads, shows its main thread OP id in the post
         ID.
         """
 
         if self.get_threads().count() > 1:
             return '{}/{}'.format(self.get_thread().get_opening_post_id(), self.id)
         else:
             return str(self.id)
 
     def connect_notifications(self):
         for reply_number in re.finditer(REGEX_NOTIFICATION, self.get_raw_text()):
             user_name = reply_number.group(1).lower()
             Notification.objects.get_or_create(name=user_name, post=self)
 
     def connect_replies(self):
         """
         Connects replies to a post to show them as a reflink map
         """
 
         for reply_number in re.finditer(REGEX_REPLY, self.get_raw_text()):
             post_id = reply_number.group(1)
 
             try:
                 referenced_post = Post.objects.get(id=post_id)
 
                 referenced_post.referenced_posts.add(self)
                 referenced_post.last_edit_time = self.pub_time
                 referenced_post.build_refmap()
                 referenced_post.save(update_fields=['refmap', 'last_edit_time'])
 
                 referenced_threads = referenced_post.get_threads().all()
                 for thread in referenced_threads:
                     if thread.can_bump():
                         thread.update_bump_status()
 
                     thread.last_edit_time = self.pub_time
                     thread.save(update_fields=['last_edit_time', 'bumpable'])
             except ObjectDoesNotExist:
                 pass
 
     def connect_threads(self):
         """
         If the referenced post is an OP in another thread,
         make this post multi-thread.
         """
 
         for reply_number in re.finditer(REGEX_MULTI_THREAD, self.get_raw_text()):
             post_id = reply_number.group(1)
 
             try:
                 referenced_post = Post.objects.get(id=post_id)
 
                 if referenced_post.is_opening():
                     referenced_threads = referenced_post.get_threads().all()
                     for thread in referenced_threads:
                         if thread.can_bump():
                             thread.update_bump_status()
 
                         thread.last_edit_time = self.pub_time
                         thread.save(update_fields=['last_edit_time', 'bumpable'])
 
                         self.threads.add(thread)
             except ObjectDoesNotExist:
                 pass

boards/templates/boards/posting_general.html

@@ -1,192 +1,191 b''
 {% extends "boards/base.html" %}
 
 {% load i18n %}
 {% load cache %}
 {% load board %}
 {% load static %}
 {% load tz %}
 
 {% block head %}
     <meta name="robots" content="noindex">
 
     {% if tag %}
         <title>{{ tag.name }} - {{ site_name }}</title>
     {% else %}
         <title>{{ site_name }}</title>
     {% endif %}
 
     {% if current_page.has_previous %}
         <link rel="prev" href="
             {% if tag %}
                 {% url "tag" tag_name=tag.name page=current_page.previous_page_number %}
             {% else %}
                 {% url "index" page=current_page.previous_page_number %}
             {% endif %}
         " />
     {% endif %}
     {% if current_page.has_next %}
         <link rel="next" href="
             {% if tag %}
                 {% url "tag" tag_name=tag.name page=current_page.next_page_number %}
             {% else %}
                 {% url "index" page=current_page.next_page_number %}
             {% endif %}
         " />
     {% endif %}
 
 {% endblock %}
 
 {% block content %}
 
     {% get_current_language as LANGUAGE_CODE %}
     {% get_current_timezone as TIME_ZONE %}
 
 {% if tag %}
     <div class="tag_info">
         <h2>
             <form action="{% url 'tag' tag.name %}" method="post" class="post-button-form">
                 {% if is_favorite %}
                     <button name="method" value="unsubscribe" class="fav">★</button>
                 {% else %}
                     <button name="method" value="subscribe" class="not_fav">★</button>
                 {% endif %}
             </form>
             <form action="{% url 'tag' tag.name %}" method="post" class="post-button-form">
                 {% if is_hidden %}
                     <button name="method" value="unhide" class="fav">H</button>
                 {% else %}
                     <button name="method" value="hide" class="not_fav">H</button>
                 {% endif %}
             </form>
             {% autoescape off %}
                 {{ tag.get_view }}
             {% endautoescape %}
             {% if moderator %}
             <span class="moderator_info">[<a href="{% url 'admin:boards_tag_change' tag.id %}">{% trans 'Edit tag' %}</a>]</span>
             {% endif %}
         </h2>
         <p>{% blocktrans with thread_count=tag.get_thread_count post_count=tag.get_post_count %}This tag has {{ thread_count }} threads and {{ post_count }} posts.{% endblocktrans %}</p>
     </div>
 {% endif %}
 
     {% if threads %}
         {% if current_page.has_previous %}
             <div class="page_link">
                 <a href="
             {% if tag %}
                 {% url "tag" tag_name=tag.name page=current_page.previous_page_number %}
             {% else %}
                 {% url "index" page=current_page.previous_page_number %}
             {% endif %}
                 ">{% trans "Previous page" %}</a>
             </div>
         {% endif %}
 
         {% for thread in threads %}
             {% cache 600 thread_short thread.id thread.last_edit_time moderator LANGUAGE_CODE TIME_ZONE %}
             <div class="thread">
                 {% post_view thread.get_opening_post moderator is_opening=True thread=thread truncated=True need_open_link=True %}
                 {% if not thread.archived %}
                     {% with last_replies=thread.get_last_replies %}
                         {% if last_replies %}
                             {% with skipped_replies_count=thread.get_skipped_replies_count %}
                                 {% if skipped_replies_count %}
                                     <div class="skipped_replies">
                                         <a href="{% url 'thread' thread.get_opening_post_id %}">
                                             {% blocktrans with count=skipped_replies_count %}Skipped {{ count }} replies. Open thread to see all replies.{% endblocktrans %}
                                         </a>
                                     </div>
                                 {% endif %}
                             {% endwith %}
                             <div class="last-replies">
                                 {% for post in last_replies %}
                                     {% post_view post is_opening=False moderator=moderator truncated=True %}
                                 {% endfor %}
                             </div>
                         {% endif %}
                     {% endwith %}
                 {% endif %}
             </div>
             {% endcache %}
         {% endfor %}
 
         {% if current_page.has_next %}
             <div class="page_link">
                 <a href="
             {% if tag %}
                 {% url "tag" tag_name=tag.name page=current_page.next_page_number %}
             {% else %}
                 {% url "index" page=current_page.next_page_number %}
             {% endif %}
                 ">{% trans "Next page" %}</a>
             </div>
         {% endif %}
     {% else %}
         <div class="post">
             {% trans 'No threads exist. Create the first one!' %}</div>
     {% endif %}
 
     <div class="post-form-w">
         <script src="{% static 'js/panel.js' %}"></script>
         <div class="post-form">
             <div class="form-title">{% trans "Create new thread" %}</div>
             <div class="swappable-form-full">
             <form enctype="multipart/form-data" method="post" id="form">{% csrf_token %}
                 {{ form.as_div }}
                 <div class="form-submit">
                     <input type="submit" value="{% trans "Post" %}"/>
                 </div>
-                (ctrl-enter)
             </form>
             </div>
             <div>
                 {% trans 'Tags must be delimited by spaces. Text or image is required.' %}
             </div>
             <div><a href="{% url "staticpage" name="help" %}">
                 {% trans 'Text syntax' %}</a></div>
         </div>
     </div>
 
     <script src="{% static 'js/form.js' %}"></script>
 
 {% endblock %}
 
 {% block metapanel %}
 
     <span class="metapanel">
         <b><a href="{% url "authors" %}">{{ site_name }}</a> {{ version }}</b>
         {% trans "Pages:" %}
         <a href="
             {% if tag %}
                 {% url "tag" tag_name=tag.name page=paginator.page_range|first %}
             {% else %}
                 {% url "index" page=paginator.page_range|first %}
             {% endif %}
         ">&lt;&lt;</a>
         [
         {% for page in paginator.center_range %}
             <a
                 {% ifequal page current_page.number %}
                     class="current_page"
                 {% endifequal %}
                 href="
             {% if tag %}
                 {% url "tag" tag_name=tag.name page=page %}
             {% else %}
                 {% url "index" page=page %}
             {% endif %}
             ">{{ page }}</a>
             {% if not forloop.last %},{% endif %}
         {% endfor %}
         ]
         <a href="
             {% if tag %}
                 {% url "tag" tag_name=tag.name page=paginator.page_range|last %}
             {% else %}
                 {% url "index" page=paginator.page_range|last %}
             {% endif %}
         ">&gt;&gt;</a>
         [<a href="rss/">RSS</a>]
     </span>
 
 {% endblock %}

boards/tests/test_parser.py

@@ -1,34 +1,35 b''
 from django.test import TestCase
+from boards.mdx_neboard import Parser
 from boards.models import Post
 
 
 class ParserTest(TestCase):
     def test_preparse_quote(self):
         raw_text = '>quote\nQuote in >line\nLine\n>Quote'
-        preparsed_text = Post.objects._preparse_text(raw_text)
+        preparsed_text = Parser().preparse(raw_text)
 
         self.assertEqual(
             '[quote]quote[/quote]\nQuote in >line\nLine\n[quote]Quote[/quote]',
             preparsed_text, 'Quote not preparsed.')
 
     def test_preparse_comment(self):
         raw_text = '//comment'
-        preparsed_text = Post.objects._preparse_text(raw_text)
+        preparsed_text = Parser().preparse(raw_text)
 
         self.assertEqual('[comment]comment[/comment]', preparsed_text,
                          'Comment not preparsed.')
 
     def test_preparse_reflink(self):
         raw_text = '>>12\nText'
-        preparsed_text = Post.objects._preparse_text(raw_text)
+        preparsed_text = Parser().preparse(raw_text)
 
         self.assertEqual('[post]12[/post]\nText',
                          preparsed_text, 'Reflink not preparsed.')
 
     def preparse_user(self):
         raw_text = '@user\[e-mail]\n@user\nuser @user'
-        preparsed_text = Post.objects._preparse_text(raw_text)
+        preparsed_text = Parser().preparse(raw_text)
 
         self.assertEqual('[user]user[/user]\[e-mail]\n[user]user[/user]\nuser [user]user[/user]',
                          preparsed_text, 'User link not preparsed.')
 

boards/views/preview.py

@@ -1,37 +1,39 b''
 from django.shortcuts import render
 from django.template import RequestContext
 from django.views.generic import View
 
-from boards.mdx_neboard import bbcode_extended
+from boards.mdx_neboard import Parser
+
 
 FORM_QUERY = 'query'
 
 CONTEXT_RESULT = 'result'
 CONTEXT_QUERY = 'query'
 
 __author__ = 'neko259'
 
 TEMPLATE = 'boards/preview.html'
 
 
 class PostPreviewView(View):
     def get(self, request):
         context = RequestContext(request)
 
         # TODO Use dict here
         return render(request, TEMPLATE, context_instance=context)
 
     def post(self, request):
         context = RequestContext(request)
 
         if FORM_QUERY in request.POST:
             raw_text = request.POST[FORM_QUERY]
 
             if len(raw_text) >= 0:
-                rendered_text = bbcode_extended(raw_text)
+                parser = Parser()
+                rendered_text = parser.parse(parser.preparse(raw_text))
 
                 context[CONTEXT_RESULT] = rendered_text
                 context[CONTEXT_QUERY] = raw_text
 
         # TODO Use dict here
         return render(request, TEMPLATE, context_instance=context)

neboard/settings.py

@@ -1,234 +1,233 b''
 # Django settings for neboard project.
 import os
-from boards.mdx_neboard import bbcode_extended
 
 DEBUG = True
 TEMPLATE_DEBUG = DEBUG
 
 ADMINS = (
     # ('Your Name', '[e-mail]'),
     ('admin', '[e-mail]')
 )
 
 MANAGERS = ADMINS
 
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
         'NAME': 'database.db',                      # Or path to database file if using sqlite3.
         'USER': '',                      # Not used with sqlite3.
         'PASSWORD': '',                  # Not used with sqlite3.
         'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
         'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
 	'CONN_MAX_AGE': None,
     }
 }
 
 # Local time zone for this installation. Choices can be found here:
 # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
 # although not all choices may be available on all operating systems.
 # In a Windows environment this must be set to your system time zone.
 TIME_ZONE = 'Europe/Kiev'
 
 # Language code for this installation. All choices can be found here:
 # http://www.i18nguy.com/unicode/language-identifiers.html
 LANGUAGE_CODE = 'en'
 
 SITE_ID = 1
 
 # If you set this to False, Django will make some optimizations so as not
 # to load the internationalization machinery.
 USE_I18N = True
 
 # If you set this to False, Django will not format dates, numbers and
 # calendars according to the current locale.
 USE_L10N = True
 
 # If you set this to False, Django will not use timezone-aware datetimes.
 USE_TZ = True
 
 # Absolute filesystem path to the directory that will hold user-uploaded files.
 # Example: "/home/media/media.lawrence.com/media/"
 MEDIA_ROOT = './media/' 
 
 # URL that handles the media served from MEDIA_ROOT. Make sure to use a
 # trailing slash.
 # Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
 MEDIA_URL = '/media/'
 
 # Absolute path to the directory static files should be collected to.
 # Don't put anything in this directory yourself; store your static files
 # in apps' "static/" subdirectories and in STATICFILES_DIRS.
 # Example: "/home/media/media.lawrence.com/static/"
 STATIC_ROOT = ''
 
 # URL prefix for static files.
 # Example: "http://media.lawrence.com/static/"
 STATIC_URL = '/static/'
 
 # Additional locations of static files
 # It is really a hack, put real paths, not related
 STATICFILES_DIRS = (
     os.path.dirname(__file__) + '/boards/static',
 
 #    '/d/work/python/django/neboard/neboard/boards/static',
     # Put strings here, like "/home/html/static" or "C:/www/django/static".
     # Always use forward slashes, even on Windows.
     # Don't forget to use absolute paths, not relative paths.
 )
 
 # List of finder classes that know how to find static files in
 # various locations.
 STATICFILES_FINDERS = (
     'django.contrib.staticfiles.finders.FileSystemFinder',
     'django.contrib.staticfiles.finders.AppDirectoriesFinder',
     'compressor.finders.CompressorFinder',
 )
 
 if DEBUG:
     STATICFILES_STORAGE = \
         'django.contrib.staticfiles.storage.StaticFilesStorage'
 else:
     STATICFILES_STORAGE = \
         'django.contrib.staticfiles.storage.CachedStaticFilesStorage'
 
 # Make this unique, and don't share it with anybody.
 SECRET_KEY = '@1rc$o(7=tt#kd+4s$u6wchm**z^)4x90)7f6z(i&55@o11*8o'
 
 # List of callables that know how to import templates from various sources.
 TEMPLATE_LOADERS = (
     'django.template.loaders.filesystem.Loader',
     'django.template.loaders.app_directories.Loader',
 )
 
 TEMPLATE_CONTEXT_PROCESSORS = (
     'django.core.context_processors.media',
     'django.core.context_processors.static',
     'django.core.context_processors.request',
     'django.contrib.auth.context_processors.auth',
     'boards.context_processors.user_and_ui_processor',
 )
 
 MIDDLEWARE_CLASSES = (
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.locale.LocaleMiddleware',
     'django.middleware.common.CommonMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'boards.middlewares.BanMiddleware',
     'boards.middlewares.TimezoneMiddleware',
 )
 
 ROOT_URLCONF = 'neboard.urls'
 
 # Python dotted path to the WSGI application used by Django's runserver.
 WSGI_APPLICATION = 'neboard.wsgi.application'
 
 TEMPLATE_DIRS = (
     # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
     # Always use forward slashes, even on Windows.
     # Don't forget to use absolute paths, not relative paths.
     'templates',
 )
 
 INSTALLED_APPS = (
     'django.contrib.auth',
     'django.contrib.contenttypes',
     'django.contrib.sessions',
     # 'django.contrib.sites',
     'django.contrib.messages',
     'django.contrib.staticfiles',
     # Uncomment the next line to enable the admin:
     'django.contrib.admin',
     # Uncomment the next line to enable admin documentation:
     # 'django.contrib.admindocs',
     'django.contrib.humanize',
     'django_cleanup',
 
     'debug_toolbar',
 
     # Search
     'haystack',
 
     'boards',
 )
 
 # A sample logging configuration. The only tangible logging
 # performed by this configuration is to send an email to
 # the site admins on every HTTP 500 error when DEBUG=False.
 # See http://docs.djangoproject.com/en/dev/topics/logging for
 # more details on how to customize your logging configuration.
 LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,
     'formatters': {
         'verbose': {
             'format': '%(levelname)s %(asctime)s %(name)s %(process)d %(thread)d %(message)s'
         },
         'simple': {
             'format': '%(levelname)s %(asctime)s [%(name)s] %(message)s'
         },
     },
     'filters': {
         'require_debug_false': {
             '()': 'django.utils.log.RequireDebugFalse'
         }
     },
     'handlers': {
         'console': {
             'level': 'DEBUG',
             'class': 'logging.StreamHandler',
             'formatter': 'simple'
         },
     },
     'loggers': {
         'boards': {
             'handlers': ['console'],
             'level': 'DEBUG',
         }
     },
 }
 
 HAYSTACK_CONNECTIONS = {
     'default': {
         'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine',
         'PATH': os.path.join(os.path.dirname(__file__), 'whoosh_index'),
     },
 }
 
 THEMES = [
     ('md', 'Mystic Dark'),
     ('md_centered', 'Mystic Dark (centered)'),
     ('sw', 'Snow White'),
     ('pg', 'Photon Gray'),
 ]
 
 POSTING_DELAY = 20  # seconds
 
 # Websocket settins
 CENTRIFUGE_HOST = 'localhost'
 CENTRIFUGE_PORT = '9090'
 
 CENTRIFUGE_ADDRESS = 'http://{}:{}'.format(CENTRIFUGE_HOST, CENTRIFUGE_PORT)
 CENTRIFUGE_PROJECT_ID = '<project id here>'
 CENTRIFUGE_PROJECT_SECRET = '<project secret here>'
 CENTRIFUGE_TIMEOUT = 5
 
 # Debug mode middlewares
 if DEBUG:
     MIDDLEWARE_CLASSES += (
         'debug_toolbar.middleware.DebugToolbarMiddleware',
     )
 
     def custom_show_toolbar(request):
         return True
 
     DEBUG_TOOLBAR_CONFIG = {
         'ENABLE_STACKTRACES': True,
         'SHOW_TOOLBAR_CALLBACK': 'neboard.settings.custom_show_toolbar',
     }
 
     # FIXME Uncommenting this fails somehow. Need to investigate this
     #DEBUG_TOOLBAR_PANELS += (
     #    'debug_toolbar.panels.profiling.ProfilingDebugPanel',
     #)