Moved text parser and preparser to a separate module (BB-64)

boards/mdx_neboard.py

@@ -2,6 +2,9 b''
 
 import re
 import bbcode
+
+from urllib.parse import unquote
+
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.urlresolvers import reverse
 
@@ -14,6 +17,7 b' import boards'
 REFLINK_PATTERN = re.compile(r'^\d+$')
 MULTI_NEWLINES_PATTERN = re.compile(r'(\r?\n){2,}')
 ONE_NEWLINE = '\n'
+REGEX_URL = re.compile(r'https?\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(/\S*)?')
 
 
 class TextFormatter():
@@ -169,36 +173,6 b' def render_notification(tag_name, value,'
         reverse('notifications', kwargs={'username': username}), username)
 
 
-def preparse_text(text):
-    """
-    Performs manual parsing before the bbcode parser is used.
-    """
-
-    return MULTI_NEWLINES_PATTERN.sub(ONE_NEWLINE, text)
-
-
-def bbcode_extended(markup):
-    # The newline hack is added because br's margin does not work in all
-    # browsers except firefox, when the div's does.
-    parser = bbcode.Parser(newline='<div class="br"></div>')
-    parser.add_formatter('post', render_reflink, strip=True)
-    parser.add_formatter('thread', render_multithread, strip=True)
-    parser.add_formatter('quote', render_quote, strip=True)
-    parser.add_formatter('user', render_notification, strip=True)
-    parser.add_simple_formatter('comment',
-                                '<span class="comment">//%(value)s</span>')
-    parser.add_simple_formatter('spoiler',
-                                '<span class="spoiler">%(value)s</span>')
-    parser.add_simple_formatter('s',
-                                '<span class="strikethrough">%(value)s</span>')
-    # TODO Why not use built-in tag?
-    parser.add_simple_formatter('code',
-                                '<pre><code>%(value)s</pre></code>',
-                                render_embedded=False)
-
-    text = preparse_text(markup)
-    return parser.format(text)
-
 formatters = [
     QuotePattern,
     SpoilerPattern,
@@ -208,3 +182,52 b' formatters = ['
     StrikeThroughPattern,
     CodePattern,
 ]
+
+
+PREPARSE_PATTERNS = {
+    r'>>>(\d+)': r'[thread]\1[/thread]',  # Multi-thread post ">>>123"
+    r'(?<!>)>>(\d+)': r'[post]\1[/post]',  # Reflink ">>123"
+    r'^>([^>].+)': r'[quote]\1[/quote]',  # Quote ">text"
+    r'^//(.+)': r'[comment]\1[/comment]',  # Comment "//text"
+    r'\B@(\w+)': r'[user]\1[/user]',  # User notification "@user"
+}
+
+
+class Parser:
+    def __init__(self):
+        # The newline hack is added because br's margin does not work in all
+        # browsers except firefox, when the div's does.
+        self.parser = bbcode.Parser(newline='<div class="br"></div>')
+
+        self.parser.add_formatter('post', render_reflink, strip=True)
+        self.parser.add_formatter('thread', render_multithread, strip=True)
+        self.parser.add_formatter('quote', render_quote, strip=True)
+        self.parser.add_formatter('user', render_notification, strip=True)
+        self.parser.add_simple_formatter(
+            'comment', '<span class="comment">//%(value)s</span>')
+        self.parser.add_simple_formatter(
+            'spoiler', '<span class="spoiler">%(value)s</span>')
+        self.parser.add_simple_formatter(
+            's', '<span class="strikethrough">%(value)s</span>')
+        # TODO Why not use built-in tag?
+        self.parser.add_simple_formatter('code',
+                                         '<pre><code>%(value)s</pre></code>',
+                                         render_embedded=False)
+
+    def preparse(self, text):
+        """
+        Performs manual parsing before the bbcode parser is used.
+        Preparsed text is saved as raw and the text before preparsing is lost.
+        """
+        new_text = MULTI_NEWLINES_PATTERN.sub(ONE_NEWLINE, text)
+
+        for key, value in PREPARSE_PATTERNS.items():
+            new_text = re.sub(key, value, new_text, flags=re.MULTILINE)
+
+        for link in REGEX_URL.findall(text):
+            new_text = new_text.replace(link, unquote(link))
+
+        return new_text
+
+    def parse(self, text):
+        return self.parser.format(text)
  No newline at end of file

boards/models/post.py

@@ -3,8 +3,6 b' from datetime import time as dtime'
 import logging
 import re
 
-from urllib.parse import unquote
-
 from adjacent import Client
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.urlresolvers import reverse
@@ -14,7 +12,7 b' from django.template.loader import rende'
 from django.utils import timezone
 
 from boards import settings
-from boards.mdx_neboard import bbcode_extended
+from boards.mdx_neboard import Parser
 from boards.models import PostImage
 from boards.models.base import Viewable
 from boards.utils import datetime_to_epoch, cached_result
@@ -45,7 +43,6 b" UNKNOWN_UA = ''"
 
 REGEX_REPLY = re.compile(r'\[post\](\d+)\[/post\]')
 REGEX_MULTI_THREAD = re.compile(r'\[thread\](\d+)\[/thread\]')
-REGEX_URL = re.compile(r'https?\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(/\S*)?')
 REGEX_NOTIFICATION = re.compile(r'\[user\](\w+)\[/user\]')
 
 PARAMETER_TRUNCATED = 'truncated'
@@ -64,14 +61,6 b" PARAMETER_REPLY_LINK = 'reply_link'"
 DIFF_TYPE_HTML = 'html'
 DIFF_TYPE_JSON = 'json'
 
-PREPARSE_PATTERNS = {
-    r'>>>(\d+)': r'[thread]\1[/thread]',  # Multi-thread post ">>>123"
-    r'(?<!>)>>(\d+)': r'[post]\1[/post]',  # Reflink ">>123"
-    r'^>([^>].+)': r'[quote]\1[/quote]',  # Quote ">text"
-    r'^//(.+)': r'[comment]\1[/comment]',  # Comment "//text"
-    r'\B@(\w+)': r'[user]\1[/user]',  # User notification "@user"
-}
-
 
 class PostManager(models.Manager):
     @transaction.atomic
@@ -92,7 +81,7 b' class PostManager(models.Manager):'
         else:
             new_thread = False
 
-        pre_text = self._preparse_text(text)
+        pre_text = Parser().preparse(text)
 
         post = self.create(title=title,
                            text=pre_text,
@@ -158,21 +147,6 b' class PostManager(models.Manager):'
 
         return ppd
     
-    # TODO Make a separate parser module and move preparser there
-    def _preparse_text(self, text: str) -> str:
-        """
-        Preparses text to change patterns like '>>' to a proper bbcode
-        tags.
-        """
-
-        for key, value in PREPARSE_PATTERNS.items():
-            text = re.sub(key, value, text, flags=re.MULTILINE)
-
-        for link in REGEX_URL.findall(text):
-            text = text.replace(link, unquote(link))
-
-        return text
-
 
 class Post(models.Model, Viewable):
     """A post is a message."""
@@ -398,7 +372,7 b' class Post(models.Model, Viewable):'
 
     def save(self, force_insert=False, force_update=False, using=None,
              update_fields=None):
-        self._text_rendered = bbcode_extended(self.get_raw_text())
+        self._text_rendered = Parser().parse(self.get_raw_text())
 
         super().save(force_insert, force_update, using, update_fields)
 

boards/templates/boards/posting_general.html

@@ -136,7 +136,6 b''
                 <div class="form-submit">
                     <input type="submit" value="{% trans "Post" %}"/>
                 </div>
-                (ctrl-enter)
             </form>
             </div>
             <div>

boards/tests/test_parser.py

@@ -1,11 +1,12 b''
 from django.test import TestCase
+from boards.mdx_neboard import Parser
 from boards.models import Post
 
 
 class ParserTest(TestCase):
     def test_preparse_quote(self):
         raw_text = '>quote\nQuote in >line\nLine\n>Quote'
-        preparsed_text = Post.objects._preparse_text(raw_text)
+        preparsed_text = Parser().preparse(raw_text)
 
         self.assertEqual(
             '[quote]quote[/quote]\nQuote in >line\nLine\n[quote]Quote[/quote]',
@@ -13,21 +14,21 b' class ParserTest(TestCase):'
 
     def test_preparse_comment(self):
         raw_text = '//comment'
-        preparsed_text = Post.objects._preparse_text(raw_text)
+        preparsed_text = Parser().preparse(raw_text)
 
         self.assertEqual('[comment]comment[/comment]', preparsed_text,
                          'Comment not preparsed.')
 
     def test_preparse_reflink(self):
         raw_text = '>>12\nText'
-        preparsed_text = Post.objects._preparse_text(raw_text)
+        preparsed_text = Parser().preparse(raw_text)
 
         self.assertEqual('[post]12[/post]\nText',
                          preparsed_text, 'Reflink not preparsed.')
 
     def preparse_user(self):
         raw_text = '@user\[e-mail]\n@user\nuser @user'
-        preparsed_text = Post.objects._preparse_text(raw_text)
+        preparsed_text = Parser().preparse(raw_text)
 
         self.assertEqual('[user]user[/user]\[e-mail]\n[user]user[/user]\nuser [user]user[/user]',
                          preparsed_text, 'User link not preparsed.')

boards/views/preview.py

@@ -2,7 +2,8 b' from django.shortcuts import render'
 from django.template import RequestContext
 from django.views.generic import View
 
-from boards.mdx_neboard import bbcode_extended
+from boards.mdx_neboard import Parser
+
 
 FORM_QUERY = 'query'
 
@@ -28,7 +29,8 b' class PostPreviewView(View):'
             raw_text = request.POST[FORM_QUERY]
 
             if len(raw_text) >= 0:
-                rendered_text = bbcode_extended(raw_text)
+                parser = Parser()
+                rendered_text = parser.parse(parser.preparse(raw_text))
 
                 context[CONTEXT_RESULT] = rendered_text
                 context[CONTEXT_QUERY] = raw_text

neboard/settings.py

@@ -1,6 +1,5 b''
 # Django settings for neboard project.
 import os
-from boards.mdx_neboard import bbcode_extended
 
 DEBUG = True
 TEMPLATE_DEBUG = DEBUG