diff --git a/boards/management/commands/sync_with_server.py b/boards/management/commands/sync_with_server.py new file mode 100644 --- /dev/null +++ b/boards/management/commands/sync_with_server.py @@ -0,0 +1,37 @@ +import re +import urllib +import httplib2 +from django.core.management import BaseCommand +from boards.models import GlobalId + +__author__ = 'neko259' + + +REGEX_GLOBAL_ID = r'\[(\w+)\]\[(\w+)\]\[(\d+)\]' + + +class Command(BaseCommand): + help = 'Send a sync or get request to the server.' + \ + 'sync_with_server [post_global_id]' + + def handle(self, *args, **options): + url = args[0] + if len(args) > 1: + global_id_str = args[1] + match = re.match(REGEX_GLOBAL_ID, global_id_str) + key_type = match.group(1) + key = match.group(2) + local_id = match.group(3) + + global_id = GlobalId(key_type=key_type, key=key, + local_id=local_id) + + xml = GlobalId.objects.generate_request_get([global_id]) + data = {'xml': xml} + body = urllib.urlencode(data) + h = httplib2.Http() + response, content = h.request(url, method="POST", body=body) + + # TODO Parse content and get the model list + else: + raise Exception('Full sync is not supported yet.') diff --git a/boards/mdx_neboard.py b/boards/mdx_neboard.py --- a/boards/mdx_neboard.py +++ b/boards/mdx_neboard.py @@ -10,6 +10,7 @@ import boards REFLINK_PATTERN = re.compile(r'^\d+$') +GLOBAL_REFLINK_PATTERN = re.compile(r'^(\w+)::([^:]+)::(\d+)$') MULTI_NEWLINES_PATTERN = re.compile(r'(\r?\n){2,}') ONE_NEWLINE = '\n' @@ -115,10 +116,28 @@ class CodePattern(TextFormatter): def render_reflink(tag_name, value, options, parent, context): - if not REFLINK_PATTERN.match(value): - return '>>%s' % value + post_id = None - post_id = int(value) + matches = REFLINK_PATTERN.findall(value) + if matches: + post_id = int(matches[0][0]) + else: + match = GLOBAL_REFLINK_PATTERN.match(value) + if match: + key_type = match.group(1) + key = match.group(2) + local_id = match.group(3) + + try: + global_id = boards.models.GlobalId.objects.get(key_type=key_type, + key=key, local_id=local_id) + for post in boards.models.Post.objects.filter(global_id=global_id).only('id'): + post_id = post.id + except boards.models.GlobalId.DoesNotExist: + pass + + if not post_id: + return value posts = boards.models.Post.objects.filter(id=post_id) if posts.exists(): diff --git a/boards/models/post.py b/boards/models/post.py --- a/boards/models/post.py +++ b/boards/models/post.py @@ -17,6 +17,7 @@ from boards.models.base import Viewable from boards.models.thread import Thread from boards import utils +ENCODING_UNICODE = 'unicode' APP_LABEL_BOARDS = 'boards' @@ -40,6 +41,7 @@ NO_IP = '0.0.0.0' UNKNOWN_UA = '' REGEX_REPLY = re.compile(r'\[post\](\d+)\[/post\]') +REGEX_GLOBAL_REPLY = re.compile(r'\[post\](\w+)::([^:]+)::(\d+)\[/post\]') TAG_MODEL = 'model' TAG_REQUEST = 'request' @@ -51,9 +53,6 @@ TAG_TITLE = 'title' TAG_TEXT = 'text' TAG_THREAD = 'thread' TAG_PUB_TIME = 'pub-time' -TAG_EDIT_TIME = 'edit-time' -TAG_PREVIOUS = 'previous' -TAG_NEXT = 'next' TAG_SIGNATURES = 'signatures' TAG_SIGNATURE = 'signature' TAG_CONTENT = 'content' @@ -151,6 +150,7 @@ class PostManager(models.Manager): for post in posts: self.delete_post(post) + # TODO This can be moved into a post def connect_replies(self, post): """ Connects replies to a post to show them as a reflink map @@ -196,25 +196,7 @@ class PostManager(models.Manager): cache.set(cache_key, ppd) return ppd - def generate_request_get(self, model_list: list): - """ - Form a get request from a list of ModelId objects. - """ - - request = et.Element(TAG_REQUEST) - request.set(ATTR_TYPE, TYPE_GET) - request.set(ATTR_VERSION, '1.0') - - model = et.SubElement(request, TAG_MODEL) - model.set(ATTR_VERSION, '1.0') - model.set(ATTR_NAME, 'post') - - for post in model_list: - tag_id = et.SubElement(model, TAG_ID) - post.global_id.to_xml_element(tag_id) - - return et.tostring(request, 'unicode') - + # TODO Make a separate sync facade? def generate_response_get(self, model_list: list): response = et.Element(TAG_RESPONSE) @@ -236,6 +218,7 @@ class PostManager(models.Manager): title.text = post.title text = et.SubElement(content_tag, TAG_TEXT) + # TODO Replace local links by global ones in the text text.text = post.text.raw if not post.is_opening(): @@ -260,14 +243,42 @@ class PostManager(models.Manager): signatures = [Signature( key_type=key.key_type, key=key.public_key, - signature=key.sign(et.tostring(model, 'unicode')), + signature=key.sign(et.tostring(model, ENCODING_UNICODE)), )] for signature in signatures: signature_tag = et.SubElement(signatures_tag, TAG_SIGNATURE) signature_tag.set(ATTR_TYPE, signature.key_type) signature_tag.set(ATTR_VALUE, signature.signature) - return et.tostring(response, 'unicode') + return et.tostring(response, ENCODING_UNICODE) + + def parse_response_get(self, response_xml): + tag_root = et.fromstring(response_xml) + tag_status = tag_root[0] + if 'success' == tag_status.text: + tag_models = tag_root[1] + for tag_model in tag_models: + tag_content = tag_model[0] + tag_id = tag_content[1] + try: + GlobalId.from_xml_element(tag_id, existing=True) + # If this post already exists, just continue + # TODO Compare post content and update the post if necessary + pass + except GlobalId.DoesNotExist: + global_id = GlobalId.from_xml_element(tag_id) + + title = tag_content.find(TAG_TITLE).text + text = tag_content.find(TAG_TEXT).text + # TODO Check that the replied posts are already present + # before adding new ones + + # TODO Pub time, thread, tags + + post = Post.objects.create(title=title, text=text) + else: + # TODO Throw an exception? + pass class Post(models.Model, Viewable): @@ -316,15 +327,7 @@ class Post(models.Model, Viewable): self.text.raw[:50] + ')' def get_title(self): - """ - Gets original post title or part of its text. - """ - - title = self.title - if not title: - title = self.text.rendered - - return title + return self.title def build_refmap(self): """ @@ -491,8 +494,25 @@ class Post(models.Model, Viewable): def get_pub_time_epoch(self): return utils.datetime_to_epoch(self.pub_time) - def get_edit_time_epoch(self): - return utils.datetime_to_epoch(self.last_edit_time) + def get_replied_ids(self): + """ + Gets ID list of the posts that this post replies. + """ - def get_replied_ids(self): - return re.findall(REGEX_REPLY, self.text.raw) + local_replied = REGEX_REPLY.findall(self.text.raw) + global_replied = [] + # TODO Similar code is used in mdx_neboard, maybe it can be extracted + # into a method? + for match in REGEX_GLOBAL_REPLY.findall(self.text.raw): + key_type = match[0] + key = match[1] + local_id = match[2] + + try: + global_id = GlobalId.objects.get(key_type=key_type, + key=key, local_id=local_id) + for post in Post.objects.filter(global_id=global_id).only('id'): + global_replied.append(post.id) + except GlobalId.DoesNotExist: + pass + return local_replied + global_replied diff --git a/boards/models/signature.py b/boards/models/signature.py --- a/boards/models/signature.py +++ b/boards/models/signature.py @@ -2,15 +2,48 @@ import xml.etree.ElementTree as et from django.db import models +TAG_MODEL = 'model' +TAG_REQUEST = 'request' +TAG_ID = 'id' + +TYPE_GET = 'get' + +ATTR_VERSION = 'version' +ATTR_TYPE = 'type' +ATTR_NAME = 'name' + ATTR_KEY = 'key' ATTR_KEY_TYPE = 'type' ATTR_LOCAL_ID = 'local-id' +class GlobalIdManager(models.Manager): + def generate_request_get(self, global_id_list: list): + """ + Form a get request from a list of ModelId objects. + """ + + request = et.Element(TAG_REQUEST) + request.set(ATTR_TYPE, TYPE_GET) + request.set(ATTR_VERSION, '1.0') + + model = et.SubElement(request, TAG_MODEL) + model.set(ATTR_VERSION, '1.0') + model.set(ATTR_NAME, 'post') + + for global_id in global_id_list: + tag_id = et.SubElement(model, TAG_ID) + global_id.to_xml_element(tag_id) + + return et.tostring(request, 'unicode') + + class GlobalId(models.Model): class Meta: app_label = 'boards' + objects = GlobalIdManager() + def __init__(self, *args, **kwargs): models.Model.__init__(self, *args, **kwargs) @@ -24,7 +57,7 @@ class GlobalId(models.Model): local_id = models.IntegerField() def __str__(self): - return '[%s][%s][%d]' % (self.key_type, self.key, self.local_id) + return '%s::%s::%d' % (self.key_type, self.key, self.local_id) def to_xml_element(self, element: et.Element): """ diff --git a/boards/models/sync_key.py b/boards/models/sync_key.py --- a/boards/models/sync_key.py +++ b/boards/models/sync_key.py @@ -52,7 +52,7 @@ class KeyPair(models.Model): primary = models.BooleanField(default=False) def __str__(self): - return '[%s][%s]' % (self.key_type, self.public_key) + return '%s::%s' % (self.key_type, self.public_key) def sign(self, string): private = SigningKey.from_string(base64.b64decode( diff --git a/boards/tests/test_keys.py b/boards/tests/test_keys.py --- a/boards/tests/test_keys.py +++ b/boards/tests/test_keys.py @@ -36,7 +36,7 @@ class KeyTest(TestCase): def test_request_get(self): post = self._create_post_with_key() - request = Post.objects.generate_request_get([post]) + request = GlobalId.objects.generate_request_get([post.global_id]) logger.debug(request) key = KeyPair.objects.get(primary=True) diff --git a/boards/tests/test_post.py b/boards/tests/test_post.py --- a/boards/tests/test_post.py +++ b/boards/tests/test_post.py @@ -1,7 +1,7 @@ from django.core.paginator import Paginator from django.test import TestCase from boards import settings -from boards.models import Tag, Post, Thread +from boards.models import Tag, Post, Thread, KeyPair class PostTests(TestCase): @@ -109,4 +109,34 @@ class PostTests(TestCase): first_post = posts_in_second_page[0] self.assertEqual(all_threads[settings.THREADS_PER_PAGE].id, - first_post.id) \ No newline at end of file + first_post.id) + + def test_reflinks(self): + """ + Tests that reflinks are parsed within post and connecting replies + to the replied posts. + + Local reflink example: [post]123[/post] + Global reflink example: [post]key_type::key::123[/post] + """ + + key = KeyPair.objects.generate_key(primary=True) + + tag = Tag.objects.create(name='test_tag') + + post = Post.objects.create_post(title='', text='', tags=[tag]) + post_local_reflink = Post.objects.create_post(title='', + text='[post]%d[/post]' % post.id, thread=post.get_thread()) + + self.assertTrue(post_local_reflink in post.referenced_posts.all(), + 'Local reflink not connecting posts.') + + post_global_reflink = Post.objects.create_post(title='', + text='[post]%s::%s::%d[/post]' % ( + post.global_id.key_type, post.global_id.key, post.id), + thread=post.get_thread()) + + self.assertTrue(post_global_reflink in post.referenced_posts.all(), + 'Global reflink not connecting posts.') + + # TODO Check that links are parsed into the rendered text diff --git a/development/run.sh b/development/run.sh new file mode 100755 --- /dev/null +++ b/development/run.sh @@ -0,0 +1,3 @@ +#! /usr/bin/env sh + +python3 manage.py runserver [::]:8000 diff --git a/development/test.sh b/development/test.sh new file mode 100755 --- /dev/null +++ b/development/test.sh @@ -0,0 +1,3 @@ +#! /usr/bin/env sh + +python3 manage.py test diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +httplib2 simplejson south>=0.8.4 haystack diff --git a/todo.txt b/todo.txt --- a/todo.txt +++ b/todo.txt @@ -15,6 +15,7 @@ * Subscribing to tag via AJAX * Add buttons to insert a named link or a named quote to the markup panel * Add support for "attention posts" that are shown in the header" +* Use absolute post reflinks in the raw text = Bugs = * Search sort order is confusing