##// END OF EJS Templates
Merged the decentral branch into default branch
neko259 -
r1546:80ed7ef8 merge default
parent child Browse files
Show More
@@ -0,0 +1,11 b''
1 from django.apps import AppConfig
2
3
4 class BoardsAppConfig(AppConfig):
5 name = 'boards'
6 verbose_name = 'Boards'
7
8 def ready(self):
9 super().ready()
10
11 import boards.signals No newline at end of file
@@ -0,0 +1,21 b''
1 __author__ = 'neko259'
2
3
4 from django.core.management import BaseCommand
5 from django.db import transaction
6
7 from boards.models import KeyPair, Post
8
9
10 class Command(BaseCommand):
11 help = 'Generates the new keypair. The first one will be primary.'
12
13 @transaction.atomic
14 def handle(self, *args, **options):
15 first_key = not KeyPair.objects.has_primary()
16 key = KeyPair.objects.generate_key(
17 primary=first_key)
18 print(key)
19
20 for post in Post.objects.filter(global_id=None):
21 post.set_global_id()
@@ -0,0 +1,80 b''
1 import re
2 import xml.etree.ElementTree as ET
3
4 import httplib2
5 from django.core.management import BaseCommand
6
7 from boards.models import GlobalId
8 from boards.models.post.sync import SyncManager
9
10 __author__ = 'neko259'
11
12
13 REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)')
14
15
16 class Command(BaseCommand):
17 help = 'Send a sync or get request to the server.'
18
19 def add_arguments(self, parser):
20 parser.add_argument('url', type=str, help='Server root url')
21 parser.add_argument('--global-id', type=str, default='',
22 help='Post global ID')
23
24 def handle(self, *args, **options):
25 url = options.get('url')
26
27 pull_url = url + 'api/sync/pull/'
28 get_url = url + 'api/sync/get/'
29 file_url = url[:-1]
30
31 global_id_str = options.get('global_id')
32 if global_id_str:
33 match = REGEX_GLOBAL_ID.match(global_id_str)
34 if match:
35 key_type = match.group(1)
36 key = match.group(2)
37 local_id = match.group(3)
38
39 global_id = GlobalId(key_type=key_type, key=key,
40 local_id=local_id)
41
42 xml = GlobalId.objects.generate_request_get([global_id])
43 # body = urllib.parse.urlencode(data)
44 h = httplib2.Http()
45 response, content = h.request(get_url, method="POST", body=xml)
46
47 SyncManager.parse_response_get(content, file_url)
48 else:
49 raise Exception('Invalid global ID')
50 else:
51 h = httplib2.Http()
52 xml = GlobalId.objects.generate_request_pull()
53 response, content = h.request(pull_url, method="POST", body=xml)
54
55 print(content.decode() + '\n')
56
57 root = ET.fromstring(content)
58 status = root.findall('status')[0].text
59 if status == 'success':
60 ids_to_sync = list()
61
62 models = root.findall('models')[0]
63 for model in models:
64 global_id, exists = GlobalId.from_xml_element(model)
65 if not exists:
66 print(global_id)
67 ids_to_sync.append(global_id)
68 print()
69
70 if len(ids_to_sync) > 0:
71 xml = GlobalId.objects.generate_request_get(ids_to_sync)
72 # body = urllib.parse.urlencode(data)
73 h = httplib2.Http()
74 response, content = h.request(get_url, method="POST", body=xml)
75
76 SyncManager.parse_response_get(content, file_url)
77 else:
78 print('Nothing to get, everything synced')
79 else:
80 raise Exception('Invalid response status')
@@ -0,0 +1,48 b''
1 # -*- coding: utf-8 -*-
2 from __future__ import unicode_literals
3
4 from django.db import models, migrations
5
6
7 class Migration(migrations.Migration):
8
9 dependencies = [
10 ('boards', '0025_auto_20150825_2049'),
11 ]
12
13 operations = [
14 migrations.CreateModel(
15 name='GlobalId',
16 fields=[
17 ('id', models.AutoField(serialize=False, verbose_name='ID', primary_key=True, auto_created=True)),
18 ('key', models.TextField()),
19 ('key_type', models.TextField()),
20 ('local_id', models.IntegerField()),
21 ],
22 ),
23 migrations.CreateModel(
24 name='KeyPair',
25 fields=[
26 ('id', models.AutoField(serialize=False, verbose_name='ID', primary_key=True, auto_created=True)),
27 ('public_key', models.TextField()),
28 ('private_key', models.TextField()),
29 ('key_type', models.TextField()),
30 ('primary', models.BooleanField(default=False)),
31 ],
32 ),
33 migrations.CreateModel(
34 name='Signature',
35 fields=[
36 ('id', models.AutoField(serialize=False, verbose_name='ID', primary_key=True, auto_created=True)),
37 ('key_type', models.TextField()),
38 ('key', models.TextField()),
39 ('signature', models.TextField()),
40 ('global_id', models.ForeignKey(to='boards.GlobalId')),
41 ],
42 ),
43 migrations.AddField(
44 model_name='post',
45 name='global_id',
46 field=models.OneToOneField(to='boards.GlobalId', null=True, blank=True),
47 ),
48 ]
@@ -0,0 +1,15 b''
1 # -*- coding: utf-8 -*-
2 from __future__ import unicode_literals
3
4 from django.db import models, migrations
5
6
7 class Migration(migrations.Migration):
8
9 dependencies = [
10 ('boards', '0030_auto_20150929_1816'),
11 ('boards', '0026_auto_20150830_2006'),
12 ]
13
14 operations = [
15 ]
@@ -0,0 +1,15 b''
1 # -*- coding: utf-8 -*-
2 from __future__ import unicode_literals
3
4 from django.db import migrations, models
5
6
7 class Migration(migrations.Migration):
8
9 dependencies = [
10 ('boards', '0031_merge'),
11 ('boards', '0035_auto_20151021_1346'),
12 ]
13
14 operations = [
15 ]
@@ -0,0 +1,15 b''
1 # -*- coding: utf-8 -*-
2 from __future__ import unicode_literals
3
4 from django.db import migrations, models
5
6
7 class Migration(migrations.Migration):
8
9 dependencies = [
10 ('boards', '0040_thread_monochrome'),
11 ('boards', '0036_merge'),
12 ]
13
14 operations = [
15 ]
@@ -0,0 +1,16 b''
1 # -*- coding: utf-8 -*-
2 # Generated by Django 1.9.5 on 2016-04-29 15:52
3 from __future__ import unicode_literals
4
5 from django.db import migrations
6
7
8 class Migration(migrations.Migration):
9
10 dependencies = [
11 ('boards', '0041_merge'),
12 ('boards', '0042_auto_20160422_1053'),
13 ]
14
15 operations = [
16 ]
@@ -0,0 +1,20 b''
1 # -*- coding: utf-8 -*-
2 # Generated by Django 1.9.5 on 2016-05-04 15:36
3 from __future__ import unicode_literals
4
5 from django.db import migrations, models
6
7
8 class Migration(migrations.Migration):
9
10 dependencies = [
11 ('boards', '0043_merge'),
12 ]
13
14 operations = [
15 migrations.AddField(
16 model_name='globalid',
17 name='content',
18 field=models.TextField(blank=True, null=True),
19 ),
20 ]
@@ -0,0 +1,270 b''
1 import xml.etree.ElementTree as et
2
3 from boards.models.attachment.downloaders import download
4 from boards.utils import get_file_mimetype, get_file_hash
5 from django.db import transaction
6 from boards.models import KeyPair, GlobalId, Signature, Post, Tag
7
8 EXCEPTION_NODE = 'Sync node returned an error: {}'
9 EXCEPTION_OP = 'Load the OP first'
10 EXCEPTION_DOWNLOAD = 'File was not downloaded'
11 EXCEPTION_HASH = 'File hash does not match attachment hash'
12 EXCEPTION_SIGNATURE = 'Invalid model signature for {}'
13 ENCODING_UNICODE = 'unicode'
14
15 TAG_MODEL = 'model'
16 TAG_REQUEST = 'request'
17 TAG_RESPONSE = 'response'
18 TAG_ID = 'id'
19 TAG_STATUS = 'status'
20 TAG_MODELS = 'models'
21 TAG_TITLE = 'title'
22 TAG_TEXT = 'text'
23 TAG_THREAD = 'thread'
24 TAG_PUB_TIME = 'pub-time'
25 TAG_SIGNATURES = 'signatures'
26 TAG_SIGNATURE = 'signature'
27 TAG_CONTENT = 'content'
28 TAG_ATTACHMENTS = 'attachments'
29 TAG_ATTACHMENT = 'attachment'
30 TAG_TAGS = 'tags'
31 TAG_TAG = 'tag'
32 TAG_ATTACHMENT_REFS = 'attachment-refs'
33 TAG_ATTACHMENT_REF = 'attachment-ref'
34
35 TYPE_GET = 'get'
36
37 ATTR_VERSION = 'version'
38 ATTR_TYPE = 'type'
39 ATTR_NAME = 'name'
40 ATTR_VALUE = 'value'
41 ATTR_MIMETYPE = 'mimetype'
42 ATTR_KEY = 'key'
43 ATTR_REF = 'ref'
44 ATTR_URL = 'url'
45
46 STATUS_SUCCESS = 'success'
47
48
49 class SyncException(Exception):
50 pass
51
52
53 class SyncManager:
54 @staticmethod
55 def generate_response_get(model_list: list):
56 response = et.Element(TAG_RESPONSE)
57
58 status = et.SubElement(response, TAG_STATUS)
59 status.text = STATUS_SUCCESS
60
61 models = et.SubElement(response, TAG_MODELS)
62
63 for post in model_list:
64 model = et.SubElement(models, TAG_MODEL)
65 model.set(ATTR_NAME, 'post')
66
67 global_id = post.global_id
68
69 images = post.images.all()
70 attachments = post.attachments.all()
71 if global_id.content:
72 model.append(et.fromstring(global_id.content))
73 if len(images) > 0 or len(attachments) > 0:
74 attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS)
75 for image in images:
76 SyncManager._attachment_to_xml(
77 None, attachment_refs, image.image.file,
78 image.hash, image.image.url)
79 for file in attachments:
80 SyncManager._attachment_to_xml(
81 None, attachment_refs, file.file.file,
82 file.hash, file.file.url)
83 else:
84 content_tag = et.SubElement(model, TAG_CONTENT)
85
86 tag_id = et.SubElement(content_tag, TAG_ID)
87 global_id.to_xml_element(tag_id)
88
89 title = et.SubElement(content_tag, TAG_TITLE)
90 title.text = post.title
91
92 text = et.SubElement(content_tag, TAG_TEXT)
93 text.text = post.get_sync_text()
94
95 thread = post.get_thread()
96 if post.is_opening():
97 tag_tags = et.SubElement(content_tag, TAG_TAGS)
98 for tag in thread.get_tags():
99 tag_tag = et.SubElement(tag_tags, TAG_TAG)
100 tag_tag.text = tag.name
101 else:
102 tag_thread = et.SubElement(content_tag, TAG_THREAD)
103 thread_id = et.SubElement(tag_thread, TAG_ID)
104 thread.get_opening_post().global_id.to_xml_element(thread_id)
105
106 pub_time = et.SubElement(content_tag, TAG_PUB_TIME)
107 pub_time.text = str(post.get_pub_time_str())
108
109 if len(images) > 0 or len(attachments) > 0:
110 attachments_tag = et.SubElement(content_tag, TAG_ATTACHMENTS)
111 attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS)
112
113 for image in images:
114 SyncManager._attachment_to_xml(
115 attachments_tag, attachment_refs, image.image.file,
116 image.hash, image.image.url)
117 for file in attachments:
118 SyncManager._attachment_to_xml(
119 attachments_tag, attachment_refs, file.file.file,
120 file.hash, file.file.url)
121
122 global_id.content = et.tostring(content_tag, ENCODING_UNICODE)
123 global_id.save()
124
125 signatures_tag = et.SubElement(model, TAG_SIGNATURES)
126 post_signatures = global_id.signature_set.all()
127 if post_signatures:
128 signatures = post_signatures
129 else:
130 key = KeyPair.objects.get(public_key=global_id.key)
131 signature = Signature(
132 key_type=key.key_type,
133 key=key.public_key,
134 signature=key.sign(global_id.content),
135 global_id=global_id,
136 )
137 signature.save()
138 signatures = [signature]
139 for signature in signatures:
140 signature_tag = et.SubElement(signatures_tag, TAG_SIGNATURE)
141 signature_tag.set(ATTR_TYPE, signature.key_type)
142 signature_tag.set(ATTR_VALUE, signature.signature)
143 signature_tag.set(ATTR_KEY, signature.key)
144
145 return et.tostring(response, ENCODING_UNICODE)
146
147 @staticmethod
148 @transaction.atomic
149 def parse_response_get(response_xml, hostname):
150 tag_root = et.fromstring(response_xml)
151 tag_status = tag_root.find(TAG_STATUS)
152 if STATUS_SUCCESS == tag_status.text:
153 tag_models = tag_root.find(TAG_MODELS)
154 for tag_model in tag_models:
155 tag_content = tag_model.find(TAG_CONTENT)
156
157 content_str = et.tostring(tag_content, ENCODING_UNICODE)
158 signatures = SyncManager._verify_model(content_str, tag_model)
159
160 tag_id = tag_content.find(TAG_ID)
161 global_id, exists = GlobalId.from_xml_element(tag_id)
162
163 if exists:
164 print('Post with same ID already exists')
165 else:
166 global_id.content = content_str
167 global_id.save()
168 for signature in signatures:
169 signature.global_id = global_id
170 signature.save()
171
172 title = tag_content.find(TAG_TITLE).text or ''
173 text = tag_content.find(TAG_TEXT).text or ''
174 pub_time = tag_content.find(TAG_PUB_TIME).text
175
176 thread = tag_content.find(TAG_THREAD)
177 tags = []
178 if thread:
179 thread_id = thread.find(TAG_ID)
180 op_global_id, exists = GlobalId.from_xml_element(thread_id)
181 if exists:
182 opening_post = Post.objects.get(global_id=op_global_id)
183 else:
184 raise SyncException(EXCEPTION_OP)
185 else:
186 opening_post = None
187 tag_tags = tag_content.find(TAG_TAGS)
188 for tag_tag in tag_tags:
189 tag, created = Tag.objects.get_or_create(
190 name=tag_tag.text)
191 tags.append(tag)
192
193 # TODO Check that the replied posts are already present
194 # before adding new ones
195
196 files = []
197 tag_attachments = tag_content.find(TAG_ATTACHMENTS) or list()
198 tag_refs = tag_model.find(TAG_ATTACHMENT_REFS)
199 for attachment in tag_attachments:
200 tag_ref = tag_refs.find("{}[@ref='{}']".format(
201 TAG_ATTACHMENT_REF, attachment.text))
202 url = tag_ref.get(ATTR_URL)
203 attached_file = download(hostname + url)
204 if attached_file is None:
205 raise SyncException(EXCEPTION_DOWNLOAD)
206
207 hash = get_file_hash(attached_file)
208 if hash != attachment.text:
209 raise SyncException(EXCEPTION_HASH)
210
211 files.append(attached_file)
212
213 Post.objects.import_post(
214 title=title, text=text, pub_time=pub_time,
215 opening_post=opening_post, tags=tags,
216 global_id=global_id, files=files)
217 else:
218 raise SyncException(EXCEPTION_NODE.format(tag_status.text))
219
220 @staticmethod
221 def generate_response_pull():
222 response = et.Element(TAG_RESPONSE)
223
224 status = et.SubElement(response, TAG_STATUS)
225 status.text = STATUS_SUCCESS
226
227 models = et.SubElement(response, TAG_MODELS)
228
229 for post in Post.objects.all():
230 tag_id = et.SubElement(models, TAG_ID)
231 post.global_id.to_xml_element(tag_id)
232
233 return et.tostring(response, ENCODING_UNICODE)
234
235 @staticmethod
236 def _verify_model(content_str, tag_model):
237 """
238 Verifies all signatures for a single model.
239 """
240
241 signatures = []
242
243 tag_signatures = tag_model.find(TAG_SIGNATURES)
244 for tag_signature in tag_signatures:
245 signature_type = tag_signature.get(ATTR_TYPE)
246 signature_value = tag_signature.get(ATTR_VALUE)
247 signature_key = tag_signature.get(ATTR_KEY)
248
249 signature = Signature(key_type=signature_type,
250 key=signature_key,
251 signature=signature_value)
252
253 if not KeyPair.objects.verify(signature, content_str):
254 raise SyncException(EXCEPTION_SIGNATURE.format(content_str))
255
256 signatures.append(signature)
257
258 return signatures
259
260 @staticmethod
261 def _attachment_to_xml(tag_attachments, tag_refs, file, hash, url):
262 if tag_attachments is not None:
263 mimetype = get_file_mimetype(file)
264 attachment = et.SubElement(tag_attachments, TAG_ATTACHMENT)
265 attachment.set(ATTR_MIMETYPE, mimetype)
266 attachment.text = hash
267
268 attachment_ref = et.SubElement(tag_refs, TAG_ATTACHMENT_REF)
269 attachment_ref.set(ATTR_REF, hash)
270 attachment_ref.set(ATTR_URL, url)
@@ -0,0 +1,144 b''
1 import xml.etree.ElementTree as et
2 from django.db import models
3
4
5 TAG_MODEL = 'model'
6 TAG_REQUEST = 'request'
7 TAG_ID = 'id'
8
9 TYPE_GET = 'get'
10 TYPE_PULL = 'pull'
11
12 ATTR_VERSION = 'version'
13 ATTR_TYPE = 'type'
14 ATTR_NAME = 'name'
15
16 ATTR_KEY = 'key'
17 ATTR_KEY_TYPE = 'type'
18 ATTR_LOCAL_ID = 'local-id'
19
20
21 class GlobalIdManager(models.Manager):
22 def generate_request_get(self, global_id_list: list):
23 """
24 Form a get request from a list of ModelId objects.
25 """
26
27 request = et.Element(TAG_REQUEST)
28 request.set(ATTR_TYPE, TYPE_GET)
29 request.set(ATTR_VERSION, '1.0')
30
31 model = et.SubElement(request, TAG_MODEL)
32 model.set(ATTR_VERSION, '1.0')
33 model.set(ATTR_NAME, 'post')
34
35 for global_id in global_id_list:
36 tag_id = et.SubElement(model, TAG_ID)
37 global_id.to_xml_element(tag_id)
38
39 return et.tostring(request, 'unicode')
40
41 def generate_request_pull(self):
42 """
43 Form a pull request from a list of ModelId objects.
44 """
45
46 request = et.Element(TAG_REQUEST)
47 request.set(ATTR_TYPE, TYPE_PULL)
48 request.set(ATTR_VERSION, '1.0')
49
50 model = et.SubElement(request, TAG_MODEL)
51 model.set(ATTR_VERSION, '1.0')
52 model.set(ATTR_NAME, 'post')
53
54 return et.tostring(request, 'unicode')
55
56 def global_id_exists(self, global_id):
57 """
58 Checks if the same global id already exists in the system.
59 """
60
61 return self.filter(key=global_id.key,
62 key_type=global_id.key_type,
63 local_id=global_id.local_id).exists()
64
65
66 class GlobalId(models.Model):
67 """
68 Global model ID and cache.
69 Key, key type and local ID make a single global identificator of the model.
70 Content is an XML cache of the model that can be passed along between nodes
71 without manual serialization each time.
72 """
73 class Meta:
74 app_label = 'boards'
75
76 objects = GlobalIdManager()
77
78 def __init__(self, *args, **kwargs):
79 models.Model.__init__(self, *args, **kwargs)
80
81 if 'key' in kwargs and 'key_type' in kwargs and 'local_id' in kwargs:
82 self.key = kwargs['key']
83 self.key_type = kwargs['key_type']
84 self.local_id = kwargs['local_id']
85
86 key = models.TextField()
87 key_type = models.TextField()
88 local_id = models.IntegerField()
89 content = models.TextField(blank=True, null=True)
90
91 def __str__(self):
92 return '%s::%s::%d' % (self.key_type, self.key, self.local_id)
93
94 def to_xml_element(self, element: et.Element):
95 """
96 Exports global id to an XML element.
97 """
98
99 element.set(ATTR_KEY, self.key)
100 element.set(ATTR_KEY_TYPE, self.key_type)
101 element.set(ATTR_LOCAL_ID, str(self.local_id))
102
103 @staticmethod
104 def from_xml_element(element: et.Element):
105 """
106 Parses XML id tag and gets global id from it.
107
108 Arguments:
109 element -- the XML 'id' element
110
111 Returns:
112 global_id -- id itself
113 exists -- True if the global id was taken from database, False if it
114 did not exist and was created.
115 """
116
117 try:
118 return GlobalId.objects.get(key=element.get(ATTR_KEY),
119 key_type=element.get(ATTR_KEY_TYPE),
120 local_id=int(element.get(
121 ATTR_LOCAL_ID))), True
122 except GlobalId.DoesNotExist:
123 return GlobalId(key=element.get(ATTR_KEY),
124 key_type=element.get(ATTR_KEY_TYPE),
125 local_id=int(element.get(ATTR_LOCAL_ID))), False
126
127
128 class Signature(models.Model):
129 class Meta:
130 app_label = 'boards'
131
132 def __init__(self, *args, **kwargs):
133 models.Model.__init__(self, *args, **kwargs)
134
135 if 'key' in kwargs and 'key_type' in kwargs and 'signature' in kwargs:
136 self.key_type = kwargs['key_type']
137 self.key = kwargs['key']
138 self.signature = kwargs['signature']
139
140 key_type = models.TextField()
141 key = models.TextField()
142 signature = models.TextField()
143
144 global_id = models.ForeignKey('GlobalId')
@@ -0,0 +1,61 b''
1 import base64
2 from ecdsa import SigningKey, VerifyingKey, BadSignatureError
3 from django.db import models
4
5 TYPE_ECDSA = 'ecdsa'
6
7 APP_LABEL_BOARDS = 'boards'
8
9
10 class KeyPairManager(models.Manager):
11 def generate_key(self, key_type=TYPE_ECDSA, primary=False):
12 if primary and self.filter(primary=True).exists():
13 raise Exception('There can be only one primary key')
14
15 if key_type == TYPE_ECDSA:
16 private = SigningKey.generate()
17 public = private.get_verifying_key()
18
19 private_key_str = base64.b64encode(private.to_string()).decode()
20 public_key_str = base64.b64encode(public.to_string()).decode()
21
22 return self.create(public_key=public_key_str,
23 private_key=private_key_str,
24 key_type=TYPE_ECDSA, primary=primary)
25 else:
26 raise Exception('Key type not supported')
27
28 def verify(self, signature, string):
29 if signature.key_type == TYPE_ECDSA:
30 public = VerifyingKey.from_string(base64.b64decode(signature.key))
31 signature_byte = base64.b64decode(signature.signature)
32 try:
33 return public.verify(signature_byte, string.encode())
34 except BadSignatureError:
35 return False
36 else:
37 raise Exception('Key type not supported')
38
39 def has_primary(self):
40 return self.filter(primary=True).exists()
41
42
43 class KeyPair(models.Model):
44 class Meta:
45 app_label = APP_LABEL_BOARDS
46
47 objects = KeyPairManager()
48
49 public_key = models.TextField()
50 private_key = models.TextField()
51 key_type = models.TextField()
52 primary = models.BooleanField(default=False)
53
54 def __str__(self):
55 return '%s::%s' % (self.key_type, self.public_key)
56
57 def sign(self, string):
58 private = SigningKey.from_string(base64.b64decode(
59 self.private_key.encode()))
60 signature_byte = private.sign_deterministic(string.encode())
61 return base64.b64encode(signature_byte).decode()
@@ -0,0 +1,89 b''
1 import re
2 from boards.mdx_neboard import get_parser
3
4 from boards.models import Post, GlobalId
5 from boards.models.post import REGEX_NOTIFICATION
6 from boards.models.post import REGEX_REPLY, REGEX_GLOBAL_REPLY
7 from boards.models.user import Notification
8 from django.db.models.signals import post_save, pre_save, pre_delete, \
9 post_delete
10 from django.dispatch import receiver
11 from django.utils import timezone
12
13
14 @receiver(post_save, sender=Post)
15 def connect_replies(instance, **kwargs):
16 for reply_number in re.finditer(REGEX_REPLY, instance.get_raw_text()):
17 post_id = reply_number.group(1)
18
19 try:
20 referenced_post = Post.objects.get(id=post_id)
21
22 if not referenced_post.referenced_posts.filter(
23 id=instance.id).exists():
24 referenced_post.referenced_posts.add(instance)
25 referenced_post.last_edit_time = instance.pub_time
26 referenced_post.build_refmap()
27 referenced_post.save(update_fields=['refmap', 'last_edit_time'])
28 except Post.DoesNotExist:
29 pass
30
31
32 @receiver(post_save, sender=Post)
33 def connect_global_replies(instance, **kwargs):
34 for reply_number in re.finditer(REGEX_GLOBAL_REPLY, instance.get_raw_text()):
35 key_type = reply_number.group(1)
36 key = reply_number.group(2)
37 local_id = reply_number.group(3)
38
39 try:
40 global_id = GlobalId.objects.get(key_type=key_type, key=key,
41 local_id=local_id)
42 referenced_post = Post.objects.get(global_id=global_id)
43 referenced_post.referenced_posts.add(instance)
44 referenced_post.last_edit_time = instance.pub_time
45 referenced_post.build_refmap()
46 referenced_post.save(update_fields=['refmap', 'last_edit_time'])
47 except (GlobalId.DoesNotExist, Post.DoesNotExist):
48 pass
49
50
51 @receiver(post_save, sender=Post)
52 def connect_notifications(instance, **kwargs):
53 for reply_number in re.finditer(REGEX_NOTIFICATION, instance.get_raw_text()):
54 user_name = reply_number.group(1).lower()
55 Notification.objects.get_or_create(name=user_name, post=instance)
56
57
58 @receiver(pre_save, sender=Post)
59 def preparse_text(instance, **kwargs):
60 instance._text_rendered = get_parser().parse(instance.get_raw_text())
61
62
63 @receiver(pre_delete, sender=Post)
64 def delete_images(instance, **kwargs):
65 for image in instance.images.all():
66 image_refs_count = image.post_images.count()
67 if image_refs_count == 1:
68 image.delete()
69
70
71 @receiver(pre_delete, sender=Post)
72 def delete_attachments(instance, **kwargs):
73 for attachment in instance.attachments.all():
74 attachment_refs_count = attachment.attachment_posts.count()
75 if attachment_refs_count == 1:
76 attachment.delete()
77
78
79 @receiver(post_delete, sender=Post)
80 def update_thread_on_delete(instance, **kwargs):
81 thread = instance.get_thread()
82 thread.last_edit_time = timezone.now()
83 thread.save()
84
85
86 @receiver(post_delete, sender=Post)
87 def delete_global_id(instance, **kwargs):
88 if instance.global_id and instance.global_id.id:
89 instance.global_id.delete()
@@ -0,0 +1,89 b''
1 from base64 import b64encode
2 import logging
3
4 from django.test import TestCase
5 from boards.models import KeyPair, GlobalId, Post, Signature
6 from boards.models.post.sync import SyncManager
7
8 logger = logging.getLogger(__name__)
9
10
11 class KeyTest(TestCase):
12 def test_create_key(self):
13 key = KeyPair.objects.generate_key('ecdsa')
14
15 self.assertIsNotNone(key, 'The key was not created.')
16
17 def test_validation(self):
18 key = KeyPair.objects.generate_key(key_type='ecdsa')
19 message = 'msg'
20 signature_value = key.sign(message)
21
22 signature = Signature(key_type='ecdsa', key=key.public_key,
23 signature=signature_value)
24 valid = KeyPair.objects.verify(signature, message)
25
26 self.assertTrue(valid, 'Message verification failed.')
27
28 def test_primary_constraint(self):
29 KeyPair.objects.generate_key(key_type='ecdsa', primary=True)
30
31 with self.assertRaises(Exception):
32 KeyPair.objects.generate_key(key_type='ecdsa', primary=True)
33
34 def test_model_id_save(self):
35 model_id = GlobalId(key_type='test', key='test key', local_id='1')
36 model_id.save()
37
38 def test_request_get(self):
39 post = self._create_post_with_key()
40
41 request = GlobalId.objects.generate_request_get([post.global_id])
42 logger.debug(request)
43
44 key = KeyPair.objects.get(primary=True)
45 self.assertTrue('<request type="get" version="1.0">'
46 '<model name="post" version="1.0">'
47 '<id key="%s" local-id="1" type="%s" />'
48 '</model>'
49 '</request>' % (
50 key.public_key,
51 key.key_type,
52 ) in request,
53 'Wrong XML generated for the GET request.')
54
55 def test_response_get(self):
56 post = self._create_post_with_key()
57 reply_post = Post.objects.create_post(title='test_title',
58 text='[post]%d[/post]' % post.id,
59 thread=post.get_thread())
60
61 response = SyncManager.generate_response_get([reply_post])
62 logger.debug(response)
63
64 key = KeyPair.objects.get(primary=True)
65 self.assertTrue('<status>success</status>'
66 '<models>'
67 '<model name="post">'
68 '<content>'
69 '<id key="%s" local-id="%d" type="%s" />'
70 '<title>test_title</title>'
71 '<text>[post]%s[/post]</text>'
72 '<thread><id key="%s" local-id="%d" type="%s" /></thread>'
73 '<pub-time>%s</pub-time>'
74 '</content>' % (
75 key.public_key,
76 reply_post.id,
77 key.key_type,
78 str(post.global_id),
79 key.public_key,
80 post.id,
81 key.key_type,
82 str(reply_post.get_pub_time_str()),
83 ) in response,
84 'Wrong XML generated for the GET response.')
85
86 def _create_post_with_key(self):
87 KeyPair.objects.generate_key(primary=True)
88
89 return Post.objects.create_post(title='test_title', text='test_text')
@@ -0,0 +1,102 b''
1 from boards.models import KeyPair, Post, Tag
2 from boards.models.post.sync import SyncManager
3 from boards.tests.mocks import MockRequest
4 from boards.views.sync import response_get
5
6 __author__ = 'neko259'
7
8
9 from django.test import TestCase
10
11
12 class SyncTest(TestCase):
13 def test_get(self):
14 """
15 Forms a GET request of a post and checks the response.
16 """
17
18 key = KeyPair.objects.generate_key(primary=True)
19 tag = Tag.objects.create(name='tag1')
20 post = Post.objects.create_post(title='test_title',
21 text='test_text\rline two',
22 tags=[tag])
23
24 request = MockRequest()
25 request.body = (
26 '<request type="get" version="1.0">'
27 '<model name="post" version="1.0">'
28 '<id key="%s" local-id="%d" type="%s" />'
29 '</model>'
30 '</request>' % (post.global_id.key,
31 post.id,
32 post.global_id.key_type)
33 )
34
35 response = response_get(request).content.decode()
36 self.assertTrue(
37 '<status>success</status>'
38 '<models>'
39 '<model name="post">'
40 '<content>'
41 '<id key="%s" local-id="%d" type="%s" />'
42 '<title>%s</title>'
43 '<text>%s</text>'
44 '<tags><tag>%s</tag></tags>'
45 '<pub-time>%s</pub-time>'
46 '</content>' % (
47 post.global_id.key,
48 post.global_id.local_id,
49 post.global_id.key_type,
50 post.title,
51 post.get_sync_text(),
52 post.get_thread().get_tags().first().name,
53 post.get_pub_time_str(),
54 ) in response,
55 'Wrong response generated for the GET request.')
56
57 post.delete()
58 key.delete()
59
60 KeyPair.objects.generate_key(primary=True)
61
62 SyncManager.parse_response_get(response, None)
63 self.assertEqual(1, Post.objects.count(),
64 'Post was not created from XML response.')
65
66 parsed_post = Post.objects.first()
67 self.assertEqual('tag1',
68 parsed_post.get_thread().get_tags().first().name,
69 'Invalid tag was parsed.')
70
71 SyncManager.parse_response_get(response, None)
72 self.assertEqual(1, Post.objects.count(),
73 'The same post was imported twice.')
74
75 self.assertEqual(1, parsed_post.global_id.signature_set.count(),
76 'Signature was not saved.')
77
78 post = parsed_post
79
80 # Trying to sync the same once more
81 response = response_get(request).content.decode()
82
83 self.assertTrue(
84 '<status>success</status>'
85 '<models>'
86 '<model name="post">'
87 '<content>'
88 '<id key="%s" local-id="%d" type="%s" />'
89 '<title>%s</title>'
90 '<text>%s</text>'
91 '<tags><tag>%s</tag></tags>'
92 '<pub-time>%s</pub-time>'
93 '</content>' % (
94 post.global_id.key,
95 post.global_id.local_id,
96 post.global_id.key_type,
97 post.title,
98 post.get_sync_text(),
99 post.get_thread().get_tags().first().name,
100 post.get_pub_time_str(),
101 ) in response,
102 'Wrong response generated for the GET request.')
@@ -0,0 +1,62 b''
1 import xml.etree.ElementTree as et
2 import xml.dom.minidom
3
4 from django.http import HttpResponse, Http404
5 from boards.models import GlobalId, Post
6 from boards.models.post.sync import SyncManager
7
8
9 def response_pull(request):
10 request_xml = request.body
11
12 if request_xml is None:
13 return HttpResponse(content='Use the API')
14
15 response_xml = SyncManager.generate_response_pull()
16
17 return HttpResponse(content=response_xml)
18
19
20 def response_get(request):
21 """
22 Processes a GET request with post ID list and returns the posts XML list.
23 Request should contain an 'xml' post attribute with the actual request XML.
24 """
25
26 request_xml = request.body
27
28 if request_xml is None:
29 return HttpResponse(content='Use the API')
30
31 posts = []
32
33 root_tag = et.fromstring(request_xml)
34 model_tag = root_tag[0]
35 for id_tag in model_tag:
36 global_id, exists = GlobalId.from_xml_element(id_tag)
37 if exists:
38 posts.append(Post.objects.get(global_id=global_id))
39
40 response_xml = SyncManager.generate_response_get(posts)
41
42 return HttpResponse(content=response_xml)
43
44
45 def get_post_sync_data(request, post_id):
46 try:
47 post = Post.objects.get(id=post_id)
48 except Post.DoesNotExist:
49 raise Http404()
50
51 xml_str = SyncManager.generate_response_get([post])
52
53 xml_repr = xml.dom.minidom.parseString(xml_str)
54 xml_repr = xml_repr.toprettyxml()
55
56 content = '=Global ID=\n%s\n\n=XML=\n%s' \
57 % (post.global_id, xml_repr)
58
59 return HttpResponse(
60 content_type='text/plain',
61 content=content,
62 ) No newline at end of file
@@ -0,0 +1,3 b''
1 #! /usr/bin/env sh
2
3 python3 manage.py runserver [::]:8000
@@ -0,0 +1,3 b''
1 #! /usr/bin/env sh
2
3 python3 manage.py test
@@ -0,0 +1,203 b''
1 # 0 Title #
2
3 DIP-1 Common protocol description
4
5 # 1 Intro #
6
7 This document describes the Data Interchange Protocol (DIP), designed to
8 exchange filtered data that can be stored as a graph structure between
9 network nodes.
10
11 # 2 Purpose #
12
13 This protocol will be used to share the models (originally imageboard posts)
14 across multiple servers. The main differnce of this protocol is that the node
15 can specify what models it wants to get and from whom. The nodes can get
16 models from a specific server, or from all except some specific servers. Also
17 the models can be filtered by timestamps or tags.
18
19 # 3 Protocol description #
20
21 The node requests other node's changes list since some time (since epoch if
22 this is the start). The other node sends a list of post ids or posts in the
23 XML format.
24
25 Protocol version is the version of the sync api. Model version is the version
26 of data models. If at least one of them is different, the sync cannot be
27 performed.
28
29 The node signs the data with its keys. The receiving node saves the key at the
30 first sync and checks it every time. If the key has changed, the info won't be
31 saved from the node (or the node id must be changed). A model can be signed
32 with several keys but at least one of them must be the same as in the global
33 ID to verify the sender.
34
35 Each node can have several keys. Nodes can have shared keys to serve as a pool
36 (several nodes with the same key).
37
38 Each post has an ID in the unique format: key-type::key::local-id
39
40 All requests pass a request type, protocol and model versions, and a list of
41 optional arguments used for filtering.
42
43 Each request has its own version. Version consists of 2 numbers: first is
44 incompatible version (1.3 and 2.0 are not compatible and must not be in sync)
45 and the second one is minor and compatible (for example, new optional field
46 is added which will be igroned by those who don't support it yet).
47
48 Post edits and reflinks are not saved to the sync model. The replied post ID
49 can be got from the post text, and reflinks can be computed when loading
50 posts. The edit time is not saved because a foreign post can be 'edited' (new
51 replies are added) but the signature must not change (so we can't update the
52 content). The inner posts can be edited, and the signature will change then
53 but the local-id won't, so the other node can detect that and replace the post
54 instead of adding a new one.
55
56 ## 3.1 Requests ##
57
58 There is no constraint on how the server should calculate the request. The
59 server can return any information by any filter and the requesting node is
60 responsible for validating it.
61
62 The server is required to return the status of request. See 3.2 for details.
63
64 ### 3.1.1 pull ###
65
66 "pull" request gets the desired model id list by the given filter (e.g. thread, tags,
67 author)
68
69 Sample request is as follows:
70
71 <?xml version="1.1" encoding="UTF-8" ?>
72 <request version="1.0" type="pull">
73 <model version="1.0" name="post">
74 <timestamp_from>0</timestamp_from>
75 <timestamp_to>0</timestamp_to>
76 <tags>
77 <tag>tag1</tag>
78 </tags>
79 <sender>
80 <allow>
81 <key>abcehy3h9t</key>
82 <key>ehoehyoe</key>
83 </allow>
84 <!-- There can be only allow block (all other are denied) or deny block (all other are allowed) -->
85 </sender>
86 </model>
87 </request>
88
89 Under the <model> tag there are filters. Filters for the "post" model can
90 be found in DIP-2.
91
92 Sample response:
93
94 <?xml version="1.1" encoding="UTF-8" ?>
95 <response>
96 <status>success</status>
97 <models>
98 <id key="id1" type="ecdsa" local-id="1" />
99 <id key="id1" type="ecdsa" local-id="2" />
100 <id key="id2" type="ecdsa" local-id="1" />
101 <id key="id2" type="ecdsa" local-id="5" />
102 </models>
103 </response>
104
105 ### 3.1.2 get ###
106
107 "get" gets models by id list.
108
109 Sample request:
110
111 <?xml version="1.1" encoding="UTF-8" ?>
112 <request version="1.0" type="get">
113 <model version="1.0" name="post">
114 <id key="id1" type="ecdsa" local-id="1" />
115 <id key="id1" type="ecdsa" local-id="2" />
116 </model>
117 </request>
118
119 Id consists of a key, key type and local id. This key is used for signing and
120 validating of data in the model content.
121
122 Sample response:
123
124 <?xml version="1.1" encoding="UTF-8" ?>
125 <response>
126 <!--
127 Valid statuses are 'success' and 'error'.
128 -->
129 <status>success</status>
130 <models>
131 <model name="post">
132 <!--
133 Content tag is the data that is signed by signatures and must
134 not be changed for the post from other node.
135 -->
136 <content>
137 <id key="id1" type="ecdsa" local-id="1" />
138 <title>13</title>
139 <text>Thirteen</text>
140 <thread><id key="id1" type="ecdsa" local-id="2" /></thread>
141 <pub-time>12</pub-time>
142 <!--
143 Images are saved as attachments and included in the
144 signature.
145 -->
146 <attachments>
147 <attachment mimetype="image/png">TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5I</attachment>
148 </attachments>
149 </content>
150 <!--
151 There can be several signatures for one model. At least one
152 signature must be made with the key used in global ID.
153 -->
154 <signatures>
155 <signature key="id1" type="ecdsa" value="dhefhtreh" />
156 <signature key="id45" type="ecdsa" value="dsgfgdhefhtreh" />
157 </signatures>
158 <attachment-refs>
159 <attachment-ref ref="TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5IGhpcyByZWFzb24sIGJ1dCBieSB0"
160 url="/media/images/12345.png" />
161 </attachment-refs>
162 </model>
163 <model name="post">
164 <content>
165 <id key="id1" type="ecdsa" local-id="id2" />
166 <title>13</title>
167 <text>Thirteen</text>
168 <pub-time>12</pub-time>
169 <edit-time>13</edit-time>
170 <tags>
171 <tag>tag1</tag>
172 </tags>
173 </content>
174 <signatures>
175 <signature key="id2" type="ecdsa" value="dehdfh" />
176 </signatures>
177 </model>
178 </models>
179 </response>
180
181 ### 3.1.3 put ###
182
183 "put" gives a model to the given node (you have no guarantee the node takes
184 it, consider you are just advising the node to take your post). This request
185 type is useful in pool where all the nodes try to duplicate all of their data
186 across the pool.
187
188 ## 3.2 Responses ##
189
190 ### 3.2.1 "not supported" ###
191
192 If the request if completely not supported, a "not supported" status will be
193 returned.
194
195 ### 3.2.2 "success" ###
196
197 "success" status means the request was processed and the result is returned.
198
199 ### 3.2.3 "error" ###
200
201 If the server knows for sure that the operation cannot be processed, it sends
202 the "error" status. Additional tags describing the error may be <description>
203 and <stack>.
@@ -0,0 +1,29 b''
1 # 0 Title #
2
3 "post" model reference
4
5 # 1 Description #
6
7 "post" is a model that defines an imageboard message, or post.
8
9 # 2 Fields #
10
11 # 2.1 Mandatory fields #
12
13 * title -- text field.
14 * text -- text field.
15 * pub-time -- timestamp (TBD: Define format).
16
17 # 2.2 Optional fields #
18
19 * attachments -- defines attachments such as images.
20 * attachment -- contains and attachment or link to attachment to be downloaded
21 manually. Required attributes are mimetype and name.
22
23 This field is used for the opening post (thread):
24
25 * tags -- text tag name list.
26
27 This field is used for non-opening post:
28
29 * thread -- ID of a post model that this post is related to.
@@ -0,0 +1,1 b''
1 default_app_config = 'boards.apps.BoardsAppConfig' No newline at end of file
@@ -1,7 +1,7 b''
1 1 from django.contrib import admin
2 2 from django.utils.translation import ugettext_lazy as _
3 3 from django.core.urlresolvers import reverse
4 from boards.models import Post, Tag, Ban, Thread, Banner, PostImage
4 from boards.models import Post, Tag, Ban, Thread, Banner, PostImage, KeyPair, GlobalId
5 5
6 6
7 7 @admin.register(Post)
@@ -62,7 +62,6 b' class TagAdmin(admin.ModelAdmin):'
62 62 super().save_model(request, obj, form, change)
63 63 for thread in obj.get_threads().all():
64 64 thread.refresh_tags()
65
66 65 list_display = ('name', 'thread_count', 'display_children')
67 66 search_fields = ('name',)
68 67
@@ -89,7 +88,6 b' class ThreadAdmin(admin.ModelAdmin):'
89 88 def save_related(self, request, form, formsets, change):
90 89 super().save_related(request, form, formsets, change)
91 90 form.instance.refresh_tags()
92
93 91 list_display = ('id', 'op', 'title', 'reply_count', 'status', 'ip',
94 92 'display_tags')
95 93 list_filter = ('bump_time', 'status')
@@ -97,6 +95,13 b' class ThreadAdmin(admin.ModelAdmin):'
97 95 filter_horizontal = ('tags',)
98 96
99 97
98 @admin.register(KeyPair)
99 class KeyPairAdmin(admin.ModelAdmin):
100 list_display = ('public_key', 'primary')
101 list_filter = ('primary',)
102 search_fields = ('public_key',)
103
104
100 105 @admin.register(Ban)
101 106 class BanAdmin(admin.ModelAdmin):
102 107 list_display = ('ip', 'can_read')
@@ -112,3 +117,11 b' class BannerAdmin(admin.ModelAdmin):'
112 117 @admin.register(PostImage)
113 118 class PostImageAdmin(admin.ModelAdmin):
114 119 search_fields = ('alias',)
120
121
122 @admin.register(GlobalId)
123 class GlobalIdAdmin(admin.ModelAdmin):
124 def is_linked(self, obj):
125 return Post.objects.filter(global_id=obj).exists()
126
127 list_display = ('__str__', 'is_linked',) No newline at end of file
@@ -15,7 +15,7 b' from django.utils import timezone'
15 15 from boards.abstracts.settingsmanager import get_settings_manager
16 16 from boards.abstracts.attachment_alias import get_image_by_alias
17 17 from boards.mdx_neboard import formatters
18 from boards.models.attachment.downloaders import Downloader
18 from boards.models.attachment.downloaders import download
19 19 from boards.models.post import TITLE_MAX_LENGTH
20 20 from boards.models import Tag, Post
21 21 from boards.utils import validate_file_size, get_file_mimetype, \
@@ -375,12 +375,7 b' class PostForm(NeboardForm):'
375 375 img_temp = None
376 376
377 377 try:
378 for downloader in Downloader.__subclasses__():
379 if downloader.handles(url):
380 return downloader.download(url)
381 # If nobody of the specific downloaders handles this, use generic
382 # one
383 return Downloader.download(url)
378 download(url)
384 379 except forms.ValidationError as e:
385 380 raise e
386 381 except Exception as e:
@@ -2,8 +2,7 b' from django.core.management import BaseC'
2 2 from django.db import transaction
3 3
4 4 from boards.models import Post
5 from boards.models.post import NO_IP
6
5 from boards.models.post.manager import NO_IP
7 6
8 7 __author__ = 'neko259'
9 8
@@ -16,6 +16,7 b' import boards'
16 16
17 17
18 18 REFLINK_PATTERN = re.compile(r'^\d+$')
19 GLOBAL_REFLINK_PATTERN = re.compile(r'(\w+)::([^:]+)::(\d+)')
19 20 MULTI_NEWLINES_PATTERN = re.compile(r'(\r?\n){2,}')
20 21 ONE_NEWLINE = '\n'
21 22 REGEX_URL = re.compile(r'https?\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(/\S*)?')
@@ -121,15 +122,27 b' class CodePattern(TextFormatter):'
121 122 def render_reflink(tag_name, value, options, parent, context):
122 123 result = '>>%s' % value
123 124
125 post = None
124 126 if REFLINK_PATTERN.match(value):
125 127 post_id = int(value)
126 128
127 129 try:
128 130 post = boards.models.Post.objects.get(id=post_id)
129 131
130 result = post.get_link_view()
131 132 except ObjectDoesNotExist:
132 133 pass
134 elif GLOBAL_REFLINK_PATTERN.match(value):
135 match = GLOBAL_REFLINK_PATTERN.search(value)
136 try:
137 global_id = boards.models.GlobalId.objects.get(
138 key_type=match.group(1), key=match.group(2),
139 local_id=match.group(3))
140 post = global_id.post
141 except ObjectDoesNotExist:
142 pass
143
144 if post is not None:
145 result = post.get_link_view()
133 146
134 147 return result
135 148
@@ -177,9 +190,6 b' def render_spoiler(tag_name, value, opti'
177 190 return '<span class="spoiler">{}{}{}</span>'.format(side_spaces, value,
178 191 side_spaces)
179 192
180 return quote_element
181
182
183 193
184 194 formatters = [
185 195 QuotePattern,
@@ -3,6 +3,8 b" STATUS_BUMPLIMIT = 'bumplimit'"
3 3 STATUS_ARCHIVE = 'archived'
4 4
5 5
6 from boards.models.signature import GlobalId, Signature
7 from boards.models.sync_key import KeyPair
6 8 from boards.models.image import PostImage
7 9 from boards.models.attachment import Attachment
8 10 from boards.models.thread import Thread
@@ -54,6 +54,15 b' class Downloader:'
54 54 return file
55 55
56 56
57 def download(url):
58 for downloader in Downloader.__subclasses__():
59 if downloader.handles(url):
60 return downloader.download(url)
61 # If nobody of the specific downloaders handles this, use generic
62 # one
63 return Downloader.download(url)
64
65
57 66 class YouTubeDownloader(Downloader):
58 67 @staticmethod
59 68 def download(url: str):
@@ -1,26 +1,19 b''
1 import logging
2 import re
3 1 import uuid
4 2
3 import re
4 from boards import settings
5 from boards.abstracts.tripcode import Tripcode
6 from boards.models import PostImage, Attachment, KeyPair, GlobalId
7 from boards.models.base import Viewable
8 from boards.models.post.export import get_exporter, DIFF_TYPE_JSON
9 from boards.models.post.manager import PostManager
5 10 from boards.utils import datetime_to_epoch
6 11 from django.core.exceptions import ObjectDoesNotExist
7 12 from django.core.urlresolvers import reverse
8 13 from django.db import models
9 14 from django.db.models import TextField, QuerySet
10 from django.template.defaultfilters import striptags, truncatewords
15 from django.template.defaultfilters import truncatewords, striptags
11 16 from django.template.loader import render_to_string
12 from django.utils import timezone
13 from django.db.models.signals import post_save, pre_save
14 from django.dispatch import receiver
15
16 from boards import settings
17 from boards.abstracts.tripcode import Tripcode
18 from boards.mdx_neboard import get_parser
19 from boards.models import PostImage, Attachment
20 from boards.models.base import Viewable
21 from boards.models.post.export import get_exporter, DIFF_TYPE_JSON
22 from boards.models.post.manager import PostManager
23 from boards.models.user import Notification
24 17
25 18 CSS_CLS_HIDDEN_POST = 'hidden_post'
26 19 CSS_CLS_DEAD_POST = 'dead_post'
@@ -39,6 +32,8 b' IMAGE_THUMB_SIZE = (200, 150)'
39 32 TITLE_MAX_LENGTH = 200
40 33
41 34 REGEX_REPLY = re.compile(r'\[post\](\d+)\[/post\]')
35 REGEX_GLOBAL_REPLY = re.compile(r'\[post\](\w+)::([^:]+)::(\d+)\[/post\]')
36 REGEX_URL = re.compile(r'https?\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(/\S*)?')
42 37 REGEX_NOTIFICATION = re.compile(r'\[user\](\w+)\[/user\]')
43 38
44 39 PARAMETER_TRUNCATED = 'truncated'
@@ -101,6 +96,11 b' class Post(models.Model, Viewable):'
101 96 url = models.TextField()
102 97 uid = models.TextField(db_index=True)
103 98
99 # Global ID with author key. If the message was downloaded from another
100 # server, this indicates the server.
101 global_id = models.OneToOneField(GlobalId, null=True, blank=True,
102 on_delete=models.CASCADE)
103
104 104 tripcode = models.CharField(max_length=50, blank=True, default='')
105 105 opening = models.BooleanField(db_index=True)
106 106 hidden = models.BooleanField(default=False)
@@ -214,29 +214,54 b' class Post(models.Model, Viewable):'
214 214 def get_first_image(self) -> PostImage:
215 215 return self.images.earliest('id')
216 216
217 def delete(self, using=None):
217 def set_global_id(self, key_pair=None):
218 218 """
219 Deletes all post images and the post itself.
219 Sets global id based on the given key pair. If no key pair is given,
220 default one is used.
220 221 """
221 222
222 for image in self.images.all():
223 image_refs_count = image.post_images.count()
224 if image_refs_count == 1:
225 image.delete()
223 if key_pair:
224 key = key_pair
225 else:
226 try:
227 key = KeyPair.objects.get(primary=True)
228 except KeyPair.DoesNotExist:
229 # Do not update the global id because there is no key defined
230 return
231 global_id = GlobalId(key_type=key.key_type,
232 key=key.public_key,
233 local_id=self.id)
234 global_id.save()
235
236 self.global_id = global_id
237
238 self.save(update_fields=['global_id'])
239
240 def get_pub_time_str(self):
241 return str(self.pub_time)
226 242
227 for attachment in self.attachments.all():
228 attachment_refs_count = attachment.attachment_posts.count()
229 if attachment_refs_count == 1:
230 attachment.delete()
243 def get_replied_ids(self):
244 """
245 Gets ID list of the posts that this post replies.
246 """
247
248 raw_text = self.get_raw_text()
231 249
232 thread = self.get_thread()
233 thread.last_edit_time = timezone.now()
234 thread.save()
250 local_replied = REGEX_REPLY.findall(raw_text)
251 global_replied = []
252 for match in REGEX_GLOBAL_REPLY.findall(raw_text):
253 key_type = match[0]
254 key = match[1]
255 local_id = match[2]
235 256
236 super(Post, self).delete(using)
237
238 logging.getLogger('boards.post.delete').info(
239 'Deleted post {}'.format(self))
257 try:
258 global_id = GlobalId.objects.get(key_type=key_type,
259 key=key, local_id=local_id)
260 for post in Post.objects.filter(global_id=global_id).only('id'):
261 global_replied.append(post.id)
262 except GlobalId.DoesNotExist:
263 pass
264 return local_replied + global_replied
240 265
241 266 def get_post_data(self, format_type=DIFF_TYPE_JSON, request=None,
242 267 include_last_update=False) -> str:
@@ -305,6 +330,24 b' class Post(models.Model, Viewable):'
305 330 def get_raw_text(self) -> str:
306 331 return self.text
307 332
333 def get_sync_text(self) -> str:
334 """
335 Returns text applicable for sync. It has absolute post reflinks.
336 """
337
338 replacements = dict()
339 for post_id in REGEX_REPLY.findall(self.get_raw_text()):
340 absolute_post_id = str(Post.objects.get(id=post_id).global_id)
341 replacements[post_id] = absolute_post_id
342
343 text = self.get_raw_text() or ''
344 for key in replacements:
345 text = text.replace('[post]{}[/post]'.format(key),
346 '[post]{}[/post]'.format(replacements[key]))
347 text = text.replace('\r\n', '\n').replace('\r', '\n')
348
349 return text
350
308 351 def connect_threads(self, opening_posts):
309 352 for opening_post in opening_posts:
310 353 threads = opening_post.get_threads().all()
@@ -336,34 +379,3 b' class Post(models.Model, Viewable):'
336 379
337 380 def set_hidden(self, hidden):
338 381 self.hidden = hidden
339
340
341 # SIGNALS (Maybe move to other module?)
342 @receiver(post_save, sender=Post)
343 def connect_replies(instance, **kwargs):
344 for reply_number in re.finditer(REGEX_REPLY, instance.get_raw_text()):
345 post_id = reply_number.group(1)
346
347 try:
348 referenced_post = Post.objects.get(id=post_id)
349
350 # Connect only to posts that are not connected to already
351 if not referenced_post.referenced_posts.filter(id=instance.id).exists():
352 referenced_post.referenced_posts.add(instance)
353 referenced_post.last_edit_time = instance.pub_time
354 referenced_post.build_refmap()
355 referenced_post.save(update_fields=['refmap', 'last_edit_time'])
356 except ObjectDoesNotExist:
357 pass
358
359
360 @receiver(post_save, sender=Post)
361 def connect_notifications(instance, **kwargs):
362 for reply_number in re.finditer(REGEX_NOTIFICATION, instance.get_raw_text()):
363 user_name = reply_number.group(1).lower()
364 Notification.objects.get_or_create(name=user_name, post=instance)
365
366
367 @receiver(pre_save, sender=Post)
368 def preparse_text(instance, **kwargs):
369 instance._text_rendered = get_parser().parse(instance.get_raw_text())
@@ -80,17 +80,13 b' class PostManager(models.Manager):'
80 80 logger.info('Created post [{}] with text [{}] by {}'.format(post,
81 81 post.get_text(),post.poster_ip))
82 82
83 # TODO Move this to other place
84 83 if file:
85 file_type = file.name.split('.')[-1].lower()
86 if file_type in IMAGE_TYPES:
87 post.images.add(PostImage.objects.create_with_hash(file))
88 else:
89 post.attachments.add(Attachment.objects.create_with_hash(file))
84 self._add_file_to_post(file, post)
90 85 for image in images:
91 86 post.images.add(image)
92 87
93 88 post.connect_threads(opening_posts)
89 post.set_global_id()
94 90
95 91 # Thread needs to be bumped only when the post is already created
96 92 if not new_thread:
@@ -131,3 +127,34 b' class PostManager(models.Manager):'
131 127
132 128 return ppd
133 129
130 @transaction.atomic
131 def import_post(self, title: str, text: str, pub_time: str, global_id,
132 opening_post=None, tags=list(), files=list()):
133 is_opening = opening_post is None
134 if is_opening:
135 thread = boards.models.thread.Thread.objects.create(
136 bump_time=pub_time, last_edit_time=pub_time)
137 list(map(thread.tags.add, tags))
138 else:
139 thread = opening_post.get_thread()
140
141 post = self.create(title=title, text=text,
142 pub_time=pub_time,
143 poster_ip=NO_IP,
144 last_edit_time=pub_time,
145 global_id=global_id,
146 opening=is_opening,
147 thread=thread)
148
149 # TODO Add files
150 for file in files:
151 self._add_file_to_post(file, post)
152
153 post.threads.add(thread)
154
155 def _add_file_to_post(self, file, post):
156 file_type = file.name.split('.')[-1].lower()
157 if file_type in IMAGE_TYPES:
158 post.images.add(PostImage.objects.create_with_hash(file))
159 else:
160 post.attachments.add(Attachment.objects.create_with_hash(file))
@@ -1,6 +1,5 b''
1 1 from django.db import models
2
3 import boards.models.post
2 import boards
4 3
5 4 __author__ = 'neko259'
6 5
@@ -492,6 +492,11 b' ul {'
492 492 font-size: 1.2em;
493 493 }
494 494
495 .global-id {
496 font-weight: bolder;
497 opacity: .5;
498 }
499
495 500 /* Reflink preview */
496 501 .post_preview {
497 502 border-left: 1px solid #777;
@@ -55,7 +55,9 b''
55 55 | <a href="{% url 'admin:boards_thread_change' thread.id %}">{% trans 'Edit thread' %}</a>
56 56 {% endif %}
57 57 {% endif %}
58 </form>
58 {% if post.global_id_id %}
59 | <a href="{% url 'post_sync_data' post.id %}">RAW</a>
60 {% endif %}
59 61 </span>
60 62 {% endif %}
61 63 </div>
@@ -42,7 +42,7 b' class FormTest(TestCase):'
42 42 # Change posting delay so we don't have to wait for 30 seconds or more
43 43 old_posting_delay = neboard.settings.POSTING_DELAY
44 44 # Wait fot the posting delay or we won't be able to post
45 settings.POSTING_DELAY = 1
45 neboard.settings.POSTING_DELAY = 1
46 46 time.sleep(neboard.settings.POSTING_DELAY + 1)
47 47 response = client.post(THREAD_PAGE_ONE, {'text': TEST_TEXT,
48 48 'tags': valid_tags})
@@ -2,7 +2,7 b' from django.core.paginator import Pagina'
2 2 from django.test import TestCase
3 3
4 4 from boards import settings
5 from boards.models import Tag, Post, Thread
5 from boards.models import Tag, Post, Thread, KeyPair
6 6 from boards.models.thread import STATUS_ARCHIVE
7 7
8 8
@@ -115,6 +115,27 b' class PostTests(TestCase):'
115 115 self.assertEqual(all_threads[settings.get_int('View', 'ThreadsPerPage')].id,
116 116 first_post.id)
117 117
118 def test_reflinks(self):
119 """
120 Tests that reflinks are parsed within post and connecting replies
121 to the replied posts.
122
123 Local reflink example: [post]123[/post]
124 Global reflink example: [post]key_type::key::123[/post]
125 """
126
127 key = KeyPair.objects.generate_key(primary=True)
128
129 tag = Tag.objects.create(name='test_tag')
130
131 post = Post.objects.create_post(title='', text='', tags=[tag])
132 post_local_reflink = Post.objects.create_post(title='',
133 text='[post]%d[/post]' % post.id, thread=post.get_thread())
134
135 self.assertTrue(post_local_reflink in post.referenced_posts.all(),
136 'Local reflink not connecting posts.')
137
138
118 139 def test_thread_replies(self):
119 140 """
120 141 Tests that the replies can be queried from a thread in all possible
@@ -11,6 +11,7 b' HTTP_CODE_OK = 200'
11 11 EXCLUDED_VIEWS = {
12 12 'banned',
13 13 'get_thread_diff',
14 'api_sync_pull',
14 15 }
15 16
16 17
@@ -2,14 +2,17 b' from django.conf.urls import url'
2 2 #from django.views.i18n import javascript_catalog
3 3
4 4 import neboard
5
5 6 from boards import views
6 7 from boards.rss import AllThreadsFeed, TagThreadsFeed, ThreadPostsFeed
7 8 from boards.views import api, tag_threads, all_threads, \
8 9 settings, all_tags, feed
9 10 from boards.views.authors import AuthorsView
10 11 from boards.views.notifications import NotificationView
12 from boards.views.search import BoardSearchView
11 13 from boards.views.static import StaticPageView
12 14 from boards.views.preview import PostPreviewView
15 from boards.views.sync import get_post_sync_data, response_get, response_pull
13 16 from boards.views.random import RandomImageView
14 17 from boards.views.tag_gallery import TagGalleryView
15 18 from boards.views.translation import cached_javascript_catalog
@@ -74,12 +77,19 b' urlpatterns = ['
74 77 url(r'^api/preview/$', api.api_get_preview, name='preview'),
75 78 url(r'^api/new_posts/$', api.api_get_new_posts, name='new_posts'),
76 79
80 # Sync protocol API
81 url(r'^api/sync/pull/$', response_pull, name='api_sync_pull'),
82 url(r'^api/sync/get/$', response_get, name='api_sync_pull'),
83 # TODO 'get' request
84
77 85 # Notifications
78 86 url(r'^notifications/(?P<username>\w+)/$', NotificationView.as_view(), name='notifications'),
79 87 url(r'^notifications/$', NotificationView.as_view(), name='notifications'),
80 88
81 89 # Post preview
82 90 url(r'^preview/$', PostPreviewView.as_view(), name='preview'),
91 url(r'^post_xml/(?P<post_id>\d+)$', get_post_sync_data,
92 name='post_sync_data'),
83 93 ]
84 94
85 95 # Search
@@ -1,26 +1,20 b''
1 from collections import OrderedDict
2 1 import json
3 2 import logging
4 3
4 from django.core import serializers
5 5 from django.db import transaction
6 from django.db.models import Count
7 6 from django.http import HttpResponse
8 7 from django.shortcuts import get_object_or_404
9 from django.core import serializers
10 from django.template.context_processors import csrf
11 8 from django.views.decorators.csrf import csrf_protect
12 9
13 from boards.abstracts.settingsmanager import get_settings_manager,\
14 FAV_THREAD_NO_UPDATES
15
10 from boards.abstracts.settingsmanager import get_settings_manager
16 11 from boards.forms import PostForm, PlainErrorList
12 from boards.mdx_neboard import Parser
17 13 from boards.models import Post, Thread, Tag
18 14 from boards.models.thread import STATUS_ARCHIVE
15 from boards.models.user import Notification
19 16 from boards.utils import datetime_to_epoch
20 17 from boards.views.thread import ThreadView
21 from boards.models.user import Notification
22 from boards.mdx_neboard import Parser
23
24 18
25 19 __author__ = 'neko259'
26 20
@@ -2,7 +2,6 b''
2 2 import os
3 3
4 4 DEBUG = True
5 TEMPLATE_DEBUG = DEBUG
6 5
7 6 ADMINS = (
8 7 # ('Your Name', 'your_email@example.com'),
@@ -67,16 +66,7 b" STATIC_ROOT = ''"
67 66 # Example: "http://media.lawrence.com/static/"
68 67 STATIC_URL = '/static/'
69 68
70 # Additional locations of static files
71 # It is really a hack, put real paths, not related
72 STATICFILES_DIRS = (
73 os.path.dirname(__file__) + '/boards/static',
74
75 # '/d/work/python/django/neboard/neboard/boards/static',
76 # Put strings here, like "/home/html/static" or "C:/www/django/static".
77 # Always use forward slashes, even on Windows.
78 # Don't forget to use absolute paths, not relative paths.
79 )
69 STATICFILES_DIRS = []
80 70
81 71 # List of finder classes that know how to find static files in
82 72 # various locations.
@@ -227,6 +217,7 b' MIDDLEWARE_CLASSES += ['
227 217 'debug_toolbar.middleware.DebugToolbarMiddleware',
228 218 ]
229 219
220
230 221 def custom_show_toolbar(request):
231 222 return request.user.has_perm('admin.debug')
232 223
@@ -9,21 +9,54 b' Site: http://neboard.me/'
9 9
10 10 # INSTALLATION #
11 11
12 1. Install all dependencies over pip or system-wide
13 2. Setup a database in `neboard/settings.py`
14 3. Run `./manage.py migrate` to apply all south migrations
15 4. Apply config changes to `boards/config/config.ini`. You can see the default settings in `boards/config/default_config.ini`
12 1. Download application and move inside it:
13
14 `hg clone https://bitbucket.org/neko259/neboard`
15
16 `cd neboard`
17
18 If you wish to use *decentral* version, change branch to *decentral*:
19
20 `hg up decentral`
21
22 2. Install all application dependencies:
23
24 Some minimal system-wide depenencies:
25
26 * python3
27 * pip/pip3
28 * jpeg
29
30 Python dependencies:
31
32 `pip3 install -r requirements.txt`
33
34 You can use virtualenv to speed up the process or avoid conflicts.
35
36 3. Setup a database in `neboard/settings.py`. You can also change other settings like search engine.
37
38 Depending on configured database and search engine, you need to install corresponding dependencies manually.
39
40 Default database is *sqlite*, default search engine is *whoosh*.
41
42 4. Setup SECRET_KEY to a secret value in `neboard/settings.py
43 5. Run `./manage.py migrate` to apply all migrations
44 6. Apply config changes to `boards/config/config.ini`. You can see the default settings in `boards/config/default_config.ini`
45 7. If you want to use decetral engine, run `./manage.py generate_keypair` to generate keys
16 46
17 47 # RUNNING #
18 48
19 You can run the server using django default embedded webserver by running
49 You can run the server using django default embedded webserver by running:
20 50
21 51 ./manage.py runserver <address>:<port>
22 52
23 See django-admin command help for details
53 See django-admin command help for details.
24 54
25 55 Also consider using wsgi or fcgi interfaces on production servers.
26 56
57 When running for the first time, you need to setup at least one section tag.
58 Go to the admin page and manually create one tag with "required" property set.
59
27 60 # UPGRADE #
28 61
29 62 1. Backup your project data.
@@ -34,4 +67,4 b' You can also just clone the mercurial pr'
34 67
35 68 # CONCLUSION #
36 69
37 Enjoy our software and thank you!
70 Enjoy our software and thank you! No newline at end of file
@@ -1,4 +1,6 b''
1 1 python-magic
2 httplib2
3 simplejson
2 4 pytube
3 5 requests
4 6 adjacent
@@ -8,3 +10,4 b' django>=1.8'
8 10 bbcode
9 11 django-debug-toolbar
10 12 pytz
13 ecdsa No newline at end of file
@@ -15,6 +15,7 b''
15 15 * Subscribing to tag via AJAX
16 16 * Add buttons to insert a named link or a named quote to the markup panel
17 17 * Add support for "attention posts" that are shown in the header"
18 * Use absolute post reflinks in the raw text
18 19 * Use default boards/settings when it is not defined. Maybe default_settings.py module?
19 20
20 21 = Bugs =
General Comments 0
You need to be logged in to leave comments. Login now