Show More
@@ -0,0 +1,19 b'' | |||
|
1 | # -*- coding: utf-8 -*- | |
|
2 | # Generated by Django 1.11 on 2017-09-27 12:38 | |
|
3 | from __future__ import unicode_literals | |
|
4 | ||
|
5 | from django.db import migrations | |
|
6 | ||
|
7 | ||
|
8 | class Migration(migrations.Migration): | |
|
9 | ||
|
10 | dependencies = [ | |
|
11 | ('boards', '0063_auto_20170301_1058'), | |
|
12 | ] | |
|
13 | ||
|
14 | operations = [ | |
|
15 | migrations.RemoveField( | |
|
16 | model_name='post', | |
|
17 | name='version', | |
|
18 | ), | |
|
19 | ] |
@@ -16,7 +16,7 b' class PostAdmin(admin.ModelAdmin):' | |||
|
16 | 16 | exclude = ('referenced_posts', 'refmap', 'images', 'global_id') |
|
17 | 17 | readonly_fields = ('poster_ip', 'thread', 'linked_images', |
|
18 | 18 | 'attachments', 'uid', 'url', 'pub_time', 'opening', 'linked_global_id', |
|
19 |
|
|
|
19 | 'foreign', 'tags') | |
|
20 | 20 | |
|
21 | 21 | def ban_poster(self, request, queryset): |
|
22 | 22 | bans = 0 |
@@ -63,7 +63,6 b' class PostAdmin(admin.ModelAdmin):' | |||
|
63 | 63 | return ', '.join([tag.get_name() for tag in obj.get_tags()]) |
|
64 | 64 | |
|
65 | 65 | def save_model(self, request, obj, form, change): |
|
66 | obj.increment_version() | |
|
67 | 66 | obj.save() |
|
68 | 67 | obj.clear_cache() |
|
69 | 68 | |
@@ -127,8 +126,6 b' class ThreadAdmin(admin.ModelAdmin):' | |||
|
127 | 126 | |
|
128 | 127 | def save_model(self, request, obj, form, change): |
|
129 | 128 | op = obj.get_opening_post() |
|
130 | op.increment_version() | |
|
131 | op.save(update_fields=['version']) | |
|
132 | 129 | obj.save() |
|
133 | 130 | op.clear_cache() |
|
134 | 131 |
@@ -4,9 +4,10 b' import xml.etree.ElementTree as ET' | |||
|
4 | 4 | |
|
5 | 5 | import httplib2 |
|
6 | 6 | from django.core.management import BaseCommand |
|
7 | from django.utils.dateparse import parse_datetime | |
|
7 | 8 | |
|
8 | 9 | from boards.models import GlobalId |
|
9 |
from boards.models.post.sync import SyncManager, TAG_ID, TAG_ |
|
|
10 | from boards.models.post.sync import SyncManager, TAG_ID, TAG_UPDATE_TIME | |
|
10 | 11 | |
|
11 | 12 | __author__ = 'neko259' |
|
12 | 13 | |
@@ -85,12 +86,12 b' class Command(BaseCommand):' | |||
|
85 | 86 | for model in models: |
|
86 | 87 | tag_id = model.find(TAG_ID) |
|
87 | 88 | global_id, exists = GlobalId.from_xml_element(tag_id) |
|
88 |
tag_ |
|
|
89 |
if tag_ |
|
|
90 |
|
|
|
89 | tag_update_time = model.find(TAG_UPDATE_TIME) | |
|
90 | if tag_update_time: | |
|
91 | update_time = tag_update_time.text | |
|
91 | 92 | else: |
|
92 |
|
|
|
93 |
if not exists or global_id.post. |
|
|
93 | update_time = None | |
|
94 | if not exists or update_time is None or global_id.post.last_edit_time < parse_datetime(update_time): | |
|
94 | 95 | logger.debug('Processed (+) post {}'.format(global_id)) |
|
95 | 96 | ids_to_sync.append(global_id) |
|
96 | 97 | else: |
@@ -100,7 +100,6 b' class Post(models.Model, Viewable):' | |||
|
100 | 100 | tripcode = models.CharField(max_length=50, blank=True, default='') |
|
101 | 101 | opening = models.BooleanField(db_index=True) |
|
102 | 102 | hidden = models.BooleanField(default=False) |
|
103 | version = models.IntegerField(default=1) | |
|
104 | 103 | |
|
105 | 104 | def __str__(self): |
|
106 | 105 | return 'P#{}/{}'.format(self.id, self.get_title()) |
@@ -341,9 +340,6 b' class Post(models.Model, Viewable):' | |||
|
341 | 340 | def set_hidden(self, hidden): |
|
342 | 341 | self.hidden = hidden |
|
343 | 342 | |
|
344 | def increment_version(self): | |
|
345 | self.version = F('version') + 1 | |
|
346 | ||
|
347 | 343 | def clear_cache(self): |
|
348 | 344 | """ |
|
349 | 345 | Clears sync data (content cache, signatures etc). |
@@ -138,7 +138,7 b' class PostManager(models.Manager):' | |||
|
138 | 138 | @transaction.atomic |
|
139 | 139 | def import_post(self, title: str, text: str, pub_time: str, global_id, |
|
140 | 140 | opening_post=None, tags=list(), files=list(), |
|
141 |
file_urls=list(), tripcode=None, |
|
|
141 | file_urls=list(), tripcode=None, last_edit_time=None): | |
|
142 | 142 | is_opening = opening_post is None |
|
143 | 143 | if is_opening: |
|
144 | 144 | thread = boards.models.thread.Thread.objects.create( |
@@ -151,12 +151,11 b' class PostManager(models.Manager):' | |||
|
151 | 151 | text=text, |
|
152 | 152 | pub_time=pub_time, |
|
153 | 153 | poster_ip=NO_IP, |
|
154 | last_edit_time=pub_time, | |
|
154 | last_edit_time=last_edit_time or pub_time, | |
|
155 | 155 | global_id=global_id, |
|
156 | 156 | opening=is_opening, |
|
157 | 157 | thread=thread, |
|
158 |
tripcode=tripcode |
|
|
159 | version=version) | |
|
158 | tripcode=tripcode) | |
|
160 | 159 | |
|
161 | 160 | for file in files: |
|
162 | 161 | self._add_file_to_post(file, post) |
@@ -170,12 +169,11 b' class PostManager(models.Manager):' | |||
|
170 | 169 | |
|
171 | 170 | @transaction.atomic |
|
172 | 171 | def update_post(self, post, title: str, text: str, pub_time: str, |
|
173 |
tags=list(), files=list(), file_urls=list(), tripcode=None |
|
|
172 | tags=list(), files=list(), file_urls=list(), tripcode=None): | |
|
174 | 173 | post.title = title |
|
175 | 174 | post.text = text |
|
176 | 175 | post.pub_time = pub_time |
|
177 | 176 | post.tripcode = tripcode |
|
178 | post.version = version | |
|
179 | 177 | post.save() |
|
180 | 178 | |
|
181 | 179 | post.clear_cache() |
@@ -1,16 +1,17 b'' | |||
|
1 | import logging | |
|
1 | 2 | import xml.etree.ElementTree as et |
|
2 | import logging | |
|
3 | from xml.etree import ElementTree | |
|
3 | ||
|
4 | from django.db import transaction | |
|
5 | from django.utils.dateparse import parse_datetime | |
|
4 | 6 | |
|
5 | 7 | from boards.abstracts.exceptions import SyncException |
|
6 | from boards.abstracts.sync_filters import ThreadFilter, TagsFilter,\ | |
|
7 |
|
|
|
8 | from boards.abstracts.sync_filters import ThreadFilter, TagsFilter, \ | |
|
9 | TimestampFromFilter | |
|
8 | 10 | from boards.models import KeyPair, GlobalId, Signature, Post, Tag |
|
9 | 11 | from boards.models.attachment.downloaders import download |
|
10 | 12 | from boards.models.signature import TAG_REQUEST, ATTR_TYPE, TYPE_GET, \ |
|
11 | 13 | ATTR_VERSION, TAG_MODEL, ATTR_NAME, TAG_ID, TYPE_LIST |
|
12 | 14 | from boards.utils import get_file_mimetype, get_file_hash |
|
13 | from django.db import transaction | |
|
14 | 15 | |
|
15 | 16 | EXCEPTION_NODE = 'Sync node returned an error: {}.' |
|
16 | 17 | EXCEPTION_DOWNLOAD = 'File was not downloaded.' |
@@ -30,6 +31,7 b" TAG_TITLE = 'title'" | |||
|
30 | 31 | TAG_TEXT = 'text' |
|
31 | 32 | TAG_THREAD = 'thread' |
|
32 | 33 | TAG_PUB_TIME = 'pub-time' |
|
34 | TAG_UPDATE_TIME = 'update-time' | |
|
33 | 35 | TAG_SIGNATURES = 'signatures' |
|
34 | 36 | TAG_SIGNATURE = 'signature' |
|
35 | 37 | TAG_CONTENT = 'content' |
@@ -59,6 +61,8 b" ID_TYPE_URL = 'url'" | |||
|
59 | 61 | |
|
60 | 62 | STATUS_SUCCESS = 'success' |
|
61 | 63 | |
|
64 | CURRENT_MODEL_VERSION = '1.1' | |
|
65 | ||
|
62 | 66 | |
|
63 | 67 | logger = logging.getLogger('boards.sync') |
|
64 | 68 | |
@@ -120,6 +124,9 b' class SyncManager:' | |||
|
120 | 124 | pub_time = et.SubElement(content_tag, TAG_PUB_TIME) |
|
121 | 125 | pub_time.text = str(post.get_pub_time_str()) |
|
122 | 126 | |
|
127 | update_time = et.SubElement(content_tag, TAG_UPDATE_TIME) | |
|
128 | update_time.text = str(post.last_edit_time) | |
|
129 | ||
|
123 | 130 | if post.tripcode: |
|
124 | 131 | tripcode = et.SubElement(content_tag, TAG_TRIPCODE) |
|
125 | 132 | tripcode.text = post.tripcode |
@@ -141,8 +148,6 b' class SyncManager:' | |||
|
141 | 148 | for file in attachments: |
|
142 | 149 | SyncManager._attachment_to_xml( |
|
143 | 150 | attachments_tag, attachment_refs, file) |
|
144 | version_tag = et.SubElement(content_tag, TAG_VERSION) | |
|
145 | version_tag.text = str(post.version) | |
|
146 | 151 | |
|
147 | 152 | global_id.content = et.tostring(content_tag, ENCODING_UNICODE) |
|
148 | 153 | global_id.save() |
@@ -191,8 +196,8 b' class SyncManager:' | |||
|
191 | 196 | global_id, exists = GlobalId.from_xml_element(tag_id) |
|
192 | 197 | signatures = SyncManager._verify_model(global_id, content_str, tag_model) |
|
193 | 198 | |
|
194 |
|
|
|
195 |
is_old = exists and global_id.post. |
|
|
199 | update_time = tag_content.find(TAG_UPDATE_TIME).text | |
|
200 | is_old = exists and global_id.post.last_edit_time < parse_datetime(update_time) | |
|
196 | 201 | if exists and not is_old: |
|
197 | 202 | logger.debug('Post {} exists and is up to date.'.format(global_id)) |
|
198 | 203 | else: |
@@ -204,13 +209,13 b' class SyncManager:' | |||
|
204 | 209 | |
|
205 | 210 | title = tag_content.find(TAG_TITLE).text or '' |
|
206 | 211 | text = tag_content.find(TAG_TEXT).text or '' |
|
207 | pub_time = tag_content.find(TAG_PUB_TIME).text | |
|
208 | 212 | tripcode_tag = tag_content.find(TAG_TRIPCODE) |
|
209 | 213 | if tripcode_tag is not None: |
|
210 | 214 | tripcode = tripcode_tag.text or '' |
|
211 | 215 | else: |
|
212 | 216 | tripcode = '' |
|
213 | 217 | |
|
218 | pub_time = tag_content.find(TAG_PUB_TIME).text | |
|
214 | 219 | thread = tag_content.find(TAG_THREAD) |
|
215 | 220 | tags = [] |
|
216 | 221 | if thread: |
@@ -257,15 +262,14 b' class SyncManager:' | |||
|
257 | 262 | Post.objects.update_post( |
|
258 | 263 | post, title=title, text=text, pub_time=pub_time, |
|
259 | 264 | tags=tags, files=files, file_urls=urls, |
|
260 | tripcode=tripcode, version=version) | |
|
265 | tripcode=tripcode, version=version, last_edit_time=update_time) | |
|
261 | 266 | logger.debug('Parsed updated post {}'.format(global_id)) |
|
262 | 267 | else: |
|
263 | 268 | Post.objects.import_post( |
|
264 | 269 | title=title, text=text, pub_time=pub_time, |
|
265 | 270 | opening_post=opening_post, tags=tags, |
|
266 | 271 | global_id=global_id, files=files, |
|
267 | file_urls=urls, tripcode=tripcode, | |
|
268 | version=version) | |
|
272 | file_urls=urls, tripcode=tripcode, last_edit_time=update_time) | |
|
269 | 273 | logger.debug('Parsed new post {}'.format(global_id)) |
|
270 | 274 | |
|
271 | 275 | @staticmethod |
@@ -285,8 +289,8 b' class SyncManager:' | |||
|
285 | 289 | tag_model = et.SubElement(models, TAG_MODEL) |
|
286 | 290 | tag_id = et.SubElement(tag_model, TAG_ID) |
|
287 | 291 | post.global_id.to_xml_element(tag_id) |
|
288 |
|
|
|
289 |
|
|
|
292 | update_time = et.SubElement(tag_model, TAG_UPDATE_TIME) | |
|
293 | update_time.text = str(post.last_edit_time) | |
|
290 | 294 | |
|
291 | 295 | return et.tostring(response, ENCODING_UNICODE) |
|
292 | 296 | |
@@ -372,7 +376,7 b' class SyncManager:' | |||
|
372 | 376 | request.set(ATTR_VERSION, '1.0') |
|
373 | 377 | |
|
374 | 378 | model = et.SubElement(request, TAG_MODEL) |
|
375 |
model.set(ATTR_VERSION, |
|
|
379 | model.set(ATTR_VERSION, CURRENT_MODEL_VERSION) | |
|
376 | 380 | model.set(ATTR_NAME, 'post') |
|
377 | 381 | |
|
378 | 382 | if opening_post: |
@@ -186,7 +186,7 b' class Thread(models.Model):' | |||
|
186 | 186 | """ |
|
187 | 187 | Gets replies with only fields that are used for viewing. |
|
188 | 188 | """ |
|
189 |
return self.get_replies().defer('text', 'last_edit_time' |
|
|
189 | return self.get_replies().defer('text', 'last_edit_time') | |
|
190 | 190 | |
|
191 | 191 | def get_top_level_replies(self) -> QuerySet: |
|
192 | 192 | return self.get_replies().exclude(refposts__threads__in=[self]) |
@@ -64,6 +64,7 b'' | |||
|
64 | 64 | <button name="method" value="subscribe" class="not_fav">★ {% trans "Add to favorites" %}</button> |
|
65 | 65 | {% endif %} |
|
66 | 66 | </form> |
|
67 | • | |
|
67 | 68 | <form action="{% url 'tag' tag.get_name %}" method="post" class="post-button-form"> |
|
68 | 69 | {% if is_hidden %} |
|
69 | 70 | <button name="method" value="unhide" class="fav">{% trans "Show" %}</button> |
@@ -71,6 +72,7 b'' | |||
|
71 | 72 | <button name="method" value="hide" class="not_fav">{% trans "Hide" %}</button> |
|
72 | 73 | {% endif %} |
|
73 | 74 | </form> |
|
75 | • | |
|
74 | 76 | <a href="{% url 'tag_gallery' tag.get_name %}">{% trans 'Gallery' %}</a> |
|
75 | 77 | </p> |
|
76 | 78 | {% if tag.get_description %} |
@@ -65,22 +65,22 b' class KeyTest(TestCase):' | |||
|
65 | 65 | '<models>' |
|
66 | 66 | '<model name="post">' |
|
67 | 67 | '<content>' |
|
68 |
'<id key=" |
|
|
68 | '<id key="{}" local-id="{}" type="{}" />' | |
|
69 | 69 | '<title>test_title</title>' |
|
70 |
'<text>[post] |
|
|
71 |
'<thread><id key=" |
|
|
72 |
'<pub-time> |
|
|
73 |
'< |
|
|
74 |
'</content>' |
|
|
70 | '<text>[post]{}[/post]</text>' | |
|
71 | '<thread><id key="{}" local-id="{}" type="{}" /></thread>' | |
|
72 | '<pub-time>{}</pub-time>' | |
|
73 | '<update-time>{}</update-time>' | |
|
74 | '</content>'.format( | |
|
75 | 75 | key.public_key, |
|
76 | 76 | reply_post.id, |
|
77 | 77 | key.key_type, |
|
78 |
|
|
|
78 | post.global_id, | |
|
79 | 79 | key.public_key, |
|
80 | 80 | post.id, |
|
81 | 81 | key.key_type, |
|
82 |
|
|
|
83 |
post. |
|
|
82 | reply_post.get_pub_time_str(), | |
|
83 | reply_post.last_edit_time, | |
|
84 | 84 | ) in response, |
|
85 | 85 | 'Wrong XML generated for the GET response.') |
|
86 | 86 |
@@ -37,13 +37,13 b' class SyncTest(TestCase):' | |||
|
37 | 37 | '<models>' |
|
38 | 38 | '<model name="post">' |
|
39 | 39 | '<content>' |
|
40 |
'<id key=" |
|
|
41 |
'<title> |
|
|
42 |
'<text> |
|
|
43 |
'<tags><tag> |
|
|
44 |
'<pub-time> |
|
|
45 | '<version>%s</version>' | |
|
46 |
'</content>' |
|
|
40 | '<id key="{}" local-id="{}" type="{}" />' | |
|
41 | '<title>{}</title>' | |
|
42 | '<text>{}</text>' | |
|
43 | '<tags><tag>{}</tag></tags>' | |
|
44 | '<pub-time>{}</pub-time>' | |
|
45 | '<update-time>{}</update-time>' | |
|
46 | '</content>'.format( | |
|
47 | 47 | post.global_id.key, |
|
48 | 48 | post.global_id.local_id, |
|
49 | 49 | post.global_id.key_type, |
@@ -51,7 +51,7 b' class SyncTest(TestCase):' | |||
|
51 | 51 | post.get_sync_text(), |
|
52 | 52 | post.get_thread().get_tags().first().get_name(), |
|
53 | 53 | post.get_pub_time_str(), |
|
54 |
post. |
|
|
54 | post.last_edit_time, | |
|
55 | 55 | ) in response, |
|
56 | 56 | 'Wrong response generated for the GET request.') |
|
57 | 57 | |
@@ -86,13 +86,13 b' class SyncTest(TestCase):' | |||
|
86 | 86 | '<models>' |
|
87 | 87 | '<model name="post">' |
|
88 | 88 | '<content>' |
|
89 |
'<id key=" |
|
|
90 |
'<title> |
|
|
91 |
'<text> |
|
|
92 |
'<tags><tag> |
|
|
93 |
'<pub-time> |
|
|
94 | '<version>%s</version>' | |
|
95 |
'</content>' |
|
|
89 | '<id key="{}" local-id="{}" type="{}" />' | |
|
90 | '<title>{}</title>' | |
|
91 | '<text>{}</text>' | |
|
92 | '<tags><tag>{}</tag></tags>' | |
|
93 | '<pub-time>{}</pub-time>' | |
|
94 | '<update-time>{}</update-time>' | |
|
95 | '</content>'.format( | |
|
96 | 96 | post.global_id.key, |
|
97 | 97 | post.global_id.local_id, |
|
98 | 98 | post.global_id.key_type, |
@@ -100,7 +100,6 b' class SyncTest(TestCase):' | |||
|
100 | 100 | post.get_sync_text(), |
|
101 | 101 | post.get_thread().get_tags().first().get_name(), |
|
102 | 102 | post.get_pub_time_str(), |
|
103 | post.version, | |
|
104 | 103 | ) in response, |
|
105 | 104 | 'Wrong response generated for the GET request.') |
|
106 | 105 | |
@@ -128,21 +127,21 b' class SyncTest(TestCase):' | |||
|
128 | 127 | '<models>' |
|
129 | 128 | '<model>' |
|
130 | 129 | '<id key="{}" local-id="{}" type="{}" />' |
|
131 |
'< |
|
|
130 | '<update-time>{}</update-time>' | |
|
132 | 131 | '</model>' |
|
133 | 132 | '<model>' |
|
134 | 133 | '<id key="{}" local-id="{}" type="{}" />' |
|
135 |
'< |
|
|
134 | '<update-time>{}</update-time>' | |
|
136 | 135 | '</model>' |
|
137 | 136 | '</models>'.format( |
|
138 | 137 | post.global_id.key, |
|
139 | 138 | post.global_id.local_id, |
|
140 | 139 | post.global_id.key_type, |
|
141 |
post. |
|
|
140 | post.last_edit_time, | |
|
142 | 141 | post2.global_id.key, |
|
143 | 142 | post2.global_id.local_id, |
|
144 | 143 | post2.global_id.key_type, |
|
145 |
post2. |
|
|
144 | post2.last_edit_time, | |
|
146 | 145 | ) in response_all, |
|
147 | 146 | 'Wrong response generated for the LIST request for all posts.') |
|
148 | 147 | |
@@ -173,13 +172,13 b' class SyncTest(TestCase):' | |||
|
173 | 172 | '<models>' |
|
174 | 173 | '<model>' |
|
175 | 174 | '<id key="{}" local-id="{}" type="{}" />' |
|
176 |
'< |
|
|
175 | '<update-time>{}</update-time>' | |
|
177 | 176 | '</model>' |
|
178 | 177 | '</models>'.format( |
|
179 | 178 | post.global_id.key, |
|
180 | 179 | post.global_id.local_id, |
|
181 | 180 | post.global_id.key_type, |
|
182 |
post. |
|
|
181 | post.last_edit_time, | |
|
183 | 182 | ) in response_thread, |
|
184 | 183 | 'Wrong response generated for the LIST request for posts of ' |
|
185 | 184 | 'existing thread.') |
@@ -240,13 +239,13 b' class SyncTest(TestCase):' | |||
|
240 | 239 | '<models>' |
|
241 | 240 | '<model>' |
|
242 | 241 | '<id key="{}" local-id="{}" type="{}" />' |
|
243 |
'< |
|
|
242 | '<update-time>{}</update_time>' | |
|
244 | 243 | '</model>' |
|
245 | 244 | '</models>'.format( |
|
246 | 245 | post2.global_id.key, |
|
247 | 246 | post2.global_id.local_id, |
|
248 | 247 | post2.global_id.key_type, |
|
249 |
post2. |
|
|
248 | post2.last_edit_time, | |
|
250 | 249 | ) in response_thread, |
|
251 | 250 | 'Wrong response generated for the LIST request for posts of ' |
|
252 | 251 | 'existing thread.') |
@@ -97,7 +97,7 b' Sample response:' | |||
|
97 | 97 | <models> |
|
98 | 98 | <model> |
|
99 | 99 | <id key="id1" type="ecdsa" local-id="1"> |
|
100 | <version>1</version> | |
|
100 | <update-time>2017-01-01 00:00:00</update-time> | |
|
101 | 101 | </model> |
|
102 | 102 | <model> |
|
103 | 103 | <id key="id1" type="ecdsa" local-id="2" /> |
@@ -1,6 +1,6 b'' | |||
|
1 | 1 | # 0 Title # |
|
2 | 2 | |
|
3 | "post" model reference | |
|
3 | "post" model reference of version 1.1 | |
|
4 | 4 | |
|
5 | 5 | # 1 Description # |
|
6 | 6 | |
@@ -13,7 +13,7 b'' | |||
|
13 | 13 | * title -- text field. |
|
14 | 14 | * text -- text field. |
|
15 | 15 | * pub-time -- timestamp (TBD: Define format). |
|
16 |
* |
|
|
16 | * update -- when post content changes, the update time should be incremented. | |
|
17 | 17 | |
|
18 | 18 | # 2.2 Optional fields # |
|
19 | 19 |
General Comments 0
You need to be logged in to leave comments.
Login now