##// END OF EJS Templates
Download attached filed to the post during sync
neko259 -
r1511:ea51d39c decentral
parent child Browse files
Show More
@@ -1,7 +1,7
1 from django.contrib import admin
1 from django.contrib import admin
2 from django.utils.translation import ugettext_lazy as _
2 from django.utils.translation import ugettext_lazy as _
3 from django.core.urlresolvers import reverse
3 from django.core.urlresolvers import reverse
4 from boards.models import Post, Tag, Ban, Thread, Banner, PostImage, KeyPair
4 from boards.models import Post, Tag, Ban, Thread, Banner, PostImage, KeyPair, GlobalId
5
5
6
6
7 @admin.register(Post)
7 @admin.register(Post)
@@ -109,3 +109,16 class BanAdmin(admin.ModelAdmin):
109 @admin.register(Banner)
109 @admin.register(Banner)
110 class BannerAdmin(admin.ModelAdmin):
110 class BannerAdmin(admin.ModelAdmin):
111 list_display = ('title', 'text')
111 list_display = ('title', 'text')
112
113
114 @admin.register(PostImage)
115 class PostImageAdmin(admin.ModelAdmin):
116 search_fields = ('alias',)
117
118
119 @admin.register(GlobalId)
120 class GlobalIdAdmin(admin.ModelAdmin):
121 def is_linked(self, obj):
122 return Post.objects.filter(global_id=obj).exists()
123
124 list_display = ('__str__', 'is_linked',) No newline at end of file
@@ -15,7 +15,7 from django.utils import timezone
15 from boards.abstracts.settingsmanager import get_settings_manager
15 from boards.abstracts.settingsmanager import get_settings_manager
16 from boards.abstracts.attachment_alias import get_image_by_alias
16 from boards.abstracts.attachment_alias import get_image_by_alias
17 from boards.mdx_neboard import formatters
17 from boards.mdx_neboard import formatters
18 from boards.models.attachment.downloaders import Downloader
18 from boards.models.attachment.downloaders import download
19 from boards.models.post import TITLE_MAX_LENGTH
19 from boards.models.post import TITLE_MAX_LENGTH
20 from boards.models import Tag, Post
20 from boards.models import Tag, Post
21 from boards.utils import validate_file_size, get_file_mimetype, \
21 from boards.utils import validate_file_size, get_file_mimetype, \
@@ -375,12 +375,7 class PostForm(NeboardForm):
375 img_temp = None
375 img_temp = None
376
376
377 try:
377 try:
378 for downloader in Downloader.__subclasses__():
378 download(url)
379 if downloader.handles(url):
380 return downloader.download(url)
381 # If nobody of the specific downloaders handles this, use generic
382 # one
383 return Downloader.download(url)
384 except forms.ValidationError as e:
379 except forms.ValidationError as e:
385 raise e
380 raise e
386 except Exception as e:
381 except Exception as e:
@@ -26,6 +26,7 class Command(BaseCommand):
26
26
27 pull_url = url + 'api/sync/pull/'
27 pull_url = url + 'api/sync/pull/'
28 get_url = url + 'api/sync/get/'
28 get_url = url + 'api/sync/get/'
29 file_url = url[:-1]
29
30
30 global_id_str = options.get('global_id')
31 global_id_str = options.get('global_id')
31 if global_id_str:
32 if global_id_str:
@@ -43,7 +44,7 class Command(BaseCommand):
43 h = httplib2.Http()
44 h = httplib2.Http()
44 response, content = h.request(get_url, method="POST", body=xml)
45 response, content = h.request(get_url, method="POST", body=xml)
45
46
46 SyncManager.parse_response_get(content)
47 SyncManager.parse_response_get(content, file_url)
47 else:
48 else:
48 raise Exception('Invalid global ID')
49 raise Exception('Invalid global ID')
49 else:
50 else:
@@ -72,7 +73,7 class Command(BaseCommand):
72 h = httplib2.Http()
73 h = httplib2.Http()
73 response, content = h.request(get_url, method="POST", body=xml)
74 response, content = h.request(get_url, method="POST", body=xml)
74
75
75 SyncManager.parse_response_get(content)
76 SyncManager.parse_response_get(content, file_url)
76 else:
77 else:
77 print('Nothing to get, everything synced')
78 print('Nothing to get, everything synced')
78 else:
79 else:
@@ -54,6 +54,15 class Downloader:
54 return file
54 return file
55
55
56
56
57 def download(url):
58 for downloader in Downloader.__subclasses__():
59 if downloader.handles(url):
60 return downloader.download(url)
61 # If nobody of the specific downloaders handles this, use generic
62 # one
63 return Downloader.download(url)
64
65
57 class YouTubeDownloader(Downloader):
66 class YouTubeDownloader(Downloader):
58 @staticmethod
67 @staticmethod
59 def download(url: str):
68 def download(url: str):
@@ -1,17 +1,7
1 import logging
1 import logging
2 import re
3 import uuid
2 import uuid
4
3
5 from django.core.exceptions import ObjectDoesNotExist
4 import re
6 from django.core.urlresolvers import reverse
7 from django.db import models
8 from django.db.models import TextField, QuerySet
9 from django.template.defaultfilters import truncatewords, striptags
10 from django.template.loader import render_to_string
11 from django.utils import timezone
12 from django.dispatch import receiver
13 from django.db.models.signals import pre_save, post_save
14
15 from boards import settings
5 from boards import settings
16 from boards.abstracts.tripcode import Tripcode
6 from boards.abstracts.tripcode import Tripcode
17 from boards.mdx_neboard import get_parser
7 from boards.mdx_neboard import get_parser
@@ -20,6 +10,16 from boards.models.base import Viewable
20 from boards.models.post.export import get_exporter, DIFF_TYPE_JSON
10 from boards.models.post.export import get_exporter, DIFF_TYPE_JSON
21 from boards.models.post.manager import PostManager
11 from boards.models.post.manager import PostManager
22 from boards.models.user import Notification
12 from boards.models.user import Notification
13 from django.core.exceptions import ObjectDoesNotExist
14 from django.core.urlresolvers import reverse
15 from django.db import models
16 from django.db.models import TextField, QuerySet
17 from django.db.models.signals import pre_save, post_save, pre_delete, \
18 post_delete
19 from django.dispatch import receiver
20 from django.template.defaultfilters import truncatewords, striptags
21 from django.template.loader import render_to_string
22 from django.utils import timezone
23
23
24 CSS_CLS_HIDDEN_POST = 'hidden_post'
24 CSS_CLS_HIDDEN_POST = 'hidden_post'
25 CSS_CLS_DEAD_POST = 'dead_post'
25 CSS_CLS_DEAD_POST = 'dead_post'
@@ -104,7 +104,8 class Post(models.Model, Viewable):
104
104
105 # Global ID with author key. If the message was downloaded from another
105 # Global ID with author key. If the message was downloaded from another
106 # server, this indicates the server.
106 # server, this indicates the server.
107 global_id = models.OneToOneField('GlobalId', null=True, blank=True)
107 global_id = models.OneToOneField(GlobalId, null=True, blank=True,
108 on_delete=models.CASCADE)
108
109
109 tripcode = models.CharField(max_length=50, blank=True, default='')
110 tripcode = models.CharField(max_length=50, blank=True, default='')
110 opening = models.BooleanField(db_index=True)
111 opening = models.BooleanField(db_index=True)
@@ -174,6 +175,7 class Post(models.Model, Viewable):
174
175
175 def get_thread_id(self):
176 def get_thread_id(self):
176 return self.thread_id
177 return self.thread_id
178
177 def get_threads(self) -> QuerySet:
179 def get_threads(self) -> QuerySet:
178 """
180 """
179 Gets post's thread.
181 Gets post's thread.
@@ -220,33 +222,6 class Post(models.Model, Viewable):
220 def get_first_image(self) -> PostImage:
222 def get_first_image(self) -> PostImage:
221 return self.images.earliest('id')
223 return self.images.earliest('id')
222
224
223 def delete(self, using=None):
224 """
225 Deletes all post images and the post itself.
226 """
227
228 for image in self.images.all():
229 image_refs_count = image.post_images.count()
230 if image_refs_count == 1:
231 image.delete()
232
233 for attachment in self.attachments.all():
234 attachment_refs_count = attachment.attachment_posts.count()
235 if attachment_refs_count == 1:
236 attachment.delete()
237
238 if self.global_id:
239 self.global_id.delete()
240
241 thread = self.get_thread()
242 thread.last_edit_time = timezone.now()
243 thread.save()
244
245 super(Post, self).delete(using)
246
247 logging.getLogger('boards.post.delete').info(
248 'Deleted post {}'.format(self))
249
250 def set_global_id(self, key_pair=None):
225 def set_global_id(self, key_pair=None):
251 """
226 """
252 Sets global id based on the given key pair. If no key pair is given,
227 Sets global id based on the given key pair. If no key pair is given,
@@ -373,7 +348,7 class Post(models.Model, Viewable):
373 absolute_post_id = str(Post.objects.get(id=post_id).global_id)
348 absolute_post_id = str(Post.objects.get(id=post_id).global_id)
374 replacements[post_id] = absolute_post_id
349 replacements[post_id] = absolute_post_id
375
350
376 text = self.get_raw_text()
351 text = self.get_raw_text() or ''
377 for key in replacements:
352 for key in replacements:
378 text = text.replace('[post]{}[/post]'.format(key),
353 text = text.replace('[post]{}[/post]'.format(key),
379 '[post]{}[/post]'.format(replacements[key]))
354 '[post]{}[/post]'.format(replacements[key]))
@@ -453,3 +428,26 def connect_notifications(instance, **kw
453 @receiver(pre_save, sender=Post)
428 @receiver(pre_save, sender=Post)
454 def preparse_text(instance, **kwargs):
429 def preparse_text(instance, **kwargs):
455 instance._text_rendered = get_parser().parse(instance.get_raw_text())
430 instance._text_rendered = get_parser().parse(instance.get_raw_text())
431
432
433 @receiver(pre_delete, sender=Post)
434 def delete_images(instance, **kwargs):
435 for image in instance.images.all():
436 image_refs_count = image.post_images.count()
437 if image_refs_count == 1:
438 image.delete()
439
440
441 @receiver(pre_delete, sender=Post)
442 def delete_attachments(instance, **kwargs):
443 for attachment in instance.attachments.all():
444 attachment_refs_count = attachment.attachment_posts.count()
445 if attachment_refs_count == 1:
446 attachment.delete()
447
448
449 @receiver(post_delete, sender=Post)
450 def update_thread_on_delete(instance, **kwargs):
451 thread = instance.get_thread()
452 thread.last_edit_time = timezone.now()
453 thread.save()
@@ -80,13 +80,8 class PostManager(models.Manager):
80 logger.info('Created post [{}] with text [{}] by {}'.format(post,
80 logger.info('Created post [{}] with text [{}] by {}'.format(post,
81 post.get_text(),post.poster_ip))
81 post.get_text(),post.poster_ip))
82
82
83 # TODO Move this to other place
84 if file:
83 if file:
85 file_type = file.name.split('.')[-1].lower()
84 self._add_file_to_post(file, post)
86 if file_type in IMAGE_TYPES:
87 post.images.add(PostImage.objects.create_with_hash(file))
88 else:
89 post.attachments.add(Attachment.objects.create_with_hash(file))
90 for image in images:
85 for image in images:
91 post.images.add(image)
86 post.images.add(image)
92
87
@@ -134,7 +129,7 class PostManager(models.Manager):
134
129
135 @transaction.atomic
130 @transaction.atomic
136 def import_post(self, title: str, text: str, pub_time: str, global_id,
131 def import_post(self, title: str, text: str, pub_time: str, global_id,
137 opening_post=None, tags=list()):
132 opening_post=None, tags=list(), files=list()):
138 is_opening = opening_post is None
133 is_opening = opening_post is None
139 if is_opening:
134 if is_opening:
140 thread = boards.models.thread.Thread.objects.create(
135 thread = boards.models.thread.Thread.objects.create(
@@ -151,4 +146,15 class PostManager(models.Manager):
151 opening=is_opening,
146 opening=is_opening,
152 thread=thread)
147 thread=thread)
153
148
149 # TODO Add files
150 for file in files:
151 self._add_file_to_post(file, post)
152
154 post.threads.add(thread)
153 post.threads.add(thread)
154
155 def _add_file_to_post(self, file, post):
156 file_type = file.name.split('.')[-1].lower()
157 if file_type in IMAGE_TYPES:
158 post.images.add(PostImage.objects.create_with_hash(file))
159 else:
160 post.attachments.add(Attachment.objects.create_with_hash(file))
@@ -1,5 +1,6
1 import xml.etree.ElementTree as et
1 import xml.etree.ElementTree as et
2
2
3 from boards.models.attachment.downloaders import download
3 from boards.utils import get_file_mimetype
4 from boards.utils import get_file_mimetype
4 from django.db import transaction
5 from django.db import transaction
5 from boards.models import KeyPair, GlobalId, Signature, Post, Tag
6 from boards.models import KeyPair, GlobalId, Signature, Post, Tag
@@ -40,6 +41,10 ATTR_URL = 'url'
40 STATUS_SUCCESS = 'success'
41 STATUS_SUCCESS = 'success'
41
42
42
43
44 class SyncException(Exception):
45 pass
46
47
43 class SyncManager:
48 class SyncManager:
44 @staticmethod
49 @staticmethod
45 def generate_response_get(model_list: list):
50 def generate_response_get(model_list: list):
@@ -118,7 +123,7 class SyncManager:
118
123
119 @staticmethod
124 @staticmethod
120 @transaction.atomic
125 @transaction.atomic
121 def parse_response_get(response_xml):
126 def parse_response_get(response_xml, hostname):
122 tag_root = et.fromstring(response_xml)
127 tag_root = et.fromstring(response_xml)
123 tag_status = tag_root.find(TAG_STATUS)
128 tag_status = tag_root.find(TAG_STATUS)
124 if STATUS_SUCCESS == tag_status.text:
129 if STATUS_SUCCESS == tag_status.text:
@@ -139,8 +144,8 class SyncManager:
139 signature.global_id = global_id
144 signature.global_id = global_id
140 signature.save()
145 signature.save()
141
146
142 title = tag_content.find(TAG_TITLE).text
147 title = tag_content.find(TAG_TITLE).text or ''
143 text = tag_content.find(TAG_TEXT).text
148 text = tag_content.find(TAG_TEXT).text or ''
144 pub_time = tag_content.find(TAG_PUB_TIME).text
149 pub_time = tag_content.find(TAG_PUB_TIME).text
145
150
146 thread = tag_content.find(TAG_THREAD)
151 thread = tag_content.find(TAG_THREAD)
@@ -151,7 +156,7 class SyncManager:
151 if exists:
156 if exists:
152 opening_post = Post.objects.get(global_id=op_global_id)
157 opening_post = Post.objects.get(global_id=op_global_id)
153 else:
158 else:
154 raise Exception('Load the OP first')
159 raise SyncException('Load the OP first')
155 else:
160 else:
156 opening_post = None
161 opening_post = None
157 tag_tags = tag_content.find(TAG_TAGS)
162 tag_tags = tag_content.find(TAG_TAGS)
@@ -163,12 +168,23 class SyncManager:
163 # TODO Check that the replied posts are already present
168 # TODO Check that the replied posts are already present
164 # before adding new ones
169 # before adding new ones
165
170
166 # TODO Get images
171 files = []
172 tag_attachments = tag_content.find(TAG_ATTACHMENTS) or list()
173 tag_refs = tag_model.find(TAG_ATTACHMENT_REFS)
174 for attachment in tag_attachments:
175 tag_ref = tag_refs.find("{}[@ref='{}']".format(
176 TAG_ATTACHMENT_REF, attachment.text))
177 url = tag_ref.get(ATTR_URL)
178 attached_file = download(hostname + url)
179 if attached_file is None:
180 raise SyncException('File was not dowloaded')
181 files.append(attached_file)
182 # TODO Check hash
167
183
168 post = Post.objects.import_post(
184 Post.objects.import_post(
169 title=title, text=text, pub_time=pub_time,
185 title=title, text=text, pub_time=pub_time,
170 opening_post=opening_post, tags=tags,
186 opening_post=opening_post, tags=tags,
171 global_id=global_id)
187 global_id=global_id, files=files)
172 else:
188 else:
173 # TODO Throw an exception?
189 # TODO Throw an exception?
174 pass
190 pass
@@ -210,7 +226,7 class SyncManager:
210
226
211 if not KeyPair.objects.verify(
227 if not KeyPair.objects.verify(
212 signature, content):
228 signature, content):
213 raise Exception('Invalid model signature for {}'.format(content))
229 raise SyncException('Invalid model signature for {}'.format(content))
214
230
215 signatures.append(signature)
231 signatures.append(signature)
216
232
@@ -59,7 +59,7 class SyncTest(TestCase):
59
59
60 KeyPair.objects.generate_key(primary=True)
60 KeyPair.objects.generate_key(primary=True)
61
61
62 SyncManager.parse_response_get(response)
62 SyncManager.parse_response_get(response, None)
63 self.assertEqual(1, Post.objects.count(),
63 self.assertEqual(1, Post.objects.count(),
64 'Post was not created from XML response.')
64 'Post was not created from XML response.')
65
65
@@ -68,7 +68,7 class SyncTest(TestCase):
68 parsed_post.get_thread().get_tags().first().name,
68 parsed_post.get_thread().get_tags().first().name,
69 'Invalid tag was parsed.')
69 'Invalid tag was parsed.')
70
70
71 SyncManager.parse_response_get(response)
71 SyncManager.parse_response_get(response, None)
72 self.assertEqual(1, Post.objects.count(),
72 self.assertEqual(1, Post.objects.count(),
73 'The same post was imported twice.')
73 'The same post was imported twice.')
74
74
@@ -1,4 +1,6
1 import xml.etree.ElementTree as et
1 import xml.etree.ElementTree as et
2 import xml.dom.minidom
3
2 from django.http import HttpResponse, Http404
4 from django.http import HttpResponse, Http404
3 from boards.models import GlobalId, Post
5 from boards.models import GlobalId, Post
4 from boards.models.post.sync import SyncManager
6 from boards.models.post.sync import SyncManager
@@ -46,9 +48,13 def get_post_sync_data(request, post_id)
46 except Post.DoesNotExist:
48 except Post.DoesNotExist:
47 raise Http404()
49 raise Http404()
48
50
49 content = 'Global ID: %s\n\nXML: %s' \
51 xml_str = SyncManager.generate_response_get([post])
50 % (post.global_id, SyncManager.generate_response_get([post]))
51
52
53 xml_repr = xml.dom.minidom.parseString(xml_str)
54 xml_repr = xml_repr.toprettyxml()
55
56 content = '=Global ID=\n%s\n\n=XML=\n%s' \
57 % (post.global_id, xml_repr)
52
58
53 return HttpResponse(
59 return HttpResponse(
54 content_type='text/plain',
60 content_type='text/plain',
General Comments 0
You need to be logged in to leave comments. Login now