##// END OF EJS Templates
Load global id to pull posts
neko259 -
r1562:31a6af1e default
parent child Browse files
Show More
@@ -1,280 +1,280 b''
1 import xml.etree.ElementTree as et
1 import xml.etree.ElementTree as et
2
2
3 from boards.models.attachment.downloaders import download
3 from boards.models.attachment.downloaders import download
4 from boards.utils import get_file_mimetype, get_file_hash
4 from boards.utils import get_file_mimetype, get_file_hash
5 from django.db import transaction
5 from django.db import transaction
6 from boards.models import KeyPair, GlobalId, Signature, Post, Tag
6 from boards.models import KeyPair, GlobalId, Signature, Post, Tag
7
7
8 EXCEPTION_NODE = 'Sync node returned an error: {}'
8 EXCEPTION_NODE = 'Sync node returned an error: {}'
9 EXCEPTION_OP = 'Load the OP first'
9 EXCEPTION_OP = 'Load the OP first'
10 EXCEPTION_DOWNLOAD = 'File was not downloaded'
10 EXCEPTION_DOWNLOAD = 'File was not downloaded'
11 EXCEPTION_HASH = 'File hash does not match attachment hash'
11 EXCEPTION_HASH = 'File hash does not match attachment hash'
12 EXCEPTION_SIGNATURE = 'Invalid model signature for {}'
12 EXCEPTION_SIGNATURE = 'Invalid model signature for {}'
13 ENCODING_UNICODE = 'unicode'
13 ENCODING_UNICODE = 'unicode'
14
14
15 TAG_MODEL = 'model'
15 TAG_MODEL = 'model'
16 TAG_REQUEST = 'request'
16 TAG_REQUEST = 'request'
17 TAG_RESPONSE = 'response'
17 TAG_RESPONSE = 'response'
18 TAG_ID = 'id'
18 TAG_ID = 'id'
19 TAG_STATUS = 'status'
19 TAG_STATUS = 'status'
20 TAG_MODELS = 'models'
20 TAG_MODELS = 'models'
21 TAG_TITLE = 'title'
21 TAG_TITLE = 'title'
22 TAG_TEXT = 'text'
22 TAG_TEXT = 'text'
23 TAG_THREAD = 'thread'
23 TAG_THREAD = 'thread'
24 TAG_PUB_TIME = 'pub-time'
24 TAG_PUB_TIME = 'pub-time'
25 TAG_SIGNATURES = 'signatures'
25 TAG_SIGNATURES = 'signatures'
26 TAG_SIGNATURE = 'signature'
26 TAG_SIGNATURE = 'signature'
27 TAG_CONTENT = 'content'
27 TAG_CONTENT = 'content'
28 TAG_ATTACHMENTS = 'attachments'
28 TAG_ATTACHMENTS = 'attachments'
29 TAG_ATTACHMENT = 'attachment'
29 TAG_ATTACHMENT = 'attachment'
30 TAG_TAGS = 'tags'
30 TAG_TAGS = 'tags'
31 TAG_TAG = 'tag'
31 TAG_TAG = 'tag'
32 TAG_ATTACHMENT_REFS = 'attachment-refs'
32 TAG_ATTACHMENT_REFS = 'attachment-refs'
33 TAG_ATTACHMENT_REF = 'attachment-ref'
33 TAG_ATTACHMENT_REF = 'attachment-ref'
34 TAG_TRIPCODE = 'tripcode'
34 TAG_TRIPCODE = 'tripcode'
35
35
36 TYPE_GET = 'get'
36 TYPE_GET = 'get'
37
37
38 ATTR_VERSION = 'version'
38 ATTR_VERSION = 'version'
39 ATTR_TYPE = 'type'
39 ATTR_TYPE = 'type'
40 ATTR_NAME = 'name'
40 ATTR_NAME = 'name'
41 ATTR_VALUE = 'value'
41 ATTR_VALUE = 'value'
42 ATTR_MIMETYPE = 'mimetype'
42 ATTR_MIMETYPE = 'mimetype'
43 ATTR_KEY = 'key'
43 ATTR_KEY = 'key'
44 ATTR_REF = 'ref'
44 ATTR_REF = 'ref'
45 ATTR_URL = 'url'
45 ATTR_URL = 'url'
46 ATTR_ID_TYPE = 'id-type'
46 ATTR_ID_TYPE = 'id-type'
47
47
48 ID_TYPE_MD5 = 'md5'
48 ID_TYPE_MD5 = 'md5'
49
49
50 STATUS_SUCCESS = 'success'
50 STATUS_SUCCESS = 'success'
51
51
52
52
53 class SyncException(Exception):
53 class SyncException(Exception):
54 pass
54 pass
55
55
56
56
57 class SyncManager:
57 class SyncManager:
58 @staticmethod
58 @staticmethod
59 def generate_response_get(model_list: list):
59 def generate_response_get(model_list: list):
60 response = et.Element(TAG_RESPONSE)
60 response = et.Element(TAG_RESPONSE)
61
61
62 status = et.SubElement(response, TAG_STATUS)
62 status = et.SubElement(response, TAG_STATUS)
63 status.text = STATUS_SUCCESS
63 status.text = STATUS_SUCCESS
64
64
65 models = et.SubElement(response, TAG_MODELS)
65 models = et.SubElement(response, TAG_MODELS)
66
66
67 for post in model_list:
67 for post in model_list:
68 model = et.SubElement(models, TAG_MODEL)
68 model = et.SubElement(models, TAG_MODEL)
69 model.set(ATTR_NAME, 'post')
69 model.set(ATTR_NAME, 'post')
70
70
71 global_id = post.global_id
71 global_id = post.global_id
72
72
73 images = post.images.all()
73 images = post.images.all()
74 attachments = post.attachments.all()
74 attachments = post.attachments.all()
75 if global_id.content:
75 if global_id.content:
76 model.append(et.fromstring(global_id.content))
76 model.append(et.fromstring(global_id.content))
77 if len(images) > 0 or len(attachments) > 0:
77 if len(images) > 0 or len(attachments) > 0:
78 attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS)
78 attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS)
79 for image in images:
79 for image in images:
80 SyncManager._attachment_to_xml(
80 SyncManager._attachment_to_xml(
81 None, attachment_refs, image.image.file,
81 None, attachment_refs, image.image.file,
82 image.hash, image.image.url)
82 image.hash, image.image.url)
83 for file in attachments:
83 for file in attachments:
84 SyncManager._attachment_to_xml(
84 SyncManager._attachment_to_xml(
85 None, attachment_refs, file.file.file,
85 None, attachment_refs, file.file.file,
86 file.hash, file.file.url)
86 file.hash, file.file.url)
87 else:
87 else:
88 content_tag = et.SubElement(model, TAG_CONTENT)
88 content_tag = et.SubElement(model, TAG_CONTENT)
89
89
90 tag_id = et.SubElement(content_tag, TAG_ID)
90 tag_id = et.SubElement(content_tag, TAG_ID)
91 global_id.to_xml_element(tag_id)
91 global_id.to_xml_element(tag_id)
92
92
93 title = et.SubElement(content_tag, TAG_TITLE)
93 title = et.SubElement(content_tag, TAG_TITLE)
94 title.text = post.title
94 title.text = post.title
95
95
96 text = et.SubElement(content_tag, TAG_TEXT)
96 text = et.SubElement(content_tag, TAG_TEXT)
97 text.text = post.get_sync_text()
97 text.text = post.get_sync_text()
98
98
99 thread = post.get_thread()
99 thread = post.get_thread()
100 if post.is_opening():
100 if post.is_opening():
101 tag_tags = et.SubElement(content_tag, TAG_TAGS)
101 tag_tags = et.SubElement(content_tag, TAG_TAGS)
102 for tag in thread.get_tags():
102 for tag in thread.get_tags():
103 tag_tag = et.SubElement(tag_tags, TAG_TAG)
103 tag_tag = et.SubElement(tag_tags, TAG_TAG)
104 tag_tag.text = tag.name
104 tag_tag.text = tag.name
105 else:
105 else:
106 tag_thread = et.SubElement(content_tag, TAG_THREAD)
106 tag_thread = et.SubElement(content_tag, TAG_THREAD)
107 thread_id = et.SubElement(tag_thread, TAG_ID)
107 thread_id = et.SubElement(tag_thread, TAG_ID)
108 thread.get_opening_post().global_id.to_xml_element(thread_id)
108 thread.get_opening_post().global_id.to_xml_element(thread_id)
109
109
110 pub_time = et.SubElement(content_tag, TAG_PUB_TIME)
110 pub_time = et.SubElement(content_tag, TAG_PUB_TIME)
111 pub_time.text = str(post.get_pub_time_str())
111 pub_time.text = str(post.get_pub_time_str())
112
112
113 if post.tripcode:
113 if post.tripcode:
114 tripcode = et.SubElement(content_tag, TAG_TRIPCODE)
114 tripcode = et.SubElement(content_tag, TAG_TRIPCODE)
115 tripcode.text = post.tripcode
115 tripcode.text = post.tripcode
116
116
117 if len(images) > 0 or len(attachments) > 0:
117 if len(images) > 0 or len(attachments) > 0:
118 attachments_tag = et.SubElement(content_tag, TAG_ATTACHMENTS)
118 attachments_tag = et.SubElement(content_tag, TAG_ATTACHMENTS)
119 attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS)
119 attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS)
120
120
121 for image in images:
121 for image in images:
122 SyncManager._attachment_to_xml(
122 SyncManager._attachment_to_xml(
123 attachments_tag, attachment_refs, image.image.file,
123 attachments_tag, attachment_refs, image.image.file,
124 image.hash, image.image.url)
124 image.hash, image.image.url)
125 for file in attachments:
125 for file in attachments:
126 SyncManager._attachment_to_xml(
126 SyncManager._attachment_to_xml(
127 attachments_tag, attachment_refs, file.file.file,
127 attachments_tag, attachment_refs, file.file.file,
128 file.hash, file.file.url)
128 file.hash, file.file.url)
129
129
130 global_id.content = et.tostring(content_tag, ENCODING_UNICODE)
130 global_id.content = et.tostring(content_tag, ENCODING_UNICODE)
131 global_id.save()
131 global_id.save()
132
132
133 signatures_tag = et.SubElement(model, TAG_SIGNATURES)
133 signatures_tag = et.SubElement(model, TAG_SIGNATURES)
134 post_signatures = global_id.signature_set.all()
134 post_signatures = global_id.signature_set.all()
135 if post_signatures:
135 if post_signatures:
136 signatures = post_signatures
136 signatures = post_signatures
137 else:
137 else:
138 key = KeyPair.objects.get(public_key=global_id.key)
138 key = KeyPair.objects.get(public_key=global_id.key)
139 signature = Signature(
139 signature = Signature(
140 key_type=key.key_type,
140 key_type=key.key_type,
141 key=key.public_key,
141 key=key.public_key,
142 signature=key.sign(global_id.content),
142 signature=key.sign(global_id.content),
143 global_id=global_id,
143 global_id=global_id,
144 )
144 )
145 signature.save()
145 signature.save()
146 signatures = [signature]
146 signatures = [signature]
147 for signature in signatures:
147 for signature in signatures:
148 signature_tag = et.SubElement(signatures_tag, TAG_SIGNATURE)
148 signature_tag = et.SubElement(signatures_tag, TAG_SIGNATURE)
149 signature_tag.set(ATTR_TYPE, signature.key_type)
149 signature_tag.set(ATTR_TYPE, signature.key_type)
150 signature_tag.set(ATTR_VALUE, signature.signature)
150 signature_tag.set(ATTR_VALUE, signature.signature)
151 signature_tag.set(ATTR_KEY, signature.key)
151 signature_tag.set(ATTR_KEY, signature.key)
152
152
153 return et.tostring(response, ENCODING_UNICODE)
153 return et.tostring(response, ENCODING_UNICODE)
154
154
155 @staticmethod
155 @staticmethod
156 @transaction.atomic
156 @transaction.atomic
157 def parse_response_get(response_xml, hostname):
157 def parse_response_get(response_xml, hostname):
158 tag_root = et.fromstring(response_xml)
158 tag_root = et.fromstring(response_xml)
159 tag_status = tag_root.find(TAG_STATUS)
159 tag_status = tag_root.find(TAG_STATUS)
160 if STATUS_SUCCESS == tag_status.text:
160 if STATUS_SUCCESS == tag_status.text:
161 tag_models = tag_root.find(TAG_MODELS)
161 tag_models = tag_root.find(TAG_MODELS)
162 for tag_model in tag_models:
162 for tag_model in tag_models:
163 tag_content = tag_model.find(TAG_CONTENT)
163 tag_content = tag_model.find(TAG_CONTENT)
164
164
165 content_str = et.tostring(tag_content, ENCODING_UNICODE)
165 content_str = et.tostring(tag_content, ENCODING_UNICODE)
166 signatures = SyncManager._verify_model(content_str, tag_model)
166 signatures = SyncManager._verify_model(content_str, tag_model)
167
167
168 tag_id = tag_content.find(TAG_ID)
168 tag_id = tag_content.find(TAG_ID)
169 global_id, exists = GlobalId.from_xml_element(tag_id)
169 global_id, exists = GlobalId.from_xml_element(tag_id)
170
170
171 if exists:
171 if exists:
172 print('Post with same ID already exists')
172 print('Post with same ID already exists')
173 else:
173 else:
174 global_id.content = content_str
174 global_id.content = content_str
175 global_id.save()
175 global_id.save()
176 for signature in signatures:
176 for signature in signatures:
177 signature.global_id = global_id
177 signature.global_id = global_id
178 signature.save()
178 signature.save()
179
179
180 title = tag_content.find(TAG_TITLE).text or ''
180 title = tag_content.find(TAG_TITLE).text or ''
181 text = tag_content.find(TAG_TEXT).text or ''
181 text = tag_content.find(TAG_TEXT).text or ''
182 pub_time = tag_content.find(TAG_PUB_TIME).text
182 pub_time = tag_content.find(TAG_PUB_TIME).text
183 tripcode = tag_content.find(TAG_TRIPCODE).text
183 tripcode = tag_content.find(TAG_TRIPCODE).text
184
184
185 thread = tag_content.find(TAG_THREAD)
185 thread = tag_content.find(TAG_THREAD)
186 tags = []
186 tags = []
187 if thread:
187 if thread:
188 thread_id = thread.find(TAG_ID)
188 thread_id = thread.find(TAG_ID)
189 op_global_id, exists = GlobalId.from_xml_element(thread_id)
189 op_global_id, exists = GlobalId.from_xml_element(thread_id)
190 if exists:
190 if exists:
191 opening_post = Post.objects.get(global_id=op_global_id)
191 opening_post = Post.objects.get(global_id=op_global_id)
192 else:
192 else:
193 raise SyncException(EXCEPTION_OP)
193 raise SyncException(EXCEPTION_OP)
194 else:
194 else:
195 opening_post = None
195 opening_post = None
196 tag_tags = tag_content.find(TAG_TAGS)
196 tag_tags = tag_content.find(TAG_TAGS)
197 for tag_tag in tag_tags:
197 for tag_tag in tag_tags:
198 tag, created = Tag.objects.get_or_create(
198 tag, created = Tag.objects.get_or_create(
199 name=tag_tag.text)
199 name=tag_tag.text)
200 tags.append(tag)
200 tags.append(tag)
201
201
202 # TODO Check that the replied posts are already present
202 # TODO Check that the replied posts are already present
203 # before adding new ones
203 # before adding new ones
204
204
205 files = []
205 files = []
206 tag_attachments = tag_content.find(TAG_ATTACHMENTS) or list()
206 tag_attachments = tag_content.find(TAG_ATTACHMENTS) or list()
207 tag_refs = tag_model.find(TAG_ATTACHMENT_REFS)
207 tag_refs = tag_model.find(TAG_ATTACHMENT_REFS)
208 for attachment in tag_attachments:
208 for attachment in tag_attachments:
209 tag_ref = tag_refs.find("{}[@ref='{}']".format(
209 tag_ref = tag_refs.find("{}[@ref='{}']".format(
210 TAG_ATTACHMENT_REF, attachment.text))
210 TAG_ATTACHMENT_REF, attachment.text))
211 url = tag_ref.get(ATTR_URL)
211 url = tag_ref.get(ATTR_URL)
212 attached_file = download(hostname + url)
212 attached_file = download(hostname + url)
213 if attached_file is None:
213 if attached_file is None:
214 raise SyncException(EXCEPTION_DOWNLOAD)
214 raise SyncException(EXCEPTION_DOWNLOAD)
215
215
216 hash = get_file_hash(attached_file)
216 hash = get_file_hash(attached_file)
217 if hash != attachment.text:
217 if hash != attachment.text:
218 raise SyncException(EXCEPTION_HASH)
218 raise SyncException(EXCEPTION_HASH)
219
219
220 files.append(attached_file)
220 files.append(attached_file)
221
221
222 Post.objects.import_post(
222 Post.objects.import_post(
223 title=title, text=text, pub_time=pub_time,
223 title=title, text=text, pub_time=pub_time,
224 opening_post=opening_post, tags=tags,
224 opening_post=opening_post, tags=tags,
225 global_id=global_id, files=files, tripcode=tripcode)
225 global_id=global_id, files=files, tripcode=tripcode)
226 else:
226 else:
227 raise SyncException(EXCEPTION_NODE.format(tag_status.text))
227 raise SyncException(EXCEPTION_NODE.format(tag_status.text))
228
228
229 @staticmethod
229 @staticmethod
230 def generate_response_pull():
230 def generate_response_pull():
231 response = et.Element(TAG_RESPONSE)
231 response = et.Element(TAG_RESPONSE)
232
232
233 status = et.SubElement(response, TAG_STATUS)
233 status = et.SubElement(response, TAG_STATUS)
234 status.text = STATUS_SUCCESS
234 status.text = STATUS_SUCCESS
235
235
236 models = et.SubElement(response, TAG_MODELS)
236 models = et.SubElement(response, TAG_MODELS)
237
237
238 for post in Post.objects.all():
238 for post in Post.objects.prefetch_related('global_id').all():
239 tag_id = et.SubElement(models, TAG_ID)
239 tag_id = et.SubElement(models, TAG_ID)
240 post.global_id.to_xml_element(tag_id)
240 post.global_id.to_xml_element(tag_id)
241
241
242 return et.tostring(response, ENCODING_UNICODE)
242 return et.tostring(response, ENCODING_UNICODE)
243
243
244 @staticmethod
244 @staticmethod
245 def _verify_model(content_str, tag_model):
245 def _verify_model(content_str, tag_model):
246 """
246 """
247 Verifies all signatures for a single model.
247 Verifies all signatures for a single model.
248 """
248 """
249
249
250 signatures = []
250 signatures = []
251
251
252 tag_signatures = tag_model.find(TAG_SIGNATURES)
252 tag_signatures = tag_model.find(TAG_SIGNATURES)
253 for tag_signature in tag_signatures:
253 for tag_signature in tag_signatures:
254 signature_type = tag_signature.get(ATTR_TYPE)
254 signature_type = tag_signature.get(ATTR_TYPE)
255 signature_value = tag_signature.get(ATTR_VALUE)
255 signature_value = tag_signature.get(ATTR_VALUE)
256 signature_key = tag_signature.get(ATTR_KEY)
256 signature_key = tag_signature.get(ATTR_KEY)
257
257
258 signature = Signature(key_type=signature_type,
258 signature = Signature(key_type=signature_type,
259 key=signature_key,
259 key=signature_key,
260 signature=signature_value)
260 signature=signature_value)
261
261
262 if not KeyPair.objects.verify(signature, content_str):
262 if not KeyPair.objects.verify(signature, content_str):
263 raise SyncException(EXCEPTION_SIGNATURE.format(content_str))
263 raise SyncException(EXCEPTION_SIGNATURE.format(content_str))
264
264
265 signatures.append(signature)
265 signatures.append(signature)
266
266
267 return signatures
267 return signatures
268
268
269 @staticmethod
269 @staticmethod
270 def _attachment_to_xml(tag_attachments, tag_refs, file, hash, url):
270 def _attachment_to_xml(tag_attachments, tag_refs, file, hash, url):
271 if tag_attachments is not None:
271 if tag_attachments is not None:
272 mimetype = get_file_mimetype(file)
272 mimetype = get_file_mimetype(file)
273 attachment = et.SubElement(tag_attachments, TAG_ATTACHMENT)
273 attachment = et.SubElement(tag_attachments, TAG_ATTACHMENT)
274 attachment.set(ATTR_MIMETYPE, mimetype)
274 attachment.set(ATTR_MIMETYPE, mimetype)
275 attachment.set(ATTR_ID_TYPE, ID_TYPE_MD5)
275 attachment.set(ATTR_ID_TYPE, ID_TYPE_MD5)
276 attachment.text = hash
276 attachment.text = hash
277
277
278 attachment_ref = et.SubElement(tag_refs, TAG_ATTACHMENT_REF)
278 attachment_ref = et.SubElement(tag_refs, TAG_ATTACHMENT_REF)
279 attachment_ref.set(ATTR_REF, hash)
279 attachment_ref.set(ATTR_REF, hash)
280 attachment_ref.set(ATTR_URL, url)
280 attachment_ref.set(ATTR_URL, url)
General Comments 0
You need to be logged in to leave comments. Login now