##// END OF EJS Templates
Added post versions to list request, changed list request to include additional attributes
neko259 -
r1571:7712a723 default
parent child Browse files
Show More
@@ -1,85 +1,90 b''
1 import re
1 import re
2 import xml.etree.ElementTree as ET
2 import xml.etree.ElementTree as ET
3
3
4 import httplib2
4 import httplib2
5 from django.core.management import BaseCommand
5 from django.core.management import BaseCommand
6
6
7 from boards.models import GlobalId
7 from boards.models import GlobalId
8 from boards.models.post.sync import SyncManager
8 from boards.models.post.sync import SyncManager, TAG_ID, TAG_VERSION
9
9
10 __author__ = 'neko259'
10 __author__ = 'neko259'
11
11
12
12
13 REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)')
13 REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)')
14
14
15
15
16 class Command(BaseCommand):
16 class Command(BaseCommand):
17 help = 'Send a sync or get request to the server.'
17 help = 'Send a sync or get request to the server.'
18
18
19 def add_arguments(self, parser):
19 def add_arguments(self, parser):
20 parser.add_argument('url', type=str, help='Server root url')
20 parser.add_argument('url', type=str, help='Server root url')
21 parser.add_argument('--global-id', type=str, default='',
21 parser.add_argument('--global-id', type=str, default='',
22 help='Post global ID')
22 help='Post global ID')
23 parser.add_argument('--split-query', type=int,
23 parser.add_argument('--split-query', type=int,
24 help='Split GET query into separate by the given'
24 help='Split GET query into separate by the given'
25 ' number of posts in one')
25 ' number of posts in one')
26
26
27 def handle(self, *args, **options):
27 def handle(self, *args, **options):
28 url = options.get('url')
28 url = options.get('url')
29
29
30 list_url = url + 'api/sync/list/'
30 list_url = url + 'api/sync/list/'
31 get_url = url + 'api/sync/get/'
31 get_url = url + 'api/sync/get/'
32 file_url = url[:-1]
32 file_url = url[:-1]
33
33
34 global_id_str = options.get('global_id')
34 global_id_str = options.get('global_id')
35 if global_id_str:
35 if global_id_str:
36 match = REGEX_GLOBAL_ID.match(global_id_str)
36 match = REGEX_GLOBAL_ID.match(global_id_str)
37 if match:
37 if match:
38 key_type = match.group(1)
38 key_type = match.group(1)
39 key = match.group(2)
39 key = match.group(2)
40 local_id = match.group(3)
40 local_id = match.group(3)
41
41
42 global_id = GlobalId(key_type=key_type, key=key,
42 global_id = GlobalId(key_type=key_type, key=key,
43 local_id=local_id)
43 local_id=local_id)
44
44
45 xml = GlobalId.objects.generate_request_get([global_id])
45 xml = GlobalId.objects.generate_request_get([global_id])
46 # body = urllib.parse.urlencode(data)
46 # body = urllib.parse.urlencode(data)
47 h = httplib2.Http()
47 h = httplib2.Http()
48 response, content = h.request(get_url, method="POST", body=xml)
48 response, content = h.request(get_url, method="POST", body=xml)
49
49
50 SyncManager.parse_response_get(content, file_url)
50 SyncManager.parse_response_get(content, file_url)
51 else:
51 else:
52 raise Exception('Invalid global ID')
52 raise Exception('Invalid global ID')
53 else:
53 else:
54 h = httplib2.Http()
54 h = httplib2.Http()
55 xml = GlobalId.objects.generate_request_list()
55 xml = GlobalId.objects.generate_request_list()
56 response, content = h.request(list_url, method="POST", body=xml)
56 response, content = h.request(list_url, method="POST", body=xml)
57
57
58 print(content.decode() + '\n')
58 print(content.decode() + '\n')
59
59
60 root = ET.fromstring(content)
60 root = ET.fromstring(content)
61 status = root.findall('status')[0].text
61 status = root.findall('status')[0].text
62 if status == 'success':
62 if status == 'success':
63 ids_to_sync = list()
63 ids_to_sync = list()
64
64
65 models = root.findall('models')[0]
65 models = root.findall('models')[0]
66 for model in models:
66 for model in models:
67 tag_id = model.find(TAG_ID)
67 global_id, exists = GlobalId.from_xml_element(model)
68 global_id, exists = GlobalId.from_xml_element(model)
68 if not exists:
69 tag_version = model.find(TAG_VERSION)
69 print(global_id)
70 if tag_version is not None:
71 version = int(tag_version.text) or 1
72 else:
73 version = 1
74 if not exists or global_id.post.version < version:
70 ids_to_sync.append(global_id)
75 ids_to_sync.append(global_id)
71 print()
76 print('Starting sync...')
72
77
73 if len(ids_to_sync) > 0:
78 if len(ids_to_sync) > 0:
74 limit = options.get('split_query', len(ids_to_sync))
79 limit = options.get('split_query', len(ids_to_sync))
75 for offset in range(0, len(ids_to_sync), limit):
80 for offset in range(0, len(ids_to_sync), limit):
76 xml = GlobalId.objects.generate_request_get(ids_to_sync[offset:offset+limit])
81 xml = GlobalId.objects.generate_request_get(ids_to_sync[offset:offset+limit])
77 # body = urllib.parse.urlencode(data)
82 # body = urllib.parse.urlencode(data)
78 h = httplib2.Http()
83 h = httplib2.Http()
79 response, content = h.request(get_url, method="POST", body=xml)
84 response, content = h.request(get_url, method="POST", body=xml)
80
85
81 SyncManager.parse_response_get(content, file_url)
86 SyncManager.parse_response_get(content, file_url)
82 else:
87 else:
83 print('Nothing to get, everything synced')
88 print('Nothing to get, everything synced')
84 else:
89 else:
85 raise Exception('Invalid response status')
90 raise Exception('Invalid response status')
@@ -1,287 +1,291 b''
1 import xml.etree.ElementTree as et
1 import xml.etree.ElementTree as et
2
2
3 from boards.models.attachment.downloaders import download
3 from boards.models.attachment.downloaders import download
4 from boards.utils import get_file_mimetype, get_file_hash
4 from boards.utils import get_file_mimetype, get_file_hash
5 from django.db import transaction
5 from django.db import transaction
6 from boards.models import KeyPair, GlobalId, Signature, Post, Tag
6 from boards.models import KeyPair, GlobalId, Signature, Post, Tag
7
7
8 EXCEPTION_NODE = 'Sync node returned an error: {}'
8 EXCEPTION_NODE = 'Sync node returned an error: {}'
9 EXCEPTION_OP = 'Load the OP first'
9 EXCEPTION_OP = 'Load the OP first'
10 EXCEPTION_DOWNLOAD = 'File was not downloaded'
10 EXCEPTION_DOWNLOAD = 'File was not downloaded'
11 EXCEPTION_HASH = 'File hash does not match attachment hash'
11 EXCEPTION_HASH = 'File hash does not match attachment hash'
12 EXCEPTION_SIGNATURE = 'Invalid model signature for {}'
12 EXCEPTION_SIGNATURE = 'Invalid model signature for {}'
13 ENCODING_UNICODE = 'unicode'
13 ENCODING_UNICODE = 'unicode'
14
14
15 TAG_MODEL = 'model'
15 TAG_MODEL = 'model'
16 TAG_REQUEST = 'request'
16 TAG_REQUEST = 'request'
17 TAG_RESPONSE = 'response'
17 TAG_RESPONSE = 'response'
18 TAG_ID = 'id'
18 TAG_ID = 'id'
19 TAG_STATUS = 'status'
19 TAG_STATUS = 'status'
20 TAG_MODELS = 'models'
20 TAG_MODELS = 'models'
21 TAG_TITLE = 'title'
21 TAG_TITLE = 'title'
22 TAG_TEXT = 'text'
22 TAG_TEXT = 'text'
23 TAG_THREAD = 'thread'
23 TAG_THREAD = 'thread'
24 TAG_PUB_TIME = 'pub-time'
24 TAG_PUB_TIME = 'pub-time'
25 TAG_SIGNATURES = 'signatures'
25 TAG_SIGNATURES = 'signatures'
26 TAG_SIGNATURE = 'signature'
26 TAG_SIGNATURE = 'signature'
27 TAG_CONTENT = 'content'
27 TAG_CONTENT = 'content'
28 TAG_ATTACHMENTS = 'attachments'
28 TAG_ATTACHMENTS = 'attachments'
29 TAG_ATTACHMENT = 'attachment'
29 TAG_ATTACHMENT = 'attachment'
30 TAG_TAGS = 'tags'
30 TAG_TAGS = 'tags'
31 TAG_TAG = 'tag'
31 TAG_TAG = 'tag'
32 TAG_ATTACHMENT_REFS = 'attachment-refs'
32 TAG_ATTACHMENT_REFS = 'attachment-refs'
33 TAG_ATTACHMENT_REF = 'attachment-ref'
33 TAG_ATTACHMENT_REF = 'attachment-ref'
34 TAG_TRIPCODE = 'tripcode'
34 TAG_TRIPCODE = 'tripcode'
35 TAG_VERSION = 'version'
35 TAG_VERSION = 'version'
36
36
37 TYPE_GET = 'get'
37 TYPE_GET = 'get'
38
38
39 ATTR_VERSION = 'version'
39 ATTR_VERSION = 'version'
40 ATTR_TYPE = 'type'
40 ATTR_TYPE = 'type'
41 ATTR_NAME = 'name'
41 ATTR_NAME = 'name'
42 ATTR_VALUE = 'value'
42 ATTR_VALUE = 'value'
43 ATTR_MIMETYPE = 'mimetype'
43 ATTR_MIMETYPE = 'mimetype'
44 ATTR_KEY = 'key'
44 ATTR_KEY = 'key'
45 ATTR_REF = 'ref'
45 ATTR_REF = 'ref'
46 ATTR_URL = 'url'
46 ATTR_URL = 'url'
47 ATTR_ID_TYPE = 'id-type'
47 ATTR_ID_TYPE = 'id-type'
48
48
49 ID_TYPE_MD5 = 'md5'
49 ID_TYPE_MD5 = 'md5'
50
50
51 STATUS_SUCCESS = 'success'
51 STATUS_SUCCESS = 'success'
52
52
53
53
54 class SyncException(Exception):
54 class SyncException(Exception):
55 pass
55 pass
56
56
57
57
58 class SyncManager:
58 class SyncManager:
59 @staticmethod
59 @staticmethod
60 def generate_response_get(model_list: list):
60 def generate_response_get(model_list: list):
61 response = et.Element(TAG_RESPONSE)
61 response = et.Element(TAG_RESPONSE)
62
62
63 status = et.SubElement(response, TAG_STATUS)
63 status = et.SubElement(response, TAG_STATUS)
64 status.text = STATUS_SUCCESS
64 status.text = STATUS_SUCCESS
65
65
66 models = et.SubElement(response, TAG_MODELS)
66 models = et.SubElement(response, TAG_MODELS)
67
67
68 for post in model_list:
68 for post in model_list:
69 model = et.SubElement(models, TAG_MODEL)
69 model = et.SubElement(models, TAG_MODEL)
70 model.set(ATTR_NAME, 'post')
70 model.set(ATTR_NAME, 'post')
71
71
72 global_id = post.global_id
72 global_id = post.global_id
73
73
74 images = post.images.all()
74 images = post.images.all()
75 attachments = post.attachments.all()
75 attachments = post.attachments.all()
76 if global_id.content:
76 if global_id.content:
77 model.append(et.fromstring(global_id.content))
77 model.append(et.fromstring(global_id.content))
78 if len(images) > 0 or len(attachments) > 0:
78 if len(images) > 0 or len(attachments) > 0:
79 attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS)
79 attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS)
80 for image in images:
80 for image in images:
81 SyncManager._attachment_to_xml(
81 SyncManager._attachment_to_xml(
82 None, attachment_refs, image.image.file,
82 None, attachment_refs, image.image.file,
83 image.hash, image.image.url)
83 image.hash, image.image.url)
84 for file in attachments:
84 for file in attachments:
85 SyncManager._attachment_to_xml(
85 SyncManager._attachment_to_xml(
86 None, attachment_refs, file.file.file,
86 None, attachment_refs, file.file.file,
87 file.hash, file.file.url)
87 file.hash, file.file.url)
88 else:
88 else:
89 content_tag = et.SubElement(model, TAG_CONTENT)
89 content_tag = et.SubElement(model, TAG_CONTENT)
90
90
91 tag_id = et.SubElement(content_tag, TAG_ID)
91 tag_id = et.SubElement(content_tag, TAG_ID)
92 global_id.to_xml_element(tag_id)
92 global_id.to_xml_element(tag_id)
93
93
94 title = et.SubElement(content_tag, TAG_TITLE)
94 title = et.SubElement(content_tag, TAG_TITLE)
95 title.text = post.title
95 title.text = post.title
96
96
97 text = et.SubElement(content_tag, TAG_TEXT)
97 text = et.SubElement(content_tag, TAG_TEXT)
98 text.text = post.get_sync_text()
98 text.text = post.get_sync_text()
99
99
100 thread = post.get_thread()
100 thread = post.get_thread()
101 if post.is_opening():
101 if post.is_opening():
102 tag_tags = et.SubElement(content_tag, TAG_TAGS)
102 tag_tags = et.SubElement(content_tag, TAG_TAGS)
103 for tag in thread.get_tags():
103 for tag in thread.get_tags():
104 tag_tag = et.SubElement(tag_tags, TAG_TAG)
104 tag_tag = et.SubElement(tag_tags, TAG_TAG)
105 tag_tag.text = tag.name
105 tag_tag.text = tag.name
106 else:
106 else:
107 tag_thread = et.SubElement(content_tag, TAG_THREAD)
107 tag_thread = et.SubElement(content_tag, TAG_THREAD)
108 thread_id = et.SubElement(tag_thread, TAG_ID)
108 thread_id = et.SubElement(tag_thread, TAG_ID)
109 thread.get_opening_post().global_id.to_xml_element(thread_id)
109 thread.get_opening_post().global_id.to_xml_element(thread_id)
110
110
111 pub_time = et.SubElement(content_tag, TAG_PUB_TIME)
111 pub_time = et.SubElement(content_tag, TAG_PUB_TIME)
112 pub_time.text = str(post.get_pub_time_str())
112 pub_time.text = str(post.get_pub_time_str())
113
113
114 if post.tripcode:
114 if post.tripcode:
115 tripcode = et.SubElement(content_tag, TAG_TRIPCODE)
115 tripcode = et.SubElement(content_tag, TAG_TRIPCODE)
116 tripcode.text = post.tripcode
116 tripcode.text = post.tripcode
117
117
118 if len(images) > 0 or len(attachments) > 0:
118 if len(images) > 0 or len(attachments) > 0:
119 attachments_tag = et.SubElement(content_tag, TAG_ATTACHMENTS)
119 attachments_tag = et.SubElement(content_tag, TAG_ATTACHMENTS)
120 attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS)
120 attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS)
121
121
122 for image in images:
122 for image in images:
123 SyncManager._attachment_to_xml(
123 SyncManager._attachment_to_xml(
124 attachments_tag, attachment_refs, image.image.file,
124 attachments_tag, attachment_refs, image.image.file,
125 image.hash, image.image.url)
125 image.hash, image.image.url)
126 for file in attachments:
126 for file in attachments:
127 SyncManager._attachment_to_xml(
127 SyncManager._attachment_to_xml(
128 attachments_tag, attachment_refs, file.file.file,
128 attachments_tag, attachment_refs, file.file.file,
129 file.hash, file.file.url)
129 file.hash, file.file.url)
130 version_tag = et.SubElement(content_tag, TAG_VERSION)
130 version_tag = et.SubElement(content_tag, TAG_VERSION)
131 version_tag.text = str(post.version)
131 version_tag.text = str(post.version)
132
132
133 global_id.content = et.tostring(content_tag, ENCODING_UNICODE)
133 global_id.content = et.tostring(content_tag, ENCODING_UNICODE)
134 global_id.save()
134 global_id.save()
135
135
136 signatures_tag = et.SubElement(model, TAG_SIGNATURES)
136 signatures_tag = et.SubElement(model, TAG_SIGNATURES)
137 post_signatures = global_id.signature_set.all()
137 post_signatures = global_id.signature_set.all()
138 if post_signatures:
138 if post_signatures:
139 signatures = post_signatures
139 signatures = post_signatures
140 else:
140 else:
141 key = KeyPair.objects.get(public_key=global_id.key)
141 key = KeyPair.objects.get(public_key=global_id.key)
142 signature = Signature(
142 signature = Signature(
143 key_type=key.key_type,
143 key_type=key.key_type,
144 key=key.public_key,
144 key=key.public_key,
145 signature=key.sign(global_id.content),
145 signature=key.sign(global_id.content),
146 global_id=global_id,
146 global_id=global_id,
147 )
147 )
148 signature.save()
148 signature.save()
149 signatures = [signature]
149 signatures = [signature]
150 for signature in signatures:
150 for signature in signatures:
151 signature_tag = et.SubElement(signatures_tag, TAG_SIGNATURE)
151 signature_tag = et.SubElement(signatures_tag, TAG_SIGNATURE)
152 signature_tag.set(ATTR_TYPE, signature.key_type)
152 signature_tag.set(ATTR_TYPE, signature.key_type)
153 signature_tag.set(ATTR_VALUE, signature.signature)
153 signature_tag.set(ATTR_VALUE, signature.signature)
154 signature_tag.set(ATTR_KEY, signature.key)
154 signature_tag.set(ATTR_KEY, signature.key)
155
155
156 return et.tostring(response, ENCODING_UNICODE)
156 return et.tostring(response, ENCODING_UNICODE)
157
157
158 @staticmethod
158 @staticmethod
159 @transaction.atomic
159 @transaction.atomic
160 def parse_response_get(response_xml, hostname):
160 def parse_response_get(response_xml, hostname):
161 tag_root = et.fromstring(response_xml)
161 tag_root = et.fromstring(response_xml)
162 tag_status = tag_root.find(TAG_STATUS)
162 tag_status = tag_root.find(TAG_STATUS)
163 if STATUS_SUCCESS == tag_status.text:
163 if STATUS_SUCCESS == tag_status.text:
164 tag_models = tag_root.find(TAG_MODELS)
164 tag_models = tag_root.find(TAG_MODELS)
165 for tag_model in tag_models:
165 for tag_model in tag_models:
166 tag_content = tag_model.find(TAG_CONTENT)
166 tag_content = tag_model.find(TAG_CONTENT)
167
167
168 content_str = et.tostring(tag_content, ENCODING_UNICODE)
168 content_str = et.tostring(tag_content, ENCODING_UNICODE)
169 signatures = SyncManager._verify_model(content_str, tag_model)
169 signatures = SyncManager._verify_model(content_str, tag_model)
170
170
171 tag_id = tag_content.find(TAG_ID)
171 tag_id = tag_content.find(TAG_ID)
172 global_id, exists = GlobalId.from_xml_element(tag_id)
172 global_id, exists = GlobalId.from_xml_element(tag_id)
173
173
174 if exists:
174 if exists:
175 print('Post with same ID already exists')
175 print('Post with same ID already exists')
176 else:
176 else:
177 global_id.content = content_str
177 global_id.content = content_str
178 global_id.save()
178 global_id.save()
179 for signature in signatures:
179 for signature in signatures:
180 signature.global_id = global_id
180 signature.global_id = global_id
181 signature.save()
181 signature.save()
182
182
183 title = tag_content.find(TAG_TITLE).text or ''
183 title = tag_content.find(TAG_TITLE).text or ''
184 text = tag_content.find(TAG_TEXT).text or ''
184 text = tag_content.find(TAG_TEXT).text or ''
185 pub_time = tag_content.find(TAG_PUB_TIME).text
185 pub_time = tag_content.find(TAG_PUB_TIME).text
186 tripcode_tag = tag_content.find(TAG_TRIPCODE)
186 tripcode_tag = tag_content.find(TAG_TRIPCODE)
187 if tripcode_tag is not None:
187 if tripcode_tag is not None:
188 tripcode = tripcode_tag.text or ''
188 tripcode = tripcode_tag.text or ''
189 else:
189 else:
190 tripcode = ''
190 tripcode = ''
191
191
192 thread = tag_content.find(TAG_THREAD)
192 thread = tag_content.find(TAG_THREAD)
193 tags = []
193 tags = []
194 if thread:
194 if thread:
195 thread_id = thread.find(TAG_ID)
195 thread_id = thread.find(TAG_ID)
196 op_global_id, exists = GlobalId.from_xml_element(thread_id)
196 op_global_id, exists = GlobalId.from_xml_element(thread_id)
197 if exists:
197 if exists:
198 opening_post = Post.objects.get(global_id=op_global_id)
198 opening_post = Post.objects.get(global_id=op_global_id)
199 else:
199 else:
200 raise SyncException(EXCEPTION_OP)
200 raise SyncException(EXCEPTION_OP)
201 else:
201 else:
202 opening_post = None
202 opening_post = None
203 tag_tags = tag_content.find(TAG_TAGS)
203 tag_tags = tag_content.find(TAG_TAGS)
204 for tag_tag in tag_tags:
204 for tag_tag in tag_tags:
205 tag, created = Tag.objects.get_or_create(
205 tag, created = Tag.objects.get_or_create(
206 name=tag_tag.text)
206 name=tag_tag.text)
207 tags.append(tag)
207 tags.append(tag)
208
208
209 # TODO Check that the replied posts are already present
209 # TODO Check that the replied posts are already present
210 # before adding new ones
210 # before adding new ones
211
211
212 files = []
212 files = []
213 tag_attachments = tag_content.find(TAG_ATTACHMENTS) or list()
213 tag_attachments = tag_content.find(TAG_ATTACHMENTS) or list()
214 tag_refs = tag_model.find(TAG_ATTACHMENT_REFS)
214 tag_refs = tag_model.find(TAG_ATTACHMENT_REFS)
215 for attachment in tag_attachments:
215 for attachment in tag_attachments:
216 tag_ref = tag_refs.find("{}[@ref='{}']".format(
216 tag_ref = tag_refs.find("{}[@ref='{}']".format(
217 TAG_ATTACHMENT_REF, attachment.text))
217 TAG_ATTACHMENT_REF, attachment.text))
218 url = tag_ref.get(ATTR_URL)
218 url = tag_ref.get(ATTR_URL)
219 attached_file = download(hostname + url)
219 attached_file = download(hostname + url)
220 if attached_file is None:
220 if attached_file is None:
221 raise SyncException(EXCEPTION_DOWNLOAD)
221 raise SyncException(EXCEPTION_DOWNLOAD)
222
222
223 hash = get_file_hash(attached_file)
223 hash = get_file_hash(attached_file)
224 if hash != attachment.text:
224 if hash != attachment.text:
225 raise SyncException(EXCEPTION_HASH)
225 raise SyncException(EXCEPTION_HASH)
226
226
227 files.append(attached_file)
227 files.append(attached_file)
228
228
229 Post.objects.import_post(
229 Post.objects.import_post(
230 title=title, text=text, pub_time=pub_time,
230 title=title, text=text, pub_time=pub_time,
231 opening_post=opening_post, tags=tags,
231 opening_post=opening_post, tags=tags,
232 global_id=global_id, files=files, tripcode=tripcode)
232 global_id=global_id, files=files, tripcode=tripcode)
233 print('Parsed post {}'.format(global_id))
233 else:
234 else:
234 raise SyncException(EXCEPTION_NODE.format(tag_status.text))
235 raise SyncException(EXCEPTION_NODE.format(tag_status.text))
235
236
236 @staticmethod
237 @staticmethod
237 def generate_response_list():
238 def generate_response_list():
238 response = et.Element(TAG_RESPONSE)
239 response = et.Element(TAG_RESPONSE)
239
240
240 status = et.SubElement(response, TAG_STATUS)
241 status = et.SubElement(response, TAG_STATUS)
241 status.text = STATUS_SUCCESS
242 status.text = STATUS_SUCCESS
242
243
243 models = et.SubElement(response, TAG_MODELS)
244 models = et.SubElement(response, TAG_MODELS)
244
245
245 for post in Post.objects.prefetch_related('global_id').all():
246 for post in Post.objects.prefetch_related('global_id').all():
246 tag_id = et.SubElement(models, TAG_ID)
247 tag_model = et.SubElement(models, TAG_MODEL)
248 tag_id = et.SubElement(tag_model, TAG_ID)
247 post.global_id.to_xml_element(tag_id)
249 post.global_id.to_xml_element(tag_id)
250 tag_version = et.SubElement(tag_model, TAG_VERSION)
251 tag_version.text = post.version
248
252
249 return et.tostring(response, ENCODING_UNICODE)
253 return et.tostring(response, ENCODING_UNICODE)
250
254
251 @staticmethod
255 @staticmethod
252 def _verify_model(content_str, tag_model):
256 def _verify_model(content_str, tag_model):
253 """
257 """
254 Verifies all signatures for a single model.
258 Verifies all signatures for a single model.
255 """
259 """
256
260
257 signatures = []
261 signatures = []
258
262
259 tag_signatures = tag_model.find(TAG_SIGNATURES)
263 tag_signatures = tag_model.find(TAG_SIGNATURES)
260 for tag_signature in tag_signatures:
264 for tag_signature in tag_signatures:
261 signature_type = tag_signature.get(ATTR_TYPE)
265 signature_type = tag_signature.get(ATTR_TYPE)
262 signature_value = tag_signature.get(ATTR_VALUE)
266 signature_value = tag_signature.get(ATTR_VALUE)
263 signature_key = tag_signature.get(ATTR_KEY)
267 signature_key = tag_signature.get(ATTR_KEY)
264
268
265 signature = Signature(key_type=signature_type,
269 signature = Signature(key_type=signature_type,
266 key=signature_key,
270 key=signature_key,
267 signature=signature_value)
271 signature=signature_value)
268
272
269 if not KeyPair.objects.verify(signature, content_str):
273 if not KeyPair.objects.verify(signature, content_str):
270 raise SyncException(EXCEPTION_SIGNATURE.format(content_str))
274 raise SyncException(EXCEPTION_SIGNATURE.format(content_str))
271
275
272 signatures.append(signature)
276 signatures.append(signature)
273
277
274 return signatures
278 return signatures
275
279
276 @staticmethod
280 @staticmethod
277 def _attachment_to_xml(tag_attachments, tag_refs, file, hash, url):
281 def _attachment_to_xml(tag_attachments, tag_refs, file, hash, url):
278 if tag_attachments is not None:
282 if tag_attachments is not None:
279 mimetype = get_file_mimetype(file)
283 mimetype = get_file_mimetype(file)
280 attachment = et.SubElement(tag_attachments, TAG_ATTACHMENT)
284 attachment = et.SubElement(tag_attachments, TAG_ATTACHMENT)
281 attachment.set(ATTR_MIMETYPE, mimetype)
285 attachment.set(ATTR_MIMETYPE, mimetype)
282 attachment.set(ATTR_ID_TYPE, ID_TYPE_MD5)
286 attachment.set(ATTR_ID_TYPE, ID_TYPE_MD5)
283 attachment.text = hash
287 attachment.text = hash
284
288
285 attachment_ref = et.SubElement(tag_refs, TAG_ATTACHMENT_REF)
289 attachment_ref = et.SubElement(tag_refs, TAG_ATTACHMENT_REF)
286 attachment_ref.set(ATTR_REF, hash)
290 attachment_ref.set(ATTR_REF, hash)
287 attachment_ref.set(ATTR_URL, url)
291 attachment_ref.set(ATTR_URL, url)
@@ -1,89 +1,91 b''
1 from base64 import b64encode
1 from base64 import b64encode
2 import logging
2 import logging
3
3
4 from django.test import TestCase
4 from django.test import TestCase
5 from boards.models import KeyPair, GlobalId, Post, Signature
5 from boards.models import KeyPair, GlobalId, Post, Signature
6 from boards.models.post.sync import SyncManager
6 from boards.models.post.sync import SyncManager
7
7
8 logger = logging.getLogger(__name__)
8 logger = logging.getLogger(__name__)
9
9
10
10
11 class KeyTest(TestCase):
11 class KeyTest(TestCase):
12 def test_create_key(self):
12 def test_create_key(self):
13 key = KeyPair.objects.generate_key('ecdsa')
13 key = KeyPair.objects.generate_key('ecdsa')
14
14
15 self.assertIsNotNone(key, 'The key was not created.')
15 self.assertIsNotNone(key, 'The key was not created.')
16
16
17 def test_validation(self):
17 def test_validation(self):
18 key = KeyPair.objects.generate_key(key_type='ecdsa')
18 key = KeyPair.objects.generate_key(key_type='ecdsa')
19 message = 'msg'
19 message = 'msg'
20 signature_value = key.sign(message)
20 signature_value = key.sign(message)
21
21
22 signature = Signature(key_type='ecdsa', key=key.public_key,
22 signature = Signature(key_type='ecdsa', key=key.public_key,
23 signature=signature_value)
23 signature=signature_value)
24 valid = KeyPair.objects.verify(signature, message)
24 valid = KeyPair.objects.verify(signature, message)
25
25
26 self.assertTrue(valid, 'Message verification failed.')
26 self.assertTrue(valid, 'Message verification failed.')
27
27
28 def test_primary_constraint(self):
28 def test_primary_constraint(self):
29 KeyPair.objects.generate_key(key_type='ecdsa', primary=True)
29 KeyPair.objects.generate_key(key_type='ecdsa', primary=True)
30
30
31 with self.assertRaises(Exception):
31 with self.assertRaises(Exception):
32 KeyPair.objects.generate_key(key_type='ecdsa', primary=True)
32 KeyPair.objects.generate_key(key_type='ecdsa', primary=True)
33
33
34 def test_model_id_save(self):
34 def test_model_id_save(self):
35 model_id = GlobalId(key_type='test', key='test key', local_id='1')
35 model_id = GlobalId(key_type='test', key='test key', local_id='1')
36 model_id.save()
36 model_id.save()
37
37
38 def test_request_get(self):
38 def test_request_get(self):
39 post = self._create_post_with_key()
39 post = self._create_post_with_key()
40
40
41 request = GlobalId.objects.generate_request_get([post.global_id])
41 request = GlobalId.objects.generate_request_get([post.global_id])
42 logger.debug(request)
42 logger.debug(request)
43
43
44 key = KeyPair.objects.get(primary=True)
44 key = KeyPair.objects.get(primary=True)
45 self.assertTrue('<request type="get" version="1.0">'
45 self.assertTrue('<request type="get" version="1.0">'
46 '<model name="post" version="1.0">'
46 '<model name="post" version="1.0">'
47 '<id key="%s" local-id="1" type="%s" />'
47 '<id key="%s" local-id="1" type="%s" />'
48 '</model>'
48 '</model>'
49 '</request>' % (
49 '</request>' % (
50 key.public_key,
50 key.public_key,
51 key.key_type,
51 key.key_type,
52 ) in request,
52 ) in request,
53 'Wrong XML generated for the GET request.')
53 'Wrong XML generated for the GET request.')
54
54
55 def test_response_get(self):
55 def test_response_get(self):
56 post = self._create_post_with_key()
56 post = self._create_post_with_key()
57 reply_post = Post.objects.create_post(title='test_title',
57 reply_post = Post.objects.create_post(title='test_title',
58 text='[post]%d[/post]' % post.id,
58 text='[post]%d[/post]' % post.id,
59 thread=post.get_thread())
59 thread=post.get_thread())
60
60
61 response = SyncManager.generate_response_get([reply_post])
61 response = SyncManager.generate_response_get([reply_post])
62 logger.debug(response)
62 logger.debug(response)
63
63
64 key = KeyPair.objects.get(primary=True)
64 key = KeyPair.objects.get(primary=True)
65 self.assertTrue('<status>success</status>'
65 self.assertTrue('<status>success</status>'
66 '<models>'
66 '<models>'
67 '<model name="post">'
67 '<model name="post">'
68 '<content>'
68 '<content>'
69 '<id key="%s" local-id="%d" type="%s" />'
69 '<id key="%s" local-id="%d" type="%s" />'
70 '<title>test_title</title>'
70 '<title>test_title</title>'
71 '<text>[post]%s[/post]</text>'
71 '<text>[post]%s[/post]</text>'
72 '<thread><id key="%s" local-id="%d" type="%s" /></thread>'
72 '<thread><id key="%s" local-id="%d" type="%s" /></thread>'
73 '<pub-time>%s</pub-time>'
73 '<pub-time>%s</pub-time>'
74 '<version>%s</version>'
74 '</content>' % (
75 '</content>' % (
75 key.public_key,
76 key.public_key,
76 reply_post.id,
77 reply_post.id,
77 key.key_type,
78 key.key_type,
78 str(post.global_id),
79 str(post.global_id),
79 key.public_key,
80 key.public_key,
80 post.id,
81 post.id,
81 key.key_type,
82 key.key_type,
82 str(reply_post.get_pub_time_str()),
83 str(reply_post.get_pub_time_str()),
84 post.version,
83 ) in response,
85 ) in response,
84 'Wrong XML generated for the GET response.')
86 'Wrong XML generated for the GET response.')
85
87
86 def _create_post_with_key(self):
88 def _create_post_with_key(self):
87 KeyPair.objects.generate_key(primary=True)
89 KeyPair.objects.generate_key(primary=True)
88
90
89 return Post.objects.create_post(title='test_title', text='test_text')
91 return Post.objects.create_post(title='test_title', text='test_text')
@@ -1,102 +1,106 b''
1 from boards.models import KeyPair, Post, Tag
1 from boards.models import KeyPair, Post, Tag
2 from boards.models.post.sync import SyncManager
2 from boards.models.post.sync import SyncManager
3 from boards.tests.mocks import MockRequest
3 from boards.tests.mocks import MockRequest
4 from boards.views.sync import response_get
4 from boards.views.sync import response_get
5
5
6 __author__ = 'neko259'
6 __author__ = 'neko259'
7
7
8
8
9 from django.test import TestCase
9 from django.test import TestCase
10
10
11
11
12 class SyncTest(TestCase):
12 class SyncTest(TestCase):
13 def test_get(self):
13 def test_get(self):
14 """
14 """
15 Forms a GET request of a post and checks the response.
15 Forms a GET request of a post and checks the response.
16 """
16 """
17
17
18 key = KeyPair.objects.generate_key(primary=True)
18 key = KeyPair.objects.generate_key(primary=True)
19 tag = Tag.objects.create(name='tag1')
19 tag = Tag.objects.create(name='tag1')
20 post = Post.objects.create_post(title='test_title',
20 post = Post.objects.create_post(title='test_title',
21 text='test_text\rline two',
21 text='test_text\rline two',
22 tags=[tag])
22 tags=[tag])
23
23
24 request = MockRequest()
24 request = MockRequest()
25 request.body = (
25 request.body = (
26 '<request type="get" version="1.0">'
26 '<request type="get" version="1.0">'
27 '<model name="post" version="1.0">'
27 '<model name="post" version="1.0">'
28 '<id key="%s" local-id="%d" type="%s" />'
28 '<id key="%s" local-id="%d" type="%s" />'
29 '</model>'
29 '</model>'
30 '</request>' % (post.global_id.key,
30 '</request>' % (post.global_id.key,
31 post.id,
31 post.id,
32 post.global_id.key_type)
32 post.global_id.key_type)
33 )
33 )
34
34
35 response = response_get(request).content.decode()
35 response = response_get(request).content.decode()
36 self.assertTrue(
36 self.assertTrue(
37 '<status>success</status>'
37 '<status>success</status>'
38 '<models>'
38 '<models>'
39 '<model name="post">'
39 '<model name="post">'
40 '<content>'
40 '<content>'
41 '<id key="%s" local-id="%d" type="%s" />'
41 '<id key="%s" local-id="%d" type="%s" />'
42 '<title>%s</title>'
42 '<title>%s</title>'
43 '<text>%s</text>'
43 '<text>%s</text>'
44 '<tags><tag>%s</tag></tags>'
44 '<tags><tag>%s</tag></tags>'
45 '<pub-time>%s</pub-time>'
45 '<pub-time>%s</pub-time>'
46 '<version>%s</version>'
46 '</content>' % (
47 '</content>' % (
47 post.global_id.key,
48 post.global_id.key,
48 post.global_id.local_id,
49 post.global_id.local_id,
49 post.global_id.key_type,
50 post.global_id.key_type,
50 post.title,
51 post.title,
51 post.get_sync_text(),
52 post.get_sync_text(),
52 post.get_thread().get_tags().first().name,
53 post.get_thread().get_tags().first().name,
53 post.get_pub_time_str(),
54 post.get_pub_time_str(),
55 post.version,
54 ) in response,
56 ) in response,
55 'Wrong response generated for the GET request.')
57 'Wrong response generated for the GET request.')
56
58
57 post.delete()
59 post.delete()
58 key.delete()
60 key.delete()
59
61
60 KeyPair.objects.generate_key(primary=True)
62 KeyPair.objects.generate_key(primary=True)
61
63
62 SyncManager.parse_response_get(response, None)
64 SyncManager.parse_response_get(response, None)
63 self.assertEqual(1, Post.objects.count(),
65 self.assertEqual(1, Post.objects.count(),
64 'Post was not created from XML response.')
66 'Post was not created from XML response.')
65
67
66 parsed_post = Post.objects.first()
68 parsed_post = Post.objects.first()
67 self.assertEqual('tag1',
69 self.assertEqual('tag1',
68 parsed_post.get_thread().get_tags().first().name,
70 parsed_post.get_thread().get_tags().first().name,
69 'Invalid tag was parsed.')
71 'Invalid tag was parsed.')
70
72
71 SyncManager.parse_response_get(response, None)
73 SyncManager.parse_response_get(response, None)
72 self.assertEqual(1, Post.objects.count(),
74 self.assertEqual(1, Post.objects.count(),
73 'The same post was imported twice.')
75 'The same post was imported twice.')
74
76
75 self.assertEqual(1, parsed_post.global_id.signature_set.count(),
77 self.assertEqual(1, parsed_post.global_id.signature_set.count(),
76 'Signature was not saved.')
78 'Signature was not saved.')
77
79
78 post = parsed_post
80 post = parsed_post
79
81
80 # Trying to sync the same once more
82 # Trying to sync the same once more
81 response = response_get(request).content.decode()
83 response = response_get(request).content.decode()
82
84
83 self.assertTrue(
85 self.assertTrue(
84 '<status>success</status>'
86 '<status>success</status>'
85 '<models>'
87 '<models>'
86 '<model name="post">'
88 '<model name="post">'
87 '<content>'
89 '<content>'
88 '<id key="%s" local-id="%d" type="%s" />'
90 '<id key="%s" local-id="%d" type="%s" />'
89 '<title>%s</title>'
91 '<title>%s</title>'
90 '<text>%s</text>'
92 '<text>%s</text>'
91 '<tags><tag>%s</tag></tags>'
93 '<tags><tag>%s</tag></tags>'
92 '<pub-time>%s</pub-time>'
94 '<pub-time>%s</pub-time>'
95 '<version>%s</version>'
93 '</content>' % (
96 '</content>' % (
94 post.global_id.key,
97 post.global_id.key,
95 post.global_id.local_id,
98 post.global_id.local_id,
96 post.global_id.key_type,
99 post.global_id.key_type,
97 post.title,
100 post.title,
98 post.get_sync_text(),
101 post.get_sync_text(),
99 post.get_thread().get_tags().first().name,
102 post.get_thread().get_tags().first().name,
100 post.get_pub_time_str(),
103 post.get_pub_time_str(),
104 post.version,
101 ) in response,
105 ) in response,
102 'Wrong response generated for the GET request.')
106 'Wrong response generated for the GET request.')
General Comments 0
You need to be logged in to leave comments. Login now