Show More
@@ -1,96 +1,96 b'' | |||||
1 | import re |
|
1 | import re | |
2 | import logging |
|
2 | import logging | |
3 | import xml.etree.ElementTree as ET |
|
3 | import xml.etree.ElementTree as ET | |
4 |
|
4 | |||
5 | import httplib2 |
|
5 | import httplib2 | |
6 | from django.core.management import BaseCommand |
|
6 | from django.core.management import BaseCommand | |
7 |
|
7 | |||
8 | from boards.models import GlobalId |
|
8 | from boards.models import GlobalId | |
9 | from boards.models.post.sync import SyncManager, TAG_ID, TAG_VERSION |
|
9 | from boards.models.post.sync import SyncManager, TAG_ID, TAG_VERSION | |
10 |
|
10 | |||
11 | __author__ = 'neko259' |
|
11 | __author__ = 'neko259' | |
12 |
|
12 | |||
13 |
|
13 | |||
14 | REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)') |
|
14 | REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)') | |
15 |
|
15 | |||
16 |
|
16 | |||
17 | class Command(BaseCommand): |
|
17 | class Command(BaseCommand): | |
18 | help = 'Send a sync or get request to the server.' |
|
18 | help = 'Send a sync or get request to the server.' | |
19 |
|
19 | |||
20 | def add_arguments(self, parser): |
|
20 | def add_arguments(self, parser): | |
21 | parser.add_argument('url', type=str, help='Server root url') |
|
21 | parser.add_argument('url', type=str, help='Server root url') | |
22 | parser.add_argument('--global-id', type=str, default='', |
|
22 | parser.add_argument('--global-id', type=str, default='', | |
23 | help='Post global ID') |
|
23 | help='Post global ID') | |
24 | parser.add_argument('--split-query', type=int, |
|
24 | parser.add_argument('--split-query', type=int, default=10, | |
25 | help='Split GET query into separate by the given' |
|
25 | help='Split GET query into separate by the given' | |
26 | ' number of posts in one') |
|
26 | ' number of posts in one') | |
27 |
|
27 | |||
28 | def handle(self, *args, **options): |
|
28 | def handle(self, *args, **options): | |
29 | logger = logging.getLogger('boards.sync') |
|
29 | logger = logging.getLogger('boards.sync') | |
30 |
|
30 | |||
31 | url = options.get('url') |
|
31 | url = options.get('url') | |
32 |
|
32 | |||
33 | list_url = url + 'api/sync/list/' |
|
33 | list_url = url + 'api/sync/list/' | |
34 | get_url = url + 'api/sync/get/' |
|
34 | get_url = url + 'api/sync/get/' | |
35 | file_url = url[:-1] |
|
35 | file_url = url[:-1] | |
36 |
|
36 | |||
37 | global_id_str = options.get('global_id') |
|
37 | global_id_str = options.get('global_id') | |
38 | if global_id_str: |
|
38 | if global_id_str: | |
39 | match = REGEX_GLOBAL_ID.match(global_id_str) |
|
39 | match = REGEX_GLOBAL_ID.match(global_id_str) | |
40 | if match: |
|
40 | if match: | |
41 | key_type = match.group(1) |
|
41 | key_type = match.group(1) | |
42 | key = match.group(2) |
|
42 | key = match.group(2) | |
43 | local_id = match.group(3) |
|
43 | local_id = match.group(3) | |
44 |
|
44 | |||
45 | global_id = GlobalId(key_type=key_type, key=key, |
|
45 | global_id = GlobalId(key_type=key_type, key=key, | |
46 | local_id=local_id) |
|
46 | local_id=local_id) | |
47 |
|
47 | |||
48 | xml = GlobalId.objects.generate_request_get([global_id]) |
|
48 | xml = GlobalId.objects.generate_request_get([global_id]) | |
49 | h = httplib2.Http() |
|
49 | h = httplib2.Http() | |
50 | response, content = h.request(get_url, method="POST", body=xml) |
|
50 | response, content = h.request(get_url, method="POST", body=xml) | |
51 |
|
51 | |||
52 | SyncManager.parse_response_get(content, file_url) |
|
52 | SyncManager.parse_response_get(content, file_url) | |
53 | else: |
|
53 | else: | |
54 | raise Exception('Invalid global ID') |
|
54 | raise Exception('Invalid global ID') | |
55 | else: |
|
55 | else: | |
56 | logger.info('Running LIST request...') |
|
56 | logger.info('Running LIST request...') | |
57 | h = httplib2.Http() |
|
57 | h = httplib2.Http() | |
58 | xml = GlobalId.objects.generate_request_list() |
|
58 | xml = GlobalId.objects.generate_request_list() | |
59 | response, content = h.request(list_url, method="POST", body=xml) |
|
59 | response, content = h.request(list_url, method="POST", body=xml) | |
60 | logger.info('Processing response...') |
|
60 | logger.info('Processing response...') | |
61 |
|
61 | |||
62 | root = ET.fromstring(content) |
|
62 | root = ET.fromstring(content) | |
63 | status = root.findall('status')[0].text |
|
63 | status = root.findall('status')[0].text | |
64 | if status == 'success': |
|
64 | if status == 'success': | |
65 | ids_to_sync = list() |
|
65 | ids_to_sync = list() | |
66 |
|
66 | |||
67 | models = root.findall('models')[0] |
|
67 | models = root.findall('models')[0] | |
68 | for model in models: |
|
68 | for model in models: | |
69 | tag_id = model.find(TAG_ID) |
|
69 | tag_id = model.find(TAG_ID) | |
70 | global_id, exists = GlobalId.from_xml_element(tag_id) |
|
70 | global_id, exists = GlobalId.from_xml_element(tag_id) | |
71 | tag_version = model.find(TAG_VERSION) |
|
71 | tag_version = model.find(TAG_VERSION) | |
72 | if tag_version is not None: |
|
72 | if tag_version is not None: | |
73 | version = int(tag_version.text) or 1 |
|
73 | version = int(tag_version.text) or 1 | |
74 | else: |
|
74 | else: | |
75 | version = 1 |
|
75 | version = 1 | |
76 | if not exists or global_id.post.version < version: |
|
76 | if not exists or global_id.post.version < version: | |
77 |
logger.debug(' |
|
77 | logger.debug('Processed (+) post {}'.format(global_id)) | |
78 | ids_to_sync.append(global_id) |
|
78 | ids_to_sync.append(global_id) | |
79 | else: |
|
79 | else: | |
80 | logger.debug('* Processed (-) post {}'.format(global_id)) |
|
80 | logger.debug('* Processed (-) post {}'.format(global_id)) | |
81 | logger.info('Starting sync...') |
|
81 | logger.info('Starting sync...') | |
82 |
|
82 | |||
83 | if len(ids_to_sync) > 0: |
|
83 | if len(ids_to_sync) > 0: | |
84 | limit = options.get('split_query', len(ids_to_sync)) |
|
84 | limit = options.get('split_query', len(ids_to_sync)) | |
85 | for offset in range(0, len(ids_to_sync), limit): |
|
85 | for offset in range(0, len(ids_to_sync), limit): | |
86 | xml = GlobalId.objects.generate_request_get(ids_to_sync[offset:offset+limit]) |
|
86 | xml = GlobalId.objects.generate_request_get(ids_to_sync[offset:offset+limit]) | |
87 | h = httplib2.Http() |
|
87 | h = httplib2.Http() | |
88 | logger.info('Running GET request...') |
|
88 | logger.info('Running GET request...') | |
89 | response, content = h.request(get_url, method="POST", body=xml) |
|
89 | response, content = h.request(get_url, method="POST", body=xml) | |
90 | logger.info('Processing response...') |
|
90 | logger.info('Processing response...') | |
91 |
|
91 | |||
92 | SyncManager.parse_response_get(content, file_url) |
|
92 | SyncManager.parse_response_get(content, file_url) | |
93 | else: |
|
93 | else: | |
94 | logger.info('Nothing to get, everything synced') |
|
94 | logger.info('Nothing to get, everything synced') | |
95 | else: |
|
95 | else: | |
96 | raise Exception('Invalid response status') |
|
96 | raise Exception('Invalid response status') |
@@ -1,298 +1,302 b'' | |||||
1 | import xml.etree.ElementTree as et |
|
1 | import xml.etree.ElementTree as et | |
|
2 | import logging | |||
2 |
|
3 | |||
3 | from boards.abstracts.exceptions import SyncException |
|
4 | from boards.abstracts.exceptions import SyncException | |
4 | from boards.models import KeyPair, GlobalId, Signature, Post, Tag |
|
5 | from boards.models import KeyPair, GlobalId, Signature, Post, Tag | |
5 | from boards.models.attachment.downloaders import download |
|
6 | from boards.models.attachment.downloaders import download | |
6 | from boards.utils import get_file_mimetype, get_file_hash |
|
7 | from boards.utils import get_file_mimetype, get_file_hash | |
7 | from django.db import transaction |
|
8 | from django.db import transaction | |
8 |
|
9 | |||
9 | EXCEPTION_NODE = 'Sync node returned an error: {}.' |
|
10 | EXCEPTION_NODE = 'Sync node returned an error: {}.' | |
10 | EXCEPTION_OP = 'Load the OP first.' |
|
11 | EXCEPTION_OP = 'Load the OP first.' | |
11 | EXCEPTION_DOWNLOAD = 'File was not downloaded.' |
|
12 | EXCEPTION_DOWNLOAD = 'File was not downloaded.' | |
12 | EXCEPTION_HASH = 'File hash does not match attachment hash.' |
|
13 | EXCEPTION_HASH = 'File hash does not match attachment hash.' | |
13 | EXCEPTION_SIGNATURE = 'Invalid model signature for {}.' |
|
14 | EXCEPTION_SIGNATURE = 'Invalid model signature for {}.' | |
14 | EXCEPTION_AUTHOR_SIGNATURE = 'Model {} has no author signature.' |
|
15 | EXCEPTION_AUTHOR_SIGNATURE = 'Model {} has no author signature.' | |
15 | ENCODING_UNICODE = 'unicode' |
|
16 | ENCODING_UNICODE = 'unicode' | |
16 |
|
17 | |||
17 | TAG_MODEL = 'model' |
|
18 | TAG_MODEL = 'model' | |
18 | TAG_REQUEST = 'request' |
|
19 | TAG_REQUEST = 'request' | |
19 | TAG_RESPONSE = 'response' |
|
20 | TAG_RESPONSE = 'response' | |
20 | TAG_ID = 'id' |
|
21 | TAG_ID = 'id' | |
21 | TAG_STATUS = 'status' |
|
22 | TAG_STATUS = 'status' | |
22 | TAG_MODELS = 'models' |
|
23 | TAG_MODELS = 'models' | |
23 | TAG_TITLE = 'title' |
|
24 | TAG_TITLE = 'title' | |
24 | TAG_TEXT = 'text' |
|
25 | TAG_TEXT = 'text' | |
25 | TAG_THREAD = 'thread' |
|
26 | TAG_THREAD = 'thread' | |
26 | TAG_PUB_TIME = 'pub-time' |
|
27 | TAG_PUB_TIME = 'pub-time' | |
27 | TAG_SIGNATURES = 'signatures' |
|
28 | TAG_SIGNATURES = 'signatures' | |
28 | TAG_SIGNATURE = 'signature' |
|
29 | TAG_SIGNATURE = 'signature' | |
29 | TAG_CONTENT = 'content' |
|
30 | TAG_CONTENT = 'content' | |
30 | TAG_ATTACHMENTS = 'attachments' |
|
31 | TAG_ATTACHMENTS = 'attachments' | |
31 | TAG_ATTACHMENT = 'attachment' |
|
32 | TAG_ATTACHMENT = 'attachment' | |
32 | TAG_TAGS = 'tags' |
|
33 | TAG_TAGS = 'tags' | |
33 | TAG_TAG = 'tag' |
|
34 | TAG_TAG = 'tag' | |
34 | TAG_ATTACHMENT_REFS = 'attachment-refs' |
|
35 | TAG_ATTACHMENT_REFS = 'attachment-refs' | |
35 | TAG_ATTACHMENT_REF = 'attachment-ref' |
|
36 | TAG_ATTACHMENT_REF = 'attachment-ref' | |
36 | TAG_TRIPCODE = 'tripcode' |
|
37 | TAG_TRIPCODE = 'tripcode' | |
37 | TAG_VERSION = 'version' |
|
38 | TAG_VERSION = 'version' | |
38 |
|
39 | |||
39 | TYPE_GET = 'get' |
|
40 | TYPE_GET = 'get' | |
40 |
|
41 | |||
41 | ATTR_VERSION = 'version' |
|
42 | ATTR_VERSION = 'version' | |
42 | ATTR_TYPE = 'type' |
|
43 | ATTR_TYPE = 'type' | |
43 | ATTR_NAME = 'name' |
|
44 | ATTR_NAME = 'name' | |
44 | ATTR_VALUE = 'value' |
|
45 | ATTR_VALUE = 'value' | |
45 | ATTR_MIMETYPE = 'mimetype' |
|
46 | ATTR_MIMETYPE = 'mimetype' | |
46 | ATTR_KEY = 'key' |
|
47 | ATTR_KEY = 'key' | |
47 | ATTR_REF = 'ref' |
|
48 | ATTR_REF = 'ref' | |
48 | ATTR_URL = 'url' |
|
49 | ATTR_URL = 'url' | |
49 | ATTR_ID_TYPE = 'id-type' |
|
50 | ATTR_ID_TYPE = 'id-type' | |
50 |
|
51 | |||
51 | ID_TYPE_MD5 = 'md5' |
|
52 | ID_TYPE_MD5 = 'md5' | |
52 |
|
53 | |||
53 | STATUS_SUCCESS = 'success' |
|
54 | STATUS_SUCCESS = 'success' | |
54 |
|
55 | |||
55 |
|
56 | |||
|
57 | logger = logging.getLogger('boards.sync') | |||
|
58 | ||||
|
59 | ||||
56 | class SyncManager: |
|
60 | class SyncManager: | |
57 | @staticmethod |
|
61 | @staticmethod | |
58 | def generate_response_get(model_list: list): |
|
62 | def generate_response_get(model_list: list): | |
59 | response = et.Element(TAG_RESPONSE) |
|
63 | response = et.Element(TAG_RESPONSE) | |
60 |
|
64 | |||
61 | status = et.SubElement(response, TAG_STATUS) |
|
65 | status = et.SubElement(response, TAG_STATUS) | |
62 | status.text = STATUS_SUCCESS |
|
66 | status.text = STATUS_SUCCESS | |
63 |
|
67 | |||
64 | models = et.SubElement(response, TAG_MODELS) |
|
68 | models = et.SubElement(response, TAG_MODELS) | |
65 |
|
69 | |||
66 | for post in model_list: |
|
70 | for post in model_list: | |
67 | model = et.SubElement(models, TAG_MODEL) |
|
71 | model = et.SubElement(models, TAG_MODEL) | |
68 | model.set(ATTR_NAME, 'post') |
|
72 | model.set(ATTR_NAME, 'post') | |
69 |
|
73 | |||
70 | global_id = post.global_id |
|
74 | global_id = post.global_id | |
71 |
|
75 | |||
72 | attachments = post.attachments.all() |
|
76 | attachments = post.attachments.all() | |
73 | if global_id.content: |
|
77 | if global_id.content: | |
74 | model.append(et.fromstring(global_id.content)) |
|
78 | model.append(et.fromstring(global_id.content)) | |
75 | if len(attachments) > 0: |
|
79 | if len(attachments) > 0: | |
76 | attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS) |
|
80 | attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS) | |
77 | for file in attachments: |
|
81 | for file in attachments: | |
78 | SyncManager._attachment_to_xml( |
|
82 | SyncManager._attachment_to_xml( | |
79 | None, attachment_refs, file.file.file, |
|
83 | None, attachment_refs, file.file.file, | |
80 | file.hash, file.file.url) |
|
84 | file.hash, file.file.url) | |
81 | else: |
|
85 | else: | |
82 | content_tag = et.SubElement(model, TAG_CONTENT) |
|
86 | content_tag = et.SubElement(model, TAG_CONTENT) | |
83 |
|
87 | |||
84 | tag_id = et.SubElement(content_tag, TAG_ID) |
|
88 | tag_id = et.SubElement(content_tag, TAG_ID) | |
85 | global_id.to_xml_element(tag_id) |
|
89 | global_id.to_xml_element(tag_id) | |
86 |
|
90 | |||
87 | title = et.SubElement(content_tag, TAG_TITLE) |
|
91 | title = et.SubElement(content_tag, TAG_TITLE) | |
88 | title.text = post.title |
|
92 | title.text = post.title | |
89 |
|
93 | |||
90 | text = et.SubElement(content_tag, TAG_TEXT) |
|
94 | text = et.SubElement(content_tag, TAG_TEXT) | |
91 | text.text = post.get_sync_text() |
|
95 | text.text = post.get_sync_text() | |
92 |
|
96 | |||
93 | thread = post.get_thread() |
|
97 | thread = post.get_thread() | |
94 | if post.is_opening(): |
|
98 | if post.is_opening(): | |
95 | tag_tags = et.SubElement(content_tag, TAG_TAGS) |
|
99 | tag_tags = et.SubElement(content_tag, TAG_TAGS) | |
96 | for tag in thread.get_tags(): |
|
100 | for tag in thread.get_tags(): | |
97 | tag_tag = et.SubElement(tag_tags, TAG_TAG) |
|
101 | tag_tag = et.SubElement(tag_tags, TAG_TAG) | |
98 | tag_tag.text = tag.name |
|
102 | tag_tag.text = tag.name | |
99 | else: |
|
103 | else: | |
100 | tag_thread = et.SubElement(content_tag, TAG_THREAD) |
|
104 | tag_thread = et.SubElement(content_tag, TAG_THREAD) | |
101 | thread_id = et.SubElement(tag_thread, TAG_ID) |
|
105 | thread_id = et.SubElement(tag_thread, TAG_ID) | |
102 | thread.get_opening_post().global_id.to_xml_element(thread_id) |
|
106 | thread.get_opening_post().global_id.to_xml_element(thread_id) | |
103 |
|
107 | |||
104 | pub_time = et.SubElement(content_tag, TAG_PUB_TIME) |
|
108 | pub_time = et.SubElement(content_tag, TAG_PUB_TIME) | |
105 | pub_time.text = str(post.get_pub_time_str()) |
|
109 | pub_time.text = str(post.get_pub_time_str()) | |
106 |
|
110 | |||
107 | if post.tripcode: |
|
111 | if post.tripcode: | |
108 | tripcode = et.SubElement(content_tag, TAG_TRIPCODE) |
|
112 | tripcode = et.SubElement(content_tag, TAG_TRIPCODE) | |
109 | tripcode.text = post.tripcode |
|
113 | tripcode.text = post.tripcode | |
110 |
|
114 | |||
111 | if len(attachments) > 0: |
|
115 | if len(attachments) > 0: | |
112 | attachments_tag = et.SubElement(content_tag, TAG_ATTACHMENTS) |
|
116 | attachments_tag = et.SubElement(content_tag, TAG_ATTACHMENTS) | |
113 | attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS) |
|
117 | attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS) | |
114 |
|
118 | |||
115 | for file in attachments: |
|
119 | for file in attachments: | |
116 | SyncManager._attachment_to_xml( |
|
120 | SyncManager._attachment_to_xml( | |
117 | attachments_tag, attachment_refs, file.file.file, |
|
121 | attachments_tag, attachment_refs, file.file.file, | |
118 | file.hash, file.file.url) |
|
122 | file.hash, file.file.url) | |
119 | version_tag = et.SubElement(content_tag, TAG_VERSION) |
|
123 | version_tag = et.SubElement(content_tag, TAG_VERSION) | |
120 | version_tag.text = str(post.version) |
|
124 | version_tag.text = str(post.version) | |
121 |
|
125 | |||
122 | global_id.content = et.tostring(content_tag, ENCODING_UNICODE) |
|
126 | global_id.content = et.tostring(content_tag, ENCODING_UNICODE) | |
123 | global_id.save() |
|
127 | global_id.save() | |
124 |
|
128 | |||
125 | signatures_tag = et.SubElement(model, TAG_SIGNATURES) |
|
129 | signatures_tag = et.SubElement(model, TAG_SIGNATURES) | |
126 | post_signatures = global_id.signature_set.all() |
|
130 | post_signatures = global_id.signature_set.all() | |
127 | if post_signatures: |
|
131 | if post_signatures: | |
128 | signatures = post_signatures |
|
132 | signatures = post_signatures | |
129 | else: |
|
133 | else: | |
130 | key = KeyPair.objects.get(public_key=global_id.key) |
|
134 | key = KeyPair.objects.get(public_key=global_id.key) | |
131 | signature = Signature( |
|
135 | signature = Signature( | |
132 | key_type=key.key_type, |
|
136 | key_type=key.key_type, | |
133 | key=key.public_key, |
|
137 | key=key.public_key, | |
134 | signature=key.sign(global_id.content), |
|
138 | signature=key.sign(global_id.content), | |
135 | global_id=global_id, |
|
139 | global_id=global_id, | |
136 | ) |
|
140 | ) | |
137 | signature.save() |
|
141 | signature.save() | |
138 | signatures = [signature] |
|
142 | signatures = [signature] | |
139 | for signature in signatures: |
|
143 | for signature in signatures: | |
140 | signature_tag = et.SubElement(signatures_tag, TAG_SIGNATURE) |
|
144 | signature_tag = et.SubElement(signatures_tag, TAG_SIGNATURE) | |
141 | signature_tag.set(ATTR_TYPE, signature.key_type) |
|
145 | signature_tag.set(ATTR_TYPE, signature.key_type) | |
142 | signature_tag.set(ATTR_VALUE, signature.signature) |
|
146 | signature_tag.set(ATTR_VALUE, signature.signature) | |
143 | signature_tag.set(ATTR_KEY, signature.key) |
|
147 | signature_tag.set(ATTR_KEY, signature.key) | |
144 |
|
148 | |||
145 | return et.tostring(response, ENCODING_UNICODE) |
|
149 | return et.tostring(response, ENCODING_UNICODE) | |
146 |
|
150 | |||
147 | @staticmethod |
|
151 | @staticmethod | |
148 | @transaction.atomic |
|
152 | @transaction.atomic | |
149 | def parse_response_get(response_xml, hostname): |
|
153 | def parse_response_get(response_xml, hostname): | |
150 | tag_root = et.fromstring(response_xml) |
|
154 | tag_root = et.fromstring(response_xml) | |
151 | tag_status = tag_root.find(TAG_STATUS) |
|
155 | tag_status = tag_root.find(TAG_STATUS) | |
152 | if STATUS_SUCCESS == tag_status.text: |
|
156 | if STATUS_SUCCESS == tag_status.text: | |
153 | tag_models = tag_root.find(TAG_MODELS) |
|
157 | tag_models = tag_root.find(TAG_MODELS) | |
154 | for tag_model in tag_models: |
|
158 | for tag_model in tag_models: | |
155 | tag_content = tag_model.find(TAG_CONTENT) |
|
159 | tag_content = tag_model.find(TAG_CONTENT) | |
156 |
|
160 | |||
157 | content_str = et.tostring(tag_content, ENCODING_UNICODE) |
|
161 | content_str = et.tostring(tag_content, ENCODING_UNICODE) | |
158 |
|
162 | |||
159 | tag_id = tag_content.find(TAG_ID) |
|
163 | tag_id = tag_content.find(TAG_ID) | |
160 | global_id, exists = GlobalId.from_xml_element(tag_id) |
|
164 | global_id, exists = GlobalId.from_xml_element(tag_id) | |
161 | signatures = SyncManager._verify_model(global_id, content_str, tag_model) |
|
165 | signatures = SyncManager._verify_model(global_id, content_str, tag_model) | |
162 |
|
166 | |||
163 | version = int(tag_content.find(TAG_VERSION).text) |
|
167 | version = int(tag_content.find(TAG_VERSION).text) | |
164 | is_old = exists and global_id.post.version < version |
|
168 | is_old = exists and global_id.post.version < version | |
165 | if exists and not is_old: |
|
169 | if exists and not is_old: | |
166 | print('Post with same ID exists and is up to date.') |
|
170 | print('Post with same ID exists and is up to date.') | |
167 | else: |
|
171 | else: | |
168 | global_id.content = content_str |
|
172 | global_id.content = content_str | |
169 | global_id.save() |
|
173 | global_id.save() | |
170 | for signature in signatures: |
|
174 | for signature in signatures: | |
171 | signature.global_id = global_id |
|
175 | signature.global_id = global_id | |
172 | signature.save() |
|
176 | signature.save() | |
173 |
|
177 | |||
174 | title = tag_content.find(TAG_TITLE).text or '' |
|
178 | title = tag_content.find(TAG_TITLE).text or '' | |
175 | text = tag_content.find(TAG_TEXT).text or '' |
|
179 | text = tag_content.find(TAG_TEXT).text or '' | |
176 | pub_time = tag_content.find(TAG_PUB_TIME).text |
|
180 | pub_time = tag_content.find(TAG_PUB_TIME).text | |
177 | tripcode_tag = tag_content.find(TAG_TRIPCODE) |
|
181 | tripcode_tag = tag_content.find(TAG_TRIPCODE) | |
178 | if tripcode_tag is not None: |
|
182 | if tripcode_tag is not None: | |
179 | tripcode = tripcode_tag.text or '' |
|
183 | tripcode = tripcode_tag.text or '' | |
180 | else: |
|
184 | else: | |
181 | tripcode = '' |
|
185 | tripcode = '' | |
182 |
|
186 | |||
183 | thread = tag_content.find(TAG_THREAD) |
|
187 | thread = tag_content.find(TAG_THREAD) | |
184 | tags = [] |
|
188 | tags = [] | |
185 | if thread: |
|
189 | if thread: | |
186 | thread_id = thread.find(TAG_ID) |
|
190 | thread_id = thread.find(TAG_ID) | |
187 | op_global_id, exists = GlobalId.from_xml_element(thread_id) |
|
191 | op_global_id, exists = GlobalId.from_xml_element(thread_id) | |
188 | if exists: |
|
192 | if exists: | |
189 | opening_post = Post.objects.get(global_id=op_global_id) |
|
193 | opening_post = Post.objects.get(global_id=op_global_id) | |
190 | else: |
|
194 | else: | |
191 | raise SyncException(EXCEPTION_OP) |
|
195 | raise SyncException(EXCEPTION_OP) | |
192 | else: |
|
196 | else: | |
193 | opening_post = None |
|
197 | opening_post = None | |
194 | tag_tags = tag_content.find(TAG_TAGS) |
|
198 | tag_tags = tag_content.find(TAG_TAGS) | |
195 | for tag_tag in tag_tags: |
|
199 | for tag_tag in tag_tags: | |
196 | tag, created = Tag.objects.get_or_create( |
|
200 | tag, created = Tag.objects.get_or_create( | |
197 | name=tag_tag.text) |
|
201 | name=tag_tag.text) | |
198 | tags.append(tag) |
|
202 | tags.append(tag) | |
199 |
|
203 | |||
200 | # TODO Check that the replied posts are already present |
|
204 | # TODO Check that the replied posts are already present | |
201 | # before adding new ones |
|
205 | # before adding new ones | |
202 |
|
206 | |||
203 | files = [] |
|
207 | files = [] | |
204 | tag_attachments = tag_content.find(TAG_ATTACHMENTS) or list() |
|
208 | tag_attachments = tag_content.find(TAG_ATTACHMENTS) or list() | |
205 | tag_refs = tag_model.find(TAG_ATTACHMENT_REFS) |
|
209 | tag_refs = tag_model.find(TAG_ATTACHMENT_REFS) | |
206 | for attachment in tag_attachments: |
|
210 | for attachment in tag_attachments: | |
207 | tag_ref = tag_refs.find("{}[@ref='{}']".format( |
|
211 | tag_ref = tag_refs.find("{}[@ref='{}']".format( | |
208 | TAG_ATTACHMENT_REF, attachment.text)) |
|
212 | TAG_ATTACHMENT_REF, attachment.text)) | |
209 | url = tag_ref.get(ATTR_URL) |
|
213 | url = tag_ref.get(ATTR_URL) | |
210 | attached_file = download(hostname + url) |
|
214 | attached_file = download(hostname + url) | |
211 | if attached_file is None: |
|
215 | if attached_file is None: | |
212 | raise SyncException(EXCEPTION_DOWNLOAD) |
|
216 | raise SyncException(EXCEPTION_DOWNLOAD) | |
213 |
|
217 | |||
214 | hash = get_file_hash(attached_file) |
|
218 | hash = get_file_hash(attached_file) | |
215 | if hash != attachment.text: |
|
219 | if hash != attachment.text: | |
216 | raise SyncException(EXCEPTION_HASH) |
|
220 | raise SyncException(EXCEPTION_HASH) | |
217 |
|
221 | |||
218 | files.append(attached_file) |
|
222 | files.append(attached_file) | |
219 |
|
223 | |||
220 | if is_old: |
|
224 | if is_old: | |
221 | post = global_id.post |
|
225 | post = global_id.post | |
222 | Post.objects.update_post( |
|
226 | Post.objects.update_post( | |
223 | post, title=title, text=text, pub_time=pub_time, |
|
227 | post, title=title, text=text, pub_time=pub_time, | |
224 | tags=tags, files=files, tripcode=tripcode, |
|
228 | tags=tags, files=files, tripcode=tripcode, | |
225 | version=version) |
|
229 | version=version) | |
226 |
|
|
230 | logger.debug('Parsed updated post {}'.format(global_id)) | |
227 | else: |
|
231 | else: | |
228 | Post.objects.import_post( |
|
232 | Post.objects.import_post( | |
229 | title=title, text=text, pub_time=pub_time, |
|
233 | title=title, text=text, pub_time=pub_time, | |
230 | opening_post=opening_post, tags=tags, |
|
234 | opening_post=opening_post, tags=tags, | |
231 | global_id=global_id, files=files, tripcode=tripcode, |
|
235 | global_id=global_id, files=files, tripcode=tripcode, | |
232 | version=version) |
|
236 | version=version) | |
233 |
|
|
237 | logger.debug('Parsed new post {}'.format(global_id)) | |
234 | else: |
|
238 | else: | |
235 | raise SyncException(EXCEPTION_NODE.format(tag_status.text)) |
|
239 | raise SyncException(EXCEPTION_NODE.format(tag_status.text)) | |
236 |
|
240 | |||
237 | @staticmethod |
|
241 | @staticmethod | |
238 | def generate_response_list(): |
|
242 | def generate_response_list(): | |
239 | response = et.Element(TAG_RESPONSE) |
|
243 | response = et.Element(TAG_RESPONSE) | |
240 |
|
244 | |||
241 | status = et.SubElement(response, TAG_STATUS) |
|
245 | status = et.SubElement(response, TAG_STATUS) | |
242 | status.text = STATUS_SUCCESS |
|
246 | status.text = STATUS_SUCCESS | |
243 |
|
247 | |||
244 | models = et.SubElement(response, TAG_MODELS) |
|
248 | models = et.SubElement(response, TAG_MODELS) | |
245 |
|
249 | |||
246 | for post in Post.objects.prefetch_related('global_id').all(): |
|
250 | for post in Post.objects.prefetch_related('global_id').all(): | |
247 | tag_model = et.SubElement(models, TAG_MODEL) |
|
251 | tag_model = et.SubElement(models, TAG_MODEL) | |
248 | tag_id = et.SubElement(tag_model, TAG_ID) |
|
252 | tag_id = et.SubElement(tag_model, TAG_ID) | |
249 | post.global_id.to_xml_element(tag_id) |
|
253 | post.global_id.to_xml_element(tag_id) | |
250 | tag_version = et.SubElement(tag_model, TAG_VERSION) |
|
254 | tag_version = et.SubElement(tag_model, TAG_VERSION) | |
251 | tag_version.text = str(post.version) |
|
255 | tag_version.text = str(post.version) | |
252 |
|
256 | |||
253 | return et.tostring(response, ENCODING_UNICODE) |
|
257 | return et.tostring(response, ENCODING_UNICODE) | |
254 |
|
258 | |||
255 | @staticmethod |
|
259 | @staticmethod | |
256 | def _verify_model(global_id, content_str, tag_model): |
|
260 | def _verify_model(global_id, content_str, tag_model): | |
257 | """ |
|
261 | """ | |
258 | Verifies all signatures for a single model. |
|
262 | Verifies all signatures for a single model. | |
259 | """ |
|
263 | """ | |
260 |
|
264 | |||
261 | signatures = [] |
|
265 | signatures = [] | |
262 |
|
266 | |||
263 | tag_signatures = tag_model.find(TAG_SIGNATURES) |
|
267 | tag_signatures = tag_model.find(TAG_SIGNATURES) | |
264 | has_author_signature = False |
|
268 | has_author_signature = False | |
265 | for tag_signature in tag_signatures: |
|
269 | for tag_signature in tag_signatures: | |
266 | signature_type = tag_signature.get(ATTR_TYPE) |
|
270 | signature_type = tag_signature.get(ATTR_TYPE) | |
267 | signature_value = tag_signature.get(ATTR_VALUE) |
|
271 | signature_value = tag_signature.get(ATTR_VALUE) | |
268 | signature_key = tag_signature.get(ATTR_KEY) |
|
272 | signature_key = tag_signature.get(ATTR_KEY) | |
269 |
|
273 | |||
270 | if global_id.key_type == signature_type and\ |
|
274 | if global_id.key_type == signature_type and\ | |
271 | global_id.key == signature_key: |
|
275 | global_id.key == signature_key: | |
272 | has_author_signature = True |
|
276 | has_author_signature = True | |
273 |
|
277 | |||
274 | signature = Signature(key_type=signature_type, |
|
278 | signature = Signature(key_type=signature_type, | |
275 | key=signature_key, |
|
279 | key=signature_key, | |
276 | signature=signature_value) |
|
280 | signature=signature_value) | |
277 |
|
281 | |||
278 | if not KeyPair.objects.verify(signature, content_str): |
|
282 | if not KeyPair.objects.verify(signature, content_str): | |
279 | raise SyncException(EXCEPTION_SIGNATURE.format(content_str)) |
|
283 | raise SyncException(EXCEPTION_SIGNATURE.format(content_str)) | |
280 |
|
284 | |||
281 | signatures.append(signature) |
|
285 | signatures.append(signature) | |
282 | if not has_author_signature: |
|
286 | if not has_author_signature: | |
283 | raise SyncException(EXCEPTION_AUTHOR_SIGNATURE.format(content_str)) |
|
287 | raise SyncException(EXCEPTION_AUTHOR_SIGNATURE.format(content_str)) | |
284 |
|
288 | |||
285 | return signatures |
|
289 | return signatures | |
286 |
|
290 | |||
287 | @staticmethod |
|
291 | @staticmethod | |
288 | def _attachment_to_xml(tag_attachments, tag_refs, file, hash, url): |
|
292 | def _attachment_to_xml(tag_attachments, tag_refs, file, hash, url): | |
289 | if tag_attachments is not None: |
|
293 | if tag_attachments is not None: | |
290 | mimetype = get_file_mimetype(file) |
|
294 | mimetype = get_file_mimetype(file) | |
291 | attachment = et.SubElement(tag_attachments, TAG_ATTACHMENT) |
|
295 | attachment = et.SubElement(tag_attachments, TAG_ATTACHMENT) | |
292 | attachment.set(ATTR_MIMETYPE, mimetype) |
|
296 | attachment.set(ATTR_MIMETYPE, mimetype) | |
293 | attachment.set(ATTR_ID_TYPE, ID_TYPE_MD5) |
|
297 | attachment.set(ATTR_ID_TYPE, ID_TYPE_MD5) | |
294 | attachment.text = hash |
|
298 | attachment.text = hash | |
295 |
|
299 | |||
296 | attachment_ref = et.SubElement(tag_refs, TAG_ATTACHMENT_REF) |
|
300 | attachment_ref = et.SubElement(tag_refs, TAG_ATTACHMENT_REF) | |
297 | attachment_ref.set(ATTR_REF, hash) |
|
301 | attachment_ref.set(ATTR_REF, hash) | |
298 | attachment_ref.set(ATTR_URL, url) |
|
302 | attachment_ref.set(ATTR_URL, url) |
General Comments 0
You need to be logged in to leave comments.
Login now