##// END OF EJS Templates
Sync fixes
neko259 -
r1386:1a1d9a43 decentral
parent child Browse files
Show More
@@ -1,74 +1,79 b''
1 1 import re
2 2 import xml.etree.ElementTree as ET
3 3
4 4 import httplib2
5 5 from django.core.management import BaseCommand
6 6
7 7 from boards.models import GlobalId
8 8 from boards.models.post.sync import SyncManager
9 9
10 10 __author__ = 'neko259'
11 11
12 12
13 13 REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)')
14 14
15 15
16 16 class Command(BaseCommand):
17 17 help = 'Send a sync or get request to the server.'
18 18
19 19 def add_arguments(self, parser):
20 parser.add_argument('url', type=str)
21 parser.add_argument('--global_id', type=str, default='',
20 parser.add_argument('url', type=str, help='Server root url')
21 parser.add_argument('--global-id', type=str, default='',
22 22 help='Post global ID')
23 23
24 24 def handle(self, *args, **options):
25 25 url = options.get('url')
26
27 pull_url = url + 'api/sync/pull/'
28 get_url = url + 'api/sync/get/'
29
26 30 global_id_str = options.get('global_id')
27 31 if global_id_str:
28 32 match = REGEX_GLOBAL_ID.match(global_id_str)
29 33 if match:
30 34 key_type = match.group(1)
31 35 key = match.group(2)
32 36 local_id = match.group(3)
33 37
34 38 global_id = GlobalId(key_type=key_type, key=key,
35 39 local_id=local_id)
36 40
37 41 xml = GlobalId.objects.generate_request_get([global_id])
38 42 # body = urllib.parse.urlencode(data)
39 43 h = httplib2.Http()
40 response, content = h.request(url, method="POST", body=xml)
44 response, content = h.request(get_url, method="POST", body=xml)
41 45
42 46 SyncManager.parse_response_get(content)
43 47 else:
44 48 raise Exception('Invalid global ID')
45 49 else:
46 50 h = httplib2.Http()
47 51 xml = GlobalId.objects.generate_request_pull()
48 response, content = h.request(url, method="POST", body=xml)
52 response, content = h.request(pull_url, method="POST", body=xml)
49 53
50 print(content)
54 print(content.decode() + '\n')
51 55
52 56 root = ET.fromstring(content)
53 57 status = root.findall('status')[0].text
54 58 if status == 'success':
55 59 ids_to_sync = list()
56 60
57 61 models = root.findall('models')[0]
58 62 for model in models:
59 63 global_id, exists = GlobalId.from_xml_element(model)
60 print(global_id)
61 64 if not exists:
65 print(global_id)
62 66 ids_to_sync.append(global_id)
67 print()
63 68
64 69 if len(ids_to_sync) > 0:
65 70 xml = GlobalId.objects.generate_request_get(ids_to_sync)
66 71 # body = urllib.parse.urlencode(data)
67 72 h = httplib2.Http()
68 response, content = h.request(url, method="POST", body=xml)
73 response, content = h.request(get_url, method="POST", body=xml)
69 74
70 75 SyncManager.parse_response_get(content)
71 76 else:
72 77 print('Nothing to get, everything synced')
73 78 else:
74 79 raise Exception('Invalid response status')
@@ -1,151 +1,152 b''
1 1 import logging
2 2
3 3 from datetime import datetime, timedelta, date
4 4 from datetime import time as dtime
5 5
6 6 from django.db import models, transaction
7 7 from django.utils import timezone
8 8
9 9 import boards
10 10
11 11 from boards.models.user import Ban
12 12 from boards.mdx_neboard import Parser
13 13 from boards.models import PostImage, Attachment
14 14 from boards import utils
15 15
16 16 __author__ = 'neko259'
17 17
18 18 IMAGE_TYPES = (
19 19 'jpeg',
20 20 'jpg',
21 21 'png',
22 22 'bmp',
23 23 'gif',
24 24 )
25 25
26 26 POSTS_PER_DAY_RANGE = 7
27 27 NO_IP = '0.0.0.0'
28 28
29 29
30 30 class PostManager(models.Manager):
31 31 @transaction.atomic
32 32 def create_post(self, title: str, text: str, file=None, thread=None,
33 33 ip=NO_IP, tags: list=None, opening_posts: list=None,
34 34 tripcode=''):
35 35 """
36 36 Creates new post
37 37 """
38 38
39 39 if not utils.is_anonymous_mode():
40 40 is_banned = Ban.objects.filter(ip=ip).exists()
41 41
42 42 # TODO Raise specific exception and catch it in the views
43 43 if is_banned:
44 44 raise Exception("This user is banned")
45 45
46 46 if not tags:
47 47 tags = []
48 48 if not opening_posts:
49 49 opening_posts = []
50 50
51 51 posting_time = timezone.now()
52 52 new_thread = False
53 53 if not thread:
54 54 thread = boards.models.thread.Thread.objects.create(
55 55 bump_time=posting_time, last_edit_time=posting_time)
56 56 list(map(thread.tags.add, tags))
57 57 boards.models.thread.Thread.objects.process_oldest_threads()
58 58 new_thread = True
59 59
60 60 pre_text = Parser().preparse(text)
61 61
62 62 post = self.create(title=title,
63 63 text=pre_text,
64 64 pub_time=posting_time,
65 65 poster_ip=ip,
66 66 thread=thread,
67 67 last_edit_time=posting_time,
68 68 tripcode=tripcode,
69 69 opening=new_thread)
70 70 post.threads.add(thread)
71 71
72 72 logger = logging.getLogger('boards.post.create')
73 73
74 74 logger.info('Created post [{}] with text [{}] by {}'.format(post,
75 75 post.get_text(),post.poster_ip))
76 76
77 77 # TODO Move this to other place
78 78 if file:
79 79 file_type = file.name.split('.')[-1].lower()
80 80 if file_type in IMAGE_TYPES:
81 81 post.images.add(PostImage.objects.create_with_hash(file))
82 82 else:
83 83 post.attachments.add(Attachment.objects.create_with_hash(file))
84 84
85 85 post.build_url()
86 86 post.connect_replies()
87 87 post.connect_threads(opening_posts)
88 88 post.connect_notifications()
89 89 post.set_global_id()
90 90
91 91 # Thread needs to be bumped only when the post is already created
92 92 if not new_thread:
93 93 thread.last_edit_time = posting_time
94 94 thread.bump()
95 95 thread.save()
96 96
97 97 return post
98 98
99 99 def delete_posts_by_ip(self, ip):
100 100 """
101 101 Deletes all posts of the author with same IP
102 102 """
103 103
104 104 posts = self.filter(poster_ip=ip)
105 105 for post in posts:
106 106 post.delete()
107 107
108 108 @utils.cached_result()
109 109 def get_posts_per_day(self) -> float:
110 110 """
111 111 Gets average count of posts per day for the last 7 days
112 112 """
113 113
114 114 day_end = date.today()
115 115 day_start = day_end - timedelta(POSTS_PER_DAY_RANGE)
116 116
117 117 day_time_start = timezone.make_aware(datetime.combine(
118 118 day_start, dtime()), timezone.get_current_timezone())
119 119 day_time_end = timezone.make_aware(datetime.combine(
120 120 day_end, dtime()), timezone.get_current_timezone())
121 121
122 122 posts_per_period = float(self.filter(
123 123 pub_time__lte=day_time_end,
124 124 pub_time__gte=day_time_start).count())
125 125
126 126 ppd = posts_per_period / POSTS_PER_DAY_RANGE
127 127
128 128 return ppd
129 129
130 130 @transaction.atomic
131 131 def import_post(self, title: str, text: str, pub_time: str, global_id,
132 132 opening_post=None, tags=list()):
133 133 is_opening = opening_post is None
134 134 if is_opening:
135 135 thread = boards.models.thread.Thread.objects.create(
136 136 bump_time=pub_time, last_edit_time=pub_time)
137 137 list(map(thread.tags.add, tags))
138 138 else:
139 139 thread = opening_post.get_thread()
140 140
141 141 post = self.create(title=title, text=text,
142 142 pub_time=pub_time,
143 143 poster_ip=NO_IP,
144 144 last_edit_time=pub_time,
145 thread_id=thread.id,
146 145 global_id=global_id,
147 opening=is_opening)
146 opening=is_opening,
147 thread=thread)
148 148
149 post.threads.add(thread)
149 150 post.build_url()
150 151 post.connect_replies()
151 152 post.connect_notifications()
@@ -1,190 +1,201 b''
1 1 import xml.etree.ElementTree as et
2 2 from django.db import transaction
3 3 from boards.models import KeyPair, GlobalId, Signature, Post, Tag
4 4
5 5 ENCODING_UNICODE = 'unicode'
6 6
7 7 TAG_MODEL = 'model'
8 8 TAG_REQUEST = 'request'
9 9 TAG_RESPONSE = 'response'
10 10 TAG_ID = 'id'
11 11 TAG_STATUS = 'status'
12 12 TAG_MODELS = 'models'
13 13 TAG_TITLE = 'title'
14 14 TAG_TEXT = 'text'
15 15 TAG_THREAD = 'thread'
16 16 TAG_PUB_TIME = 'pub-time'
17 17 TAG_SIGNATURES = 'signatures'
18 18 TAG_SIGNATURE = 'signature'
19 19 TAG_CONTENT = 'content'
20 20 TAG_ATTACHMENTS = 'attachments'
21 21 TAG_ATTACHMENT = 'attachment'
22 22 TAG_TAGS = 'tags'
23 23 TAG_TAG = 'tag'
24 24
25 25 TYPE_GET = 'get'
26 26
27 27 ATTR_VERSION = 'version'
28 28 ATTR_TYPE = 'type'
29 29 ATTR_NAME = 'name'
30 30 ATTR_VALUE = 'value'
31 31 ATTR_MIMETYPE = 'mimetype'
32 32 ATTR_KEY = 'key'
33 33
34 34 STATUS_SUCCESS = 'success'
35 35
36 36
37 37 class SyncManager:
38 38 @staticmethod
39 39 def generate_response_get(model_list: list):
40 40 response = et.Element(TAG_RESPONSE)
41 41
42 42 status = et.SubElement(response, TAG_STATUS)
43 43 status.text = STATUS_SUCCESS
44 44
45 45 models = et.SubElement(response, TAG_MODELS)
46 46
47 47 for post in model_list:
48 48 model = et.SubElement(models, TAG_MODEL)
49 49 model.set(ATTR_NAME, 'post')
50 50
51 51 content_tag = et.SubElement(model, TAG_CONTENT)
52 52
53 53 tag_id = et.SubElement(content_tag, TAG_ID)
54 54 post.global_id.to_xml_element(tag_id)
55 55
56 56 title = et.SubElement(content_tag, TAG_TITLE)
57 57 title.text = post.title
58 58
59 59 text = et.SubElement(content_tag, TAG_TEXT)
60 60 text.text = post.get_sync_text()
61 61
62 62 thread = post.get_thread()
63 63 if post.is_opening():
64 64 tag_tags = et.SubElement(content_tag, TAG_TAGS)
65 65 for tag in thread.get_tags():
66 66 tag_tag = et.SubElement(tag_tags, TAG_TAG)
67 67 tag_tag.text = tag.name
68 68 else:
69 69 tag_thread = et.SubElement(content_tag, TAG_THREAD)
70 70 thread_id = et.SubElement(tag_thread, TAG_ID)
71 71 thread.get_opening_post().global_id.to_xml_element(thread_id)
72 72
73 73 pub_time = et.SubElement(content_tag, TAG_PUB_TIME)
74 74 pub_time.text = str(post.get_pub_time_str())
75 75
76 76 signatures_tag = et.SubElement(model, TAG_SIGNATURES)
77 77 post_signatures = post.global_id.signature_set.all()
78 78 if post_signatures:
79 79 signatures = post_signatures
80 80 # TODO Adding signature to a post is not yet added. For now this
81 81 # block is useless
82 82 else:
83 83 # TODO Maybe the signature can be computed only once after
84 84 # the post is added? Need to add some on_save signal queue
85 85 # and add this there.
86 86 key = KeyPair.objects.get(public_key=post.global_id.key)
87 signatures = [Signature(
87 signature = Signature(
88 88 key_type=key.key_type,
89 89 key=key.public_key,
90 90 signature=key.sign(et.tostring(content_tag, ENCODING_UNICODE)),
91 )]
91 global_id=post.global_id,
92 )
93 signature.save()
94 signatures = [signature]
92 95 for signature in signatures:
93 96 signature_tag = et.SubElement(signatures_tag, TAG_SIGNATURE)
94 97 signature_tag.set(ATTR_TYPE, signature.key_type)
95 98 signature_tag.set(ATTR_VALUE, signature.signature)
96 99 signature_tag.set(ATTR_KEY, signature.key)
97 100
98 101 return et.tostring(response, ENCODING_UNICODE)
99 102
100 103 @staticmethod
101 104 @transaction.atomic
102 105 def parse_response_get(response_xml):
103 106 tag_root = et.fromstring(response_xml)
104 107 tag_status = tag_root.find(TAG_STATUS)
105 108 if STATUS_SUCCESS == tag_status.text:
106 109 tag_models = tag_root.find(TAG_MODELS)
107 110 for tag_model in tag_models:
108 111 tag_content = tag_model.find(TAG_CONTENT)
109 112
110 113 signatures = SyncManager._verify_model(tag_content, tag_model)
111 114
112 115 tag_id = tag_content.find(TAG_ID)
113 116 global_id, exists = GlobalId.from_xml_element(tag_id)
114 117
115 118 if exists:
116 119 print('Post with same ID already exists')
117 120 else:
118 121 global_id.save()
119 122 for signature in signatures:
120 123 signature.global_id = global_id
121 124 signature.save()
122 125
123 126 title = tag_content.find(TAG_TITLE).text
124 127 text = tag_content.find(TAG_TEXT).text
125 128 pub_time = tag_content.find(TAG_PUB_TIME).text
126 129
127 130 thread = tag_content.find(TAG_THREAD)
128 131 tags = []
129 132 if thread:
130 opening_post = Post.objects.get(
131 id=thread.find(TAG_ID).text)
133 thread_id = thread.find(TAG_ID)
134 op_global_id, exists = GlobalId.from_xml_element(thread_id)
135 if exists:
136 opening_post = Post.objects.get(global_id=op_global_id)
137 else:
138 raise Exception('Load the OP first')
132 139 else:
133 140 opening_post = None
134 141 tag_tags = tag_content.find(TAG_TAGS)
135 142 for tag_tag in tag_tags:
136 tag, created = Tag.objects.get_or_create(name=tag_tag.text)
143 tag, created = Tag.objects.get_or_create(
144 name=tag_tag.text)
137 145 tags.append(tag)
138 146
139 147 # TODO Check that the replied posts are already present
140 148 # before adding new ones
141 149
142 150 # TODO Get images
143 151
144 152 post = Post.objects.import_post(
145 153 title=title, text=text, pub_time=pub_time,
146 154 opening_post=opening_post, tags=tags,
147 155 global_id=global_id)
148 156 else:
149 157 # TODO Throw an exception?
150 158 pass
151 159
152 160 @staticmethod
153 161 def generate_response_pull():
154 162 response = et.Element(TAG_RESPONSE)
155 163
156 164 status = et.SubElement(response, TAG_STATUS)
157 165 status.text = STATUS_SUCCESS
158 166
159 167 models = et.SubElement(response, TAG_MODELS)
160 168
161 169 for post in Post.objects.all():
162 170 tag_id = et.SubElement(models, TAG_ID)
163 171 post.global_id.to_xml_element(tag_id)
164 172
165 173 return et.tostring(response, ENCODING_UNICODE)
166 174
167 175 @staticmethod
168 176 def _verify_model(tag_content, tag_model):
169 177 """
170 178 Verifies all signatures for a single model.
171 179 """
172 180
173 181 signatures = []
174 182
175 183 tag_signatures = tag_model.find(TAG_SIGNATURES)
176 184 for tag_signature in tag_signatures:
177 185 signature_type = tag_signature.get(ATTR_TYPE)
178 186 signature_value = tag_signature.get(ATTR_VALUE)
179 187 signature_key = tag_signature.get(ATTR_KEY)
180 188
181 189 signature = Signature(key_type=signature_type,
182 190 key=signature_key,
183 191 signature=signature_value)
184 signatures.append(signature)
192
193 content = et.tostring(tag_content, ENCODING_UNICODE)
185 194
186 195 if not KeyPair.objects.verify(
187 signature, et.tostring(tag_content, ENCODING_UNICODE)):
188 raise Exception('Invalid model signature')
196 signature, content):
197 raise Exception('Invalid model signature for {}'.format(content))
198
199 signatures.append(signature)
189 200
190 201 return signatures
General Comments 0
You need to be logged in to leave comments. Login now