Show More
@@ -0,0 +1,31 b'' | |||
|
1 | import xml.etree.ElementTree as et | |
|
2 | ||
|
3 | from boards.models import Post | |
|
4 | ||
|
5 | TAG_THREAD = 'thread' | |
|
6 | ||
|
7 | ||
|
8 | class PostFilter: | |
|
9 | def __init__(self, content=None): | |
|
10 | self.content = content | |
|
11 | ||
|
12 | def filter(self, posts): | |
|
13 | return posts | |
|
14 | ||
|
15 | def add_filter(self, model_tag, value): | |
|
16 | return model_tag | |
|
17 | ||
|
18 | ||
|
19 | class ThreadFilter(PostFilter): | |
|
20 | def filter(self, posts): | |
|
21 | op_id = self.content.text | |
|
22 | ||
|
23 | op = Post.objects.filter(opening=True, id=op_id).first() | |
|
24 | if op: | |
|
25 | return posts.filter(thread=op.get_thread()) | |
|
26 | else: | |
|
27 | return posts.none() | |
|
28 | ||
|
29 | def add_filter(self, model_tag, value): | |
|
30 | thread_tag = et.SubElement(model_tag, TAG_THREAD) | |
|
31 | thread_tag.text = str(value) |
@@ -0,0 +1,24 b'' | |||
|
1 | # -*- coding: utf-8 -*- | |
|
2 | # Generated by Django 1.10.5 on 2017-01-23 14:20 | |
|
3 | from __future__ import unicode_literals | |
|
4 | ||
|
5 | from django.db import migrations, models | |
|
6 | ||
|
7 | ||
|
8 | class Migration(migrations.Migration): | |
|
9 | ||
|
10 | dependencies = [ | |
|
11 | ('boards', '0055_auto_20161229_1132'), | |
|
12 | ] | |
|
13 | ||
|
14 | operations = [ | |
|
15 | migrations.AlterModelOptions( | |
|
16 | name='attachment', | |
|
17 | options={'ordering': ('id',)}, | |
|
18 | ), | |
|
19 | migrations.AlterField( | |
|
20 | model_name='post', | |
|
21 | name='uid', | |
|
22 | field=models.TextField(), | |
|
23 | ), | |
|
24 | ] |
@@ -1,48 +1,49 b'' | |||
|
1 | 1 | [Version] |
|
2 | 2 | Version = 4.1.0 2017 |
|
3 | 3 | SiteName = Neboard DEV |
|
4 | 4 | |
|
5 | 5 | [Cache] |
|
6 | 6 | # Timeout for caching, if cache is used |
|
7 | 7 | CacheTimeout = 600 |
|
8 | 8 | |
|
9 | 9 | [Forms] |
|
10 | 10 | # Max post length in characters |
|
11 | 11 | MaxTextLength = 30000 |
|
12 | 12 | MaxFileSize = 8000000 |
|
13 | 13 | LimitFirstPosting = true |
|
14 | 14 | LimitPostingSpeed = false |
|
15 | 15 | PowDifficulty = 0 |
|
16 | 16 | # Delay in seconds |
|
17 | 17 | PostingDelay = 30 |
|
18 | 18 | Autoban = false |
|
19 | 19 | DefaultTag = test |
|
20 | 20 | MaxFileCount = 1 |
|
21 | AdditionalSpoilerSpaces = false | |
|
21 | 22 | |
|
22 | 23 | [Messages] |
|
23 | 24 | # Thread bumplimit |
|
24 | 25 | MaxPostsPerThread = 10 |
|
25 | 26 | ThreadArchiveDays = 300 |
|
26 | 27 | AnonymousMode = false |
|
27 | 28 | |
|
28 | 29 | [View] |
|
29 | 30 | DefaultTheme = md |
|
30 | 31 | DefaultImageViewer = simple |
|
31 | 32 | LastRepliesCount = 3 |
|
32 | 33 | ThreadsPerPage = 3 |
|
33 | 34 | PostsPerPage = 10 |
|
34 | 35 | ImagesPerPageGallery = 20 |
|
35 | 36 | MaxFavoriteThreads = 20 |
|
36 | 37 | MaxLandingThreads = 20 |
|
37 | 38 | Themes=md:Mystic Dark,md_centered:Mystic Dark (centered),sw:Snow White,pg:Photon Grey,ad:Amanita Dark,iw:Inocibe White |
|
38 | 39 | ImageViewers=simple:Simple,popup:Popup |
|
39 | 40 | |
|
40 | 41 | [Storage] |
|
41 | 42 | # Enable archiving threads instead of deletion when the thread limit is reached |
|
42 | 43 | ArchiveThreads = true |
|
43 | 44 | |
|
44 | 45 | [RSS] |
|
45 | 46 | MaxItems = 20 |
|
46 | 47 | |
|
47 | 48 | [External] |
|
48 | 49 | ImageSearchHost= |
@@ -1,96 +1,99 b'' | |||
|
1 | 1 | import re |
|
2 | 2 | import logging |
|
3 | 3 | import xml.etree.ElementTree as ET |
|
4 | 4 | |
|
5 | 5 | import httplib2 |
|
6 | 6 | from django.core.management import BaseCommand |
|
7 | 7 | |
|
8 | 8 | from boards.models import GlobalId |
|
9 | 9 | from boards.models.post.sync import SyncManager, TAG_ID, TAG_VERSION |
|
10 | 10 | |
|
11 | 11 | __author__ = 'neko259' |
|
12 | 12 | |
|
13 | 13 | |
|
14 | 14 | REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)') |
|
15 | 15 | |
|
16 | 16 | |
|
17 | 17 | class Command(BaseCommand): |
|
18 | 18 | help = 'Send a sync or get request to the server.' |
|
19 | 19 | |
|
20 | 20 | def add_arguments(self, parser): |
|
21 | 21 | parser.add_argument('url', type=str, help='Server root url') |
|
22 | 22 | parser.add_argument('--global-id', type=str, default='', |
|
23 | 23 | help='Post global ID') |
|
24 | 24 | parser.add_argument('--split-query', type=int, default=1, |
|
25 | 25 | help='Split GET query into separate by the given' |
|
26 | 26 | ' number of posts in one') |
|
27 | parser.add_argument('--thread', type=int, | |
|
28 | help='Get posts of one specific thread') | |
|
27 | 29 | |
|
28 | 30 | def handle(self, *args, **options): |
|
29 | 31 | logger = logging.getLogger('boards.sync') |
|
30 | 32 | |
|
31 | 33 | url = options.get('url') |
|
32 | 34 | |
|
33 | 35 | list_url = url + 'api/sync/list/' |
|
34 | 36 | get_url = url + 'api/sync/get/' |
|
35 | 37 | file_url = url[:-1] |
|
36 | 38 | |
|
37 | 39 | global_id_str = options.get('global_id') |
|
38 | 40 | if global_id_str: |
|
39 | 41 | match = REGEX_GLOBAL_ID.match(global_id_str) |
|
40 | 42 | if match: |
|
41 | 43 | key_type = match.group(1) |
|
42 | 44 | key = match.group(2) |
|
43 | 45 | local_id = match.group(3) |
|
44 | 46 | |
|
45 | 47 | global_id = GlobalId(key_type=key_type, key=key, |
|
46 | 48 | local_id=local_id) |
|
47 | 49 | |
|
48 |
xml = |
|
|
50 | xml = SyncManager.generate_request_get([global_id]) | |
|
49 | 51 | h = httplib2.Http() |
|
50 | 52 | response, content = h.request(get_url, method="POST", body=xml) |
|
51 | 53 | |
|
52 | 54 | SyncManager.parse_response_get(content, file_url) |
|
53 | 55 | else: |
|
54 | 56 | raise Exception('Invalid global ID') |
|
55 | 57 | else: |
|
56 | 58 | logger.info('Running LIST request...') |
|
57 | 59 | h = httplib2.Http() |
|
58 |
xml = |
|
|
60 | xml = SyncManager.generate_request_list( | |
|
61 | opening_post=options.get('thread')) | |
|
59 | 62 | response, content = h.request(list_url, method="POST", body=xml) |
|
60 | 63 | logger.info('Processing response...') |
|
61 | 64 | |
|
62 | 65 | root = ET.fromstring(content) |
|
63 | 66 | status = root.findall('status')[0].text |
|
64 | 67 | if status == 'success': |
|
65 | 68 | ids_to_sync = list() |
|
66 | 69 | |
|
67 | 70 | models = root.findall('models')[0] |
|
68 | 71 | for model in models: |
|
69 | 72 | tag_id = model.find(TAG_ID) |
|
70 | 73 | global_id, exists = GlobalId.from_xml_element(tag_id) |
|
71 | 74 | tag_version = model.find(TAG_VERSION) |
|
72 | 75 | if tag_version is not None: |
|
73 | 76 | version = int(tag_version.text) or 1 |
|
74 | 77 | else: |
|
75 | 78 | version = 1 |
|
76 | 79 | if not exists or global_id.post.version < version: |
|
77 | 80 | logger.debug('Processed (+) post {}'.format(global_id)) |
|
78 | 81 | ids_to_sync.append(global_id) |
|
79 | 82 | else: |
|
80 | 83 | logger.debug('* Processed (-) post {}'.format(global_id)) |
|
81 | 84 | logger.info('Starting sync...') |
|
82 | 85 | |
|
83 | 86 | if len(ids_to_sync) > 0: |
|
84 | 87 | limit = options.get('split_query', len(ids_to_sync)) |
|
85 | 88 | for offset in range(0, len(ids_to_sync), limit): |
|
86 |
xml = |
|
|
89 | xml = SyncManager.generate_request_get(ids_to_sync[offset:offset + limit]) | |
|
87 | 90 | h = httplib2.Http() |
|
88 | 91 | logger.info('Running GET request...') |
|
89 | 92 | response, content = h.request(get_url, method="POST", body=xml) |
|
90 | 93 | logger.info('Processing response...') |
|
91 | 94 | |
|
92 | 95 | SyncManager.parse_response_get(content, file_url) |
|
93 | 96 | else: |
|
94 | 97 | logger.info('Nothing to get, everything synced') |
|
95 | 98 | else: |
|
96 | 99 | raise Exception('Invalid response status') |
@@ -1,276 +1,282 b'' | |||
|
1 | 1 | # coding=utf-8 |
|
2 | 2 | |
|
3 | 3 | import re |
|
4 | 4 | import random |
|
5 | 5 | import bbcode |
|
6 | 6 | |
|
7 | 7 | from urllib.parse import unquote |
|
8 | 8 | |
|
9 | 9 | from django.core.exceptions import ObjectDoesNotExist |
|
10 | 10 | from django.core.urlresolvers import reverse |
|
11 | 11 | |
|
12 | 12 | import boards |
|
13 | from boards import settings | |
|
13 | 14 | |
|
14 | 15 | |
|
15 | 16 | __author__ = 'neko259' |
|
16 | 17 | |
|
17 | 18 | |
|
18 | 19 | REFLINK_PATTERN = re.compile(r'^\d+$') |
|
19 | 20 | GLOBAL_REFLINK_PATTERN = re.compile(r'(\w+)::([^:]+)::(\d+)') |
|
20 | 21 | MULTI_NEWLINES_PATTERN = re.compile(r'(\r?\n){2,}') |
|
21 | 22 | ONE_NEWLINE = '\n' |
|
22 | 23 | REGEX_URL = re.compile(r'https?\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(/\S*)?') |
|
23 | 24 | LINE_BREAK_HTML = '<div class="br"></div>' |
|
24 | 25 | SPOILER_SPACE = ' ' |
|
25 | 26 | |
|
26 | 27 | MAX_SPOILER_MULTIPLIER = 2 |
|
28 | MAX_SPOILER_SPACE_COUNT = 20 | |
|
27 | 29 | |
|
28 | 30 | |
|
29 | 31 | class TextFormatter(): |
|
30 | 32 | """ |
|
31 | 33 | An interface for formatter that can be used in the text format panel |
|
32 | 34 | """ |
|
33 | 35 | |
|
34 | 36 | def __init__(self): |
|
35 | 37 | pass |
|
36 | 38 | |
|
37 | 39 | name = '' |
|
38 | 40 | |
|
39 | 41 | # Left and right tags for the button preview |
|
40 | 42 | preview_left = '' |
|
41 | 43 | preview_right = '' |
|
42 | 44 | |
|
43 | 45 | # Left and right characters for the textarea input |
|
44 | 46 | format_left = '' |
|
45 | 47 | format_right = '' |
|
46 | 48 | |
|
47 | 49 | |
|
48 | 50 | class AutolinkPattern(): |
|
49 | 51 | def handleMatch(self, m): |
|
50 | 52 | link_element = etree.Element('a') |
|
51 | 53 | href = m.group(2) |
|
52 | 54 | link_element.set('href', href) |
|
53 | 55 | link_element.text = href |
|
54 | 56 | |
|
55 | 57 | return link_element |
|
56 | 58 | |
|
57 | 59 | |
|
58 | 60 | class QuotePattern(TextFormatter): |
|
59 | 61 | name = '>q' |
|
60 | 62 | preview_left = '<span class="quote">' |
|
61 | 63 | preview_right = '</span>' |
|
62 | 64 | |
|
63 | 65 | format_left = '[quote]' |
|
64 | 66 | format_right = '[/quote]' |
|
65 | 67 | |
|
66 | 68 | |
|
67 | 69 | class SpoilerPattern(TextFormatter): |
|
68 | 70 | name = 'spoiler' |
|
69 | 71 | preview_left = '<span class="spoiler">' |
|
70 | 72 | preview_right = '</span>' |
|
71 | 73 | |
|
72 | 74 | format_left = '[spoiler]' |
|
73 | 75 | format_right = '[/spoiler]' |
|
74 | 76 | |
|
75 | 77 | |
|
76 | 78 | class CommentPattern(TextFormatter): |
|
77 | 79 | name = '' |
|
78 | 80 | preview_left = '<span class="comment">// ' |
|
79 | 81 | preview_right = '</span>' |
|
80 | 82 | |
|
81 | 83 | format_left = '[comment]' |
|
82 | 84 | format_right = '[/comment]' |
|
83 | 85 | |
|
84 | 86 | |
|
85 | 87 | # TODO Use <s> tag here |
|
86 | 88 | class StrikeThroughPattern(TextFormatter): |
|
87 | 89 | name = 's' |
|
88 | 90 | preview_left = '<span class="strikethrough">' |
|
89 | 91 | preview_right = '</span>' |
|
90 | 92 | |
|
91 | 93 | format_left = '[s]' |
|
92 | 94 | format_right = '[/s]' |
|
93 | 95 | |
|
94 | 96 | |
|
95 | 97 | class ItalicPattern(TextFormatter): |
|
96 | 98 | name = 'i' |
|
97 | 99 | preview_left = '<i>' |
|
98 | 100 | preview_right = '</i>' |
|
99 | 101 | |
|
100 | 102 | format_left = '[i]' |
|
101 | 103 | format_right = '[/i]' |
|
102 | 104 | |
|
103 | 105 | |
|
104 | 106 | class BoldPattern(TextFormatter): |
|
105 | 107 | name = 'b' |
|
106 | 108 | preview_left = '<b>' |
|
107 | 109 | preview_right = '</b>' |
|
108 | 110 | |
|
109 | 111 | format_left = '[b]' |
|
110 | 112 | format_right = '[/b]' |
|
111 | 113 | |
|
112 | 114 | |
|
113 | 115 | class CodePattern(TextFormatter): |
|
114 | 116 | name = 'code' |
|
115 | 117 | preview_left = '<code>' |
|
116 | 118 | preview_right = '</code>' |
|
117 | 119 | |
|
118 | 120 | format_left = '[code]' |
|
119 | 121 | format_right = '[/code]' |
|
120 | 122 | |
|
121 | 123 | |
|
122 | 124 | class HintPattern(TextFormatter): |
|
123 | 125 | name = 'hint' |
|
124 | 126 | preview_left = '<span class="hint">' |
|
125 | 127 | preview_right = '</span>' |
|
126 | 128 | |
|
127 | 129 | format_left = '[hint]' |
|
128 | 130 | format_right = '[/hint]' |
|
129 | 131 | |
|
130 | 132 | |
|
131 | 133 | def render_reflink(tag_name, value, options, parent, context): |
|
132 | 134 | result = '>>%s' % value |
|
133 | 135 | |
|
134 | 136 | post = None |
|
135 | 137 | if REFLINK_PATTERN.match(value): |
|
136 | 138 | post_id = int(value) |
|
137 | 139 | |
|
138 | 140 | try: |
|
139 | 141 | post = boards.models.Post.objects.get(id=post_id) |
|
140 | 142 | |
|
141 | 143 | except ObjectDoesNotExist: |
|
142 | 144 | pass |
|
143 | 145 | elif GLOBAL_REFLINK_PATTERN.match(value): |
|
144 | 146 | match = GLOBAL_REFLINK_PATTERN.search(value) |
|
145 | 147 | try: |
|
146 | 148 | global_id = boards.models.GlobalId.objects.get( |
|
147 | 149 | key_type=match.group(1), key=match.group(2), |
|
148 | 150 | local_id=match.group(3)) |
|
149 | 151 | post = global_id.post |
|
150 | 152 | except ObjectDoesNotExist: |
|
151 | 153 | pass |
|
152 | 154 | |
|
153 | 155 | if post is not None: |
|
154 | 156 | result = post.get_link_view() |
|
155 | 157 | |
|
156 | 158 | return result |
|
157 | 159 | |
|
158 | 160 | |
|
159 | 161 | def render_quote(tag_name, value, options, parent, context): |
|
160 | 162 | source = '' |
|
161 | 163 | if 'source' in options: |
|
162 | 164 | source = options['source'] |
|
163 | 165 | elif 'quote' in options: |
|
164 | 166 | source = options['quote'] |
|
165 | 167 | |
|
166 | 168 | if source: |
|
167 | 169 | result = '<div class="multiquote"><div class="quote-header">%s</div><div class="quote-text">%s</div></div>' % (source, value) |
|
168 | 170 | else: |
|
169 | 171 | # Insert a ">" at the start of every line |
|
170 | 172 | result = '<span class="quote">>{}</span>'.format( |
|
171 | 173 | value.replace(LINE_BREAK_HTML, |
|
172 | 174 | '{}>'.format(LINE_BREAK_HTML))) |
|
173 | 175 | |
|
174 | 176 | return result |
|
175 | 177 | |
|
176 | 178 | |
|
177 | 179 | def render_hint(tag_name, value, options, parent, context): |
|
178 | 180 | if 'hint' in options: |
|
179 | 181 | hint = options['hint'] |
|
180 | 182 | result = '<span class="hint" title="{}">{}</span>'.format(hint, value) |
|
181 | 183 | else: |
|
182 | 184 | result = value |
|
183 | 185 | return result |
|
184 | 186 | |
|
185 | 187 | |
|
186 | 188 | def render_notification(tag_name, value, options, parent, content): |
|
187 | 189 | username = value.lower() |
|
188 | 190 | |
|
189 | 191 | return '<a href="{}" class="user-cast">@{}</a>'.format( |
|
190 | 192 | reverse('notifications', kwargs={'username': username}), username) |
|
191 | 193 | |
|
192 | 194 | |
|
193 | 195 | def render_tag(tag_name, value, options, parent, context): |
|
194 | 196 | tag_name = value.lower() |
|
195 | 197 | |
|
196 | 198 | try: |
|
197 | 199 | url = boards.models.Tag.objects.get(name=tag_name).get_view() |
|
198 | 200 | except ObjectDoesNotExist: |
|
199 | 201 | url = tag_name |
|
200 | 202 | |
|
201 | 203 | return url |
|
202 | 204 | |
|
203 | 205 | |
|
204 | 206 | def render_spoiler(tag_name, value, options, parent, context): |
|
205 | text_len = len(value) | |
|
206 | space_count = random.randint(0, text_len * MAX_SPOILER_MULTIPLIER) | |
|
207 | side_spaces = SPOILER_SPACE * (space_count // 2) | |
|
208 | return '<span class="spoiler">{}{}{}</span>'.format(side_spaces, value, | |
|
209 | side_spaces) | |
|
207 | if settings.get_bool('Forms', 'AdditionalSpoilerSpaces'): | |
|
208 | text_len = len(value) | |
|
209 | space_count = min(random.randint(0, text_len * MAX_SPOILER_MULTIPLIER), | |
|
210 | MAX_SPOILER_SPACE_COUNT) | |
|
211 | side_spaces = SPOILER_SPACE * (space_count // 2) | |
|
212 | else: | |
|
213 | side_spaces = '' | |
|
214 | return '<span class="spoiler">{}{}{}</span>'.format(side_spaces, | |
|
215 | value, side_spaces) | |
|
210 | 216 | |
|
211 | 217 | |
|
212 | 218 | formatters = [ |
|
213 | 219 | QuotePattern, |
|
214 | 220 | SpoilerPattern, |
|
215 | 221 | ItalicPattern, |
|
216 | 222 | BoldPattern, |
|
217 | 223 | CommentPattern, |
|
218 | 224 | StrikeThroughPattern, |
|
219 | 225 | CodePattern, |
|
220 | 226 | HintPattern, |
|
221 | 227 | ] |
|
222 | 228 | |
|
223 | 229 | |
|
224 | 230 | PREPARSE_PATTERNS = { |
|
225 | 231 | r'(?<!>)>>(\d+)': r'[post]\1[/post]', # Reflink ">>123" |
|
226 | 232 | r'^>([^>].+)': r'[quote]\1[/quote]', # Quote ">text" |
|
227 | 233 | r'^//\s?(.+)': r'[comment]\1[/comment]', # Comment "//text" |
|
228 | 234 | r'\B@(\w+)': r'[user]\1[/user]', # User notification "@user" |
|
229 | 235 | } |
|
230 | 236 | |
|
231 | 237 | |
|
232 | 238 | class Parser: |
|
233 | 239 | def __init__(self): |
|
234 | 240 | # The newline hack is added because br's margin does not work in all |
|
235 | 241 | # browsers except firefox, when the div's does. |
|
236 | 242 | self.parser = bbcode.Parser(newline=LINE_BREAK_HTML) |
|
237 | 243 | |
|
238 | 244 | self.parser.add_formatter('post', render_reflink, strip=True) |
|
239 | 245 | self.parser.add_formatter('quote', render_quote, strip=True) |
|
240 | 246 | self.parser.add_formatter('hint', render_hint, strip=True) |
|
241 | 247 | self.parser.add_formatter('user', render_notification, strip=True) |
|
242 | 248 | self.parser.add_formatter('tag', render_tag, strip=True) |
|
243 | 249 | self.parser.add_formatter('spoiler', render_spoiler, strip=True) |
|
244 | 250 | self.parser.add_simple_formatter( |
|
245 | 251 | 'comment', '<span class="comment">// %(value)s</span>', strip=True) |
|
246 | 252 | self.parser.add_simple_formatter( |
|
247 | 253 | 's', '<span class="strikethrough">%(value)s</span>') |
|
248 | 254 | self.parser.add_simple_formatter('code', |
|
249 | 255 | '<pre><code>%(value)s</pre></code>', |
|
250 | 256 | render_embedded=False, |
|
251 | 257 | escape_html=True, |
|
252 | 258 | replace_links=False, |
|
253 | 259 | replace_cosmetic=False) |
|
254 | 260 | |
|
255 | 261 | def preparse(self, text): |
|
256 | 262 | """ |
|
257 | 263 | Performs manual parsing before the bbcode parser is used. |
|
258 | 264 | Preparsed text is saved as raw and the text before preparsing is lost. |
|
259 | 265 | """ |
|
260 | 266 | new_text = MULTI_NEWLINES_PATTERN.sub(ONE_NEWLINE, text) |
|
261 | 267 | |
|
262 | 268 | for key, value in PREPARSE_PATTERNS.items(): |
|
263 | 269 | new_text = re.sub(key, value, new_text, flags=re.MULTILINE) |
|
264 | 270 | |
|
265 | 271 | for link in REGEX_URL.findall(text): |
|
266 | 272 | new_text = new_text.replace(link, unquote(link)) |
|
267 | 273 | |
|
268 | 274 | return new_text |
|
269 | 275 | |
|
270 | 276 | def parse(self, text): |
|
271 | 277 | return self.parser.format(text) |
|
272 | 278 | |
|
273 | 279 | |
|
274 | 280 | parser = Parser() |
|
275 | 281 | def get_parser(): |
|
276 | 282 | return parser |
@@ -1,101 +1,101 b'' | |||
|
1 | 1 | import re |
|
2 | 2 | |
|
3 | 3 | import requests |
|
4 | 4 | from django.core.files.uploadedfile import TemporaryUploadedFile |
|
5 | 5 | from pytube import YouTube |
|
6 | 6 | |
|
7 | 7 | from boards.utils import validate_file_size |
|
8 | 8 | |
|
9 | 9 | YOUTUBE_VIDEO_FORMAT = 'webm' |
|
10 | 10 | |
|
11 | 11 | HTTP_RESULT_OK = 200 |
|
12 | 12 | |
|
13 | 13 | HEADER_CONTENT_LENGTH = 'content-length' |
|
14 | 14 | HEADER_CONTENT_TYPE = 'content-type' |
|
15 | 15 | |
|
16 | 16 | FILE_DOWNLOAD_CHUNK_BYTES = 200000 |
|
17 | 17 | |
|
18 | 18 | REGEX_YOUTUBE_URL = re.compile(r'https?://((www\.)?youtube\.com/watch\?v=|youtu.be/)[-\w]+') |
|
19 | 19 | REGEX_MAGNET = re.compile(r'magnet:\?xt=urn:(btih:)?[a-z0-9]{20,50}.*') |
|
20 | 20 | |
|
21 | 21 | TYPE_URL_ONLY = ( |
|
22 | 22 | 'application/xhtml+xml', |
|
23 | 23 | 'text/html', |
|
24 | 24 | ) |
|
25 | 25 | |
|
26 | 26 | |
|
27 | 27 | class Downloader: |
|
28 | 28 | @staticmethod |
|
29 | 29 | def handles(url: str) -> bool: |
|
30 | 30 | return True |
|
31 | 31 | |
|
32 | 32 | @staticmethod |
|
33 | def download(url: str): | |
|
33 | def download(url: str, validate): | |
|
34 | 34 | # Verify content headers |
|
35 | 35 | response_head = requests.head(url, verify=False) |
|
36 | 36 | content_type = response_head.headers[HEADER_CONTENT_TYPE].split(';')[0] |
|
37 | if content_type in TYPE_URL_ONLY: | |
|
37 | if validate and content_type in TYPE_URL_ONLY: | |
|
38 | 38 | return None |
|
39 | 39 | |
|
40 | 40 | length_header = response_head.headers.get(HEADER_CONTENT_LENGTH) |
|
41 | if length_header: | |
|
41 | if validate and length_header: | |
|
42 | 42 | length = int(length_header) |
|
43 | 43 | validate_file_size(length) |
|
44 | 44 | # Get the actual content into memory |
|
45 | 45 | response = requests.get(url, verify=False, stream=True) |
|
46 | 46 | |
|
47 | 47 | if response.status_code == HTTP_RESULT_OK: |
|
48 | 48 | # Download file, stop if the size exceeds limit |
|
49 | 49 | size = 0 |
|
50 | 50 | |
|
51 | 51 | # Set a dummy file name that will be replaced |
|
52 | 52 | # anyway, just keep the valid extension |
|
53 | 53 | filename = 'file.' + content_type.split('/')[1] |
|
54 | 54 | |
|
55 | 55 | file = TemporaryUploadedFile(filename, content_type, 0, None, None) |
|
56 | 56 | for chunk in response.iter_content(FILE_DOWNLOAD_CHUNK_BYTES): |
|
57 | 57 | size += len(chunk) |
|
58 | 58 | validate_file_size(size) |
|
59 | 59 | file.write(chunk) |
|
60 | 60 | |
|
61 | 61 | return file |
|
62 | 62 | |
|
63 | 63 | |
|
64 | 64 | class YouTubeDownloader(Downloader): |
|
65 | 65 | @staticmethod |
|
66 | def download(url: str): | |
|
66 | def download(url: str, validate): | |
|
67 | 67 | yt = YouTube() |
|
68 | 68 | yt.from_url(url) |
|
69 | 69 | videos = yt.filter(YOUTUBE_VIDEO_FORMAT) |
|
70 | 70 | if len(videos) > 0: |
|
71 | 71 | video = videos[0] |
|
72 | 72 | return Downloader.download(video.url) |
|
73 | 73 | |
|
74 | 74 | @staticmethod |
|
75 | 75 | def handles(url: str) -> bool: |
|
76 | 76 | return REGEX_YOUTUBE_URL.match(url) is not None |
|
77 | 77 | |
|
78 | 78 | |
|
79 | 79 | class NothingDownloader(Downloader): |
|
80 | 80 | @staticmethod |
|
81 | 81 | def handles(url: str) -> bool: |
|
82 | 82 | return REGEX_MAGNET.match(url) |
|
83 | 83 | |
|
84 | 84 | @staticmethod |
|
85 | def download(url: str): | |
|
85 | def download(url: str, validate): | |
|
86 | 86 | return None |
|
87 | 87 | |
|
88 | 88 | |
|
89 | 89 | DOWNLOADERS = ( |
|
90 | 90 | YouTubeDownloader, |
|
91 | 91 | NothingDownloader, |
|
92 | 92 | Downloader, |
|
93 | 93 | ) |
|
94 | 94 | |
|
95 | 95 | |
|
96 | def download(url): | |
|
96 | def download(url, validate=True): | |
|
97 | 97 | for downloader in DOWNLOADERS: |
|
98 | 98 | if downloader.handles(url): |
|
99 | return downloader.download(url) | |
|
99 | return downloader.download(url, validate=validate) | |
|
100 | 100 | raise Exception('No downloader supports this URL.') |
|
101 | 101 |
@@ -1,364 +1,364 b'' | |||
|
1 | 1 | import uuid |
|
2 | 2 | import hashlib |
|
3 | 3 | import re |
|
4 | 4 | |
|
5 | 5 | from boards import settings |
|
6 | 6 | from boards.abstracts.tripcode import Tripcode |
|
7 | 7 | from boards.models import Attachment, KeyPair, GlobalId |
|
8 | 8 | from boards.models.attachment import FILE_TYPES_IMAGE |
|
9 | 9 | from boards.models.base import Viewable |
|
10 | 10 | from boards.models.post.export import get_exporter, DIFF_TYPE_JSON |
|
11 | 11 | from boards.models.post.manager import PostManager, NO_IP |
|
12 | 12 | from boards.utils import datetime_to_epoch |
|
13 | 13 | from django.core.exceptions import ObjectDoesNotExist |
|
14 | 14 | from django.core.urlresolvers import reverse |
|
15 | 15 | from django.db import models |
|
16 | 16 | from django.db.models import TextField, QuerySet, F |
|
17 | 17 | from django.template.defaultfilters import truncatewords, striptags |
|
18 | 18 | from django.template.loader import render_to_string |
|
19 | 19 | |
|
20 | 20 | CSS_CLS_HIDDEN_POST = 'hidden_post' |
|
21 | 21 | CSS_CLS_DEAD_POST = 'dead_post' |
|
22 | 22 | CSS_CLS_ARCHIVE_POST = 'archive_post' |
|
23 | 23 | CSS_CLS_POST = 'post' |
|
24 | 24 | CSS_CLS_MONOCHROME = 'monochrome' |
|
25 | 25 | |
|
26 | 26 | TITLE_MAX_WORDS = 10 |
|
27 | 27 | |
|
28 | 28 | APP_LABEL_BOARDS = 'boards' |
|
29 | 29 | |
|
30 | 30 | BAN_REASON_AUTO = 'Auto' |
|
31 | 31 | |
|
32 | 32 | TITLE_MAX_LENGTH = 200 |
|
33 | 33 | |
|
34 | 34 | REGEX_REPLY = re.compile(r'\[post\](\d+)\[/post\]') |
|
35 | 35 | REGEX_GLOBAL_REPLY = re.compile(r'\[post\](\w+)::([^:]+)::(\d+)\[/post\]') |
|
36 | 36 | REGEX_URL = re.compile(r'https?\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(/\S*)?') |
|
37 | 37 | REGEX_NOTIFICATION = re.compile(r'\[user\](\w+)\[/user\]') |
|
38 | 38 | |
|
39 | 39 | PARAMETER_TRUNCATED = 'truncated' |
|
40 | 40 | PARAMETER_TAG = 'tag' |
|
41 | 41 | PARAMETER_OFFSET = 'offset' |
|
42 | 42 | PARAMETER_DIFF_TYPE = 'type' |
|
43 | 43 | PARAMETER_CSS_CLASS = 'css_class' |
|
44 | 44 | PARAMETER_THREAD = 'thread' |
|
45 | 45 | PARAMETER_IS_OPENING = 'is_opening' |
|
46 | 46 | PARAMETER_POST = 'post' |
|
47 | 47 | PARAMETER_OP_ID = 'opening_post_id' |
|
48 | 48 | PARAMETER_NEED_OPEN_LINK = 'need_open_link' |
|
49 | 49 | PARAMETER_REPLY_LINK = 'reply_link' |
|
50 | 50 | PARAMETER_NEED_OP_DATA = 'need_op_data' |
|
51 | 51 | |
|
52 | 52 | POST_VIEW_PARAMS = ( |
|
53 | 53 | 'need_op_data', |
|
54 | 54 | 'reply_link', |
|
55 | 55 | 'need_open_link', |
|
56 | 56 | 'truncated', |
|
57 | 57 | 'mode_tree', |
|
58 | 58 | 'perms', |
|
59 | 59 | 'tree_depth', |
|
60 | 60 | ) |
|
61 | 61 | |
|
62 | 62 | |
|
63 | 63 | class Post(models.Model, Viewable): |
|
64 | 64 | """A post is a message.""" |
|
65 | 65 | |
|
66 | 66 | objects = PostManager() |
|
67 | 67 | |
|
68 | 68 | class Meta: |
|
69 | 69 | app_label = APP_LABEL_BOARDS |
|
70 | 70 | ordering = ('id',) |
|
71 | 71 | |
|
72 | 72 | title = models.CharField(max_length=TITLE_MAX_LENGTH, blank=True, default='') |
|
73 | 73 | pub_time = models.DateTimeField(db_index=True) |
|
74 | 74 | text = TextField(blank=True, default='') |
|
75 | 75 | _text_rendered = TextField(blank=True, null=True, editable=False) |
|
76 | 76 | |
|
77 | 77 | attachments = models.ManyToManyField(Attachment, null=True, blank=True, |
|
78 | 78 | related_name='attachment_posts') |
|
79 | 79 | |
|
80 | 80 | poster_ip = models.GenericIPAddressField() |
|
81 | 81 | |
|
82 | 82 | # Used for cache and threads updating |
|
83 | 83 | last_edit_time = models.DateTimeField() |
|
84 | 84 | |
|
85 | 85 | referenced_posts = models.ManyToManyField('Post', symmetrical=False, |
|
86 | 86 | null=True, |
|
87 | 87 | blank=True, related_name='refposts', |
|
88 | 88 | db_index=True) |
|
89 | 89 | refmap = models.TextField(null=True, blank=True) |
|
90 | 90 | thread = models.ForeignKey('Thread', db_index=True, related_name='replies') |
|
91 | 91 | |
|
92 | 92 | url = models.TextField() |
|
93 |
uid = models.TextField( |
|
|
93 | uid = models.TextField() | |
|
94 | 94 | |
|
95 | 95 | # Global ID with author key. If the message was downloaded from another |
|
96 | 96 | # server, this indicates the server. |
|
97 | 97 | global_id = models.OneToOneField(GlobalId, null=True, blank=True, |
|
98 | 98 | on_delete=models.CASCADE) |
|
99 | 99 | |
|
100 | 100 | tripcode = models.CharField(max_length=50, blank=True, default='') |
|
101 | 101 | opening = models.BooleanField(db_index=True) |
|
102 | 102 | hidden = models.BooleanField(default=False) |
|
103 | 103 | version = models.IntegerField(default=1) |
|
104 | 104 | |
|
105 | 105 | def __str__(self): |
|
106 | 106 | return 'P#{}/{}'.format(self.id, self.get_title()) |
|
107 | 107 | |
|
108 | 108 | def get_title(self) -> str: |
|
109 | 109 | return self.title |
|
110 | 110 | |
|
111 | 111 | def get_title_or_text(self): |
|
112 | 112 | title = self.get_title() |
|
113 | 113 | if not title: |
|
114 | 114 | title = truncatewords(striptags(self.get_text()), TITLE_MAX_WORDS) |
|
115 | 115 | |
|
116 | 116 | return title |
|
117 | 117 | |
|
118 | 118 | def build_refmap(self, excluded_ids=None) -> None: |
|
119 | 119 | """ |
|
120 | 120 | Builds a replies map string from replies list. This is a cache to stop |
|
121 | 121 | the server from recalculating the map on every post show. |
|
122 | 122 | """ |
|
123 | 123 | |
|
124 | 124 | replies = self.referenced_posts |
|
125 | 125 | if excluded_ids is not None: |
|
126 | 126 | replies = replies.exclude(id__in=excluded_ids) |
|
127 | 127 | else: |
|
128 | 128 | replies = replies.all() |
|
129 | 129 | |
|
130 | 130 | post_urls = [refpost.get_link_view() for refpost in replies] |
|
131 | 131 | |
|
132 | 132 | self.refmap = ', '.join(post_urls) |
|
133 | 133 | |
|
134 | 134 | def is_referenced(self) -> bool: |
|
135 | 135 | return self.refmap and len(self.refmap) > 0 |
|
136 | 136 | |
|
137 | 137 | def is_opening(self) -> bool: |
|
138 | 138 | """ |
|
139 | 139 | Checks if this is an opening post or just a reply. |
|
140 | 140 | """ |
|
141 | 141 | |
|
142 | 142 | return self.opening |
|
143 | 143 | |
|
144 | 144 | def get_absolute_url(self, thread=None): |
|
145 | 145 | # Url is cached only for the "main" thread. When getting url |
|
146 | 146 | # for other threads, do it manually. |
|
147 | 147 | return self.url |
|
148 | 148 | |
|
149 | 149 | def get_thread(self): |
|
150 | 150 | return self.thread |
|
151 | 151 | |
|
152 | 152 | def get_thread_id(self): |
|
153 | 153 | return self.thread_id |
|
154 | 154 | |
|
155 | 155 | def _get_cache_key(self): |
|
156 | 156 | return [datetime_to_epoch(self.last_edit_time)] |
|
157 | 157 | |
|
158 | 158 | def get_view_params(self, *args, **kwargs): |
|
159 | 159 | """ |
|
160 | 160 | Gets the parameters required for viewing the post based on the arguments |
|
161 | 161 | given and the post itself. |
|
162 | 162 | """ |
|
163 | 163 | thread = kwargs.get('thread') or self.get_thread() |
|
164 | 164 | |
|
165 | 165 | css_classes = [CSS_CLS_POST] |
|
166 | 166 | if thread.is_archived(): |
|
167 | 167 | css_classes.append(CSS_CLS_ARCHIVE_POST) |
|
168 | 168 | elif not thread.can_bump(): |
|
169 | 169 | css_classes.append(CSS_CLS_DEAD_POST) |
|
170 | 170 | if self.is_hidden(): |
|
171 | 171 | css_classes.append(CSS_CLS_HIDDEN_POST) |
|
172 | 172 | if thread.is_monochrome(): |
|
173 | 173 | css_classes.append(CSS_CLS_MONOCHROME) |
|
174 | 174 | |
|
175 | 175 | params = dict() |
|
176 | 176 | for param in POST_VIEW_PARAMS: |
|
177 | 177 | if param in kwargs: |
|
178 | 178 | params[param] = kwargs[param] |
|
179 | 179 | |
|
180 | 180 | params.update({ |
|
181 | 181 | PARAMETER_POST: self, |
|
182 | 182 | PARAMETER_IS_OPENING: self.is_opening(), |
|
183 | 183 | PARAMETER_THREAD: thread, |
|
184 | 184 | PARAMETER_CSS_CLASS: ' '.join(css_classes), |
|
185 | 185 | }) |
|
186 | 186 | |
|
187 | 187 | return params |
|
188 | 188 | |
|
189 | 189 | def get_view(self, *args, **kwargs) -> str: |
|
190 | 190 | """ |
|
191 | 191 | Renders post's HTML view. Some of the post params can be passed over |
|
192 | 192 | kwargs for the means of caching (if we view the thread, some params |
|
193 | 193 | are same for every post and don't need to be computed over and over. |
|
194 | 194 | """ |
|
195 | 195 | params = self.get_view_params(*args, **kwargs) |
|
196 | 196 | |
|
197 | 197 | return render_to_string('boards/post.html', params) |
|
198 | 198 | |
|
199 | 199 | def get_images(self) -> Attachment: |
|
200 | 200 | return self.attachments.filter(mimetype__in=FILE_TYPES_IMAGE) |
|
201 | 201 | |
|
202 | 202 | def get_first_image(self) -> Attachment: |
|
203 | 203 | try: |
|
204 | 204 | return self.get_images().earliest('-id') |
|
205 | 205 | except Attachment.DoesNotExist: |
|
206 | 206 | return None |
|
207 | 207 | |
|
208 | 208 | def set_global_id(self, key_pair=None): |
|
209 | 209 | """ |
|
210 | 210 | Sets global id based on the given key pair. If no key pair is given, |
|
211 | 211 | default one is used. |
|
212 | 212 | """ |
|
213 | 213 | |
|
214 | 214 | if key_pair: |
|
215 | 215 | key = key_pair |
|
216 | 216 | else: |
|
217 | 217 | try: |
|
218 | 218 | key = KeyPair.objects.get(primary=True) |
|
219 | 219 | except KeyPair.DoesNotExist: |
|
220 | 220 | # Do not update the global id because there is no key defined |
|
221 | 221 | return |
|
222 | 222 | global_id = GlobalId(key_type=key.key_type, |
|
223 | 223 | key=key.public_key, |
|
224 | 224 | local_id=self.id) |
|
225 | 225 | global_id.save() |
|
226 | 226 | |
|
227 | 227 | self.global_id = global_id |
|
228 | 228 | |
|
229 | 229 | self.save(update_fields=['global_id']) |
|
230 | 230 | |
|
231 | 231 | def get_pub_time_str(self): |
|
232 | 232 | return str(self.pub_time) |
|
233 | 233 | |
|
234 | 234 | def get_replied_ids(self): |
|
235 | 235 | """ |
|
236 | 236 | Gets ID list of the posts that this post replies. |
|
237 | 237 | """ |
|
238 | 238 | |
|
239 | 239 | raw_text = self.get_raw_text() |
|
240 | 240 | |
|
241 | 241 | local_replied = REGEX_REPLY.findall(raw_text) |
|
242 | 242 | global_replied = [] |
|
243 | 243 | for match in REGEX_GLOBAL_REPLY.findall(raw_text): |
|
244 | 244 | key_type = match[0] |
|
245 | 245 | key = match[1] |
|
246 | 246 | local_id = match[2] |
|
247 | 247 | |
|
248 | 248 | try: |
|
249 | 249 | global_id = GlobalId.objects.get(key_type=key_type, |
|
250 | 250 | key=key, local_id=local_id) |
|
251 | 251 | for post in Post.objects.filter(global_id=global_id).only('id'): |
|
252 | 252 | global_replied.append(post.id) |
|
253 | 253 | except GlobalId.DoesNotExist: |
|
254 | 254 | pass |
|
255 | 255 | return local_replied + global_replied |
|
256 | 256 | |
|
257 | 257 | def get_post_data(self, format_type=DIFF_TYPE_JSON, request=None, |
|
258 | 258 | include_last_update=False) -> str: |
|
259 | 259 | """ |
|
260 | 260 | Gets post HTML or JSON data that can be rendered on a page or used by |
|
261 | 261 | API. |
|
262 | 262 | """ |
|
263 | 263 | |
|
264 | 264 | return get_exporter(format_type).export(self, request, |
|
265 | 265 | include_last_update) |
|
266 | 266 | |
|
267 | 267 | def _build_url(self): |
|
268 | 268 | opening = self.is_opening() |
|
269 | 269 | opening_id = self.id if opening else self.get_thread().get_opening_post_id() |
|
270 | 270 | url = reverse('thread', kwargs={'post_id': opening_id}) |
|
271 | 271 | if not opening: |
|
272 | 272 | url += '#' + str(self.id) |
|
273 | 273 | |
|
274 | 274 | return url |
|
275 | 275 | |
|
276 | 276 | def save(self, force_insert=False, force_update=False, using=None, |
|
277 | 277 | update_fields=None): |
|
278 | 278 | new_post = self.id is None |
|
279 | 279 | |
|
280 | 280 | self.uid = str(uuid.uuid4()) |
|
281 | 281 | if update_fields is not None and 'uid' not in update_fields: |
|
282 | 282 | update_fields += ['uid'] |
|
283 | 283 | |
|
284 | 284 | if not new_post: |
|
285 | 285 | thread = self.get_thread() |
|
286 | 286 | if thread: |
|
287 | 287 | thread.last_edit_time = self.last_edit_time |
|
288 | 288 | thread.save(update_fields=['last_edit_time', 'status']) |
|
289 | 289 | |
|
290 | 290 | super().save(force_insert, force_update, using, update_fields) |
|
291 | 291 | |
|
292 | 292 | if new_post: |
|
293 | 293 | self.url = self._build_url() |
|
294 | 294 | super().save(update_fields=['url']) |
|
295 | 295 | |
|
296 | 296 | def get_text(self) -> str: |
|
297 | 297 | return self._text_rendered |
|
298 | 298 | |
|
299 | 299 | def get_raw_text(self) -> str: |
|
300 | 300 | return self.text |
|
301 | 301 | |
|
302 | 302 | def get_sync_text(self) -> str: |
|
303 | 303 | """ |
|
304 | 304 | Returns text applicable for sync. It has absolute post reflinks. |
|
305 | 305 | """ |
|
306 | 306 | |
|
307 | 307 | replacements = dict() |
|
308 | 308 | for post_id in REGEX_REPLY.findall(self.get_raw_text()): |
|
309 | 309 | try: |
|
310 | 310 | absolute_post_id = str(Post.objects.get(id=post_id).global_id) |
|
311 | 311 | replacements[post_id] = absolute_post_id |
|
312 | 312 | except Post.DoesNotExist: |
|
313 | 313 | pass |
|
314 | 314 | |
|
315 | 315 | text = self.get_raw_text() or '' |
|
316 | 316 | for key in replacements: |
|
317 | 317 | text = text.replace('[post]{}[/post]'.format(key), |
|
318 | 318 | '[post]{}[/post]'.format(replacements[key])) |
|
319 | 319 | text = text.replace('\r\n', '\n').replace('\r', '\n') |
|
320 | 320 | |
|
321 | 321 | return text |
|
322 | 322 | |
|
323 | 323 | def get_tripcode(self): |
|
324 | 324 | if self.tripcode: |
|
325 | 325 | return Tripcode(self.tripcode) |
|
326 | 326 | |
|
327 | 327 | def get_link_view(self): |
|
328 | 328 | """ |
|
329 | 329 | Gets view of a reflink to the post. |
|
330 | 330 | """ |
|
331 | 331 | result = '<a href="{}">>>{}</a>'.format(self.get_absolute_url(), |
|
332 | 332 | self.id) |
|
333 | 333 | if self.is_opening(): |
|
334 | 334 | result = '<b>{}</b>'.format(result) |
|
335 | 335 | |
|
336 | 336 | return result |
|
337 | 337 | |
|
338 | 338 | def is_hidden(self) -> bool: |
|
339 | 339 | return self.hidden |
|
340 | 340 | |
|
341 | 341 | def set_hidden(self, hidden): |
|
342 | 342 | self.hidden = hidden |
|
343 | 343 | |
|
344 | 344 | def increment_version(self): |
|
345 | 345 | self.version = F('version') + 1 |
|
346 | 346 | |
|
347 | 347 | def clear_cache(self): |
|
348 | 348 | """ |
|
349 | 349 | Clears sync data (content cache, signatures etc). |
|
350 | 350 | """ |
|
351 | 351 | global_id = self.global_id |
|
352 | 352 | if global_id is not None and global_id.is_local()\ |
|
353 | 353 | and global_id.content is not None: |
|
354 | 354 | global_id.clear_cache() |
|
355 | 355 | |
|
356 | 356 | def get_tags(self): |
|
357 | 357 | return self.get_thread().get_tags() |
|
358 | 358 | |
|
359 | 359 | def get_ip_color(self): |
|
360 | 360 | return hashlib.md5(self.poster_ip.encode()).hexdigest()[:6] |
|
361 | 361 | |
|
362 | 362 | def has_ip(self): |
|
363 | 363 | return self.poster_ip != NO_IP |
|
364 | 364 |
@@ -1,337 +1,345 b'' | |||
|
1 | 1 | import xml.etree.ElementTree as et |
|
2 | 2 | import logging |
|
3 | from xml.etree import ElementTree | |
|
3 | 4 | |
|
4 | 5 | from boards.abstracts.exceptions import SyncException |
|
6 | from boards.abstracts.sync_filters import ThreadFilter | |
|
5 | 7 | from boards.models import KeyPair, GlobalId, Signature, Post, Tag |
|
6 | 8 | from boards.models.attachment.downloaders import download |
|
9 | from boards.models.signature import TAG_REQUEST, ATTR_TYPE, TYPE_GET, \ | |
|
10 | ATTR_VERSION, TAG_MODEL, ATTR_NAME, TAG_ID, TYPE_LIST | |
|
7 | 11 | from boards.utils import get_file_mimetype, get_file_hash |
|
8 | 12 | from django.db import transaction |
|
9 | 13 | from django import forms |
|
10 | 14 | |
|
11 | 15 | EXCEPTION_NODE = 'Sync node returned an error: {}.' |
|
12 | 16 | EXCEPTION_DOWNLOAD = 'File was not downloaded.' |
|
13 | 17 | EXCEPTION_HASH = 'File hash does not match attachment hash.' |
|
14 | 18 | EXCEPTION_SIGNATURE = 'Invalid model signature for {}.' |
|
15 | 19 | EXCEPTION_AUTHOR_SIGNATURE = 'Model {} has no author signature.' |
|
16 | 20 | ENCODING_UNICODE = 'unicode' |
|
17 | 21 | |
|
18 | 22 | TAG_MODEL = 'model' |
|
19 | 23 | TAG_REQUEST = 'request' |
|
20 | 24 | TAG_RESPONSE = 'response' |
|
21 | 25 | TAG_ID = 'id' |
|
22 | 26 | TAG_STATUS = 'status' |
|
23 | 27 | TAG_MODELS = 'models' |
|
24 | 28 | TAG_TITLE = 'title' |
|
25 | 29 | TAG_TEXT = 'text' |
|
26 | 30 | TAG_THREAD = 'thread' |
|
27 | 31 | TAG_PUB_TIME = 'pub-time' |
|
28 | 32 | TAG_SIGNATURES = 'signatures' |
|
29 | 33 | TAG_SIGNATURE = 'signature' |
|
30 | 34 | TAG_CONTENT = 'content' |
|
31 | 35 | TAG_ATTACHMENTS = 'attachments' |
|
32 | 36 | TAG_ATTACHMENT = 'attachment' |
|
33 | 37 | TAG_TAGS = 'tags' |
|
34 | 38 | TAG_TAG = 'tag' |
|
35 | 39 | TAG_ATTACHMENT_REFS = 'attachment-refs' |
|
36 | 40 | TAG_ATTACHMENT_REF = 'attachment-ref' |
|
37 | 41 | TAG_TRIPCODE = 'tripcode' |
|
38 | 42 | TAG_VERSION = 'version' |
|
39 | 43 | |
|
40 | 44 | TYPE_GET = 'get' |
|
41 | 45 | |
|
42 | 46 | ATTR_VERSION = 'version' |
|
43 | 47 | ATTR_TYPE = 'type' |
|
44 | 48 | ATTR_NAME = 'name' |
|
45 | 49 | ATTR_VALUE = 'value' |
|
46 | 50 | ATTR_MIMETYPE = 'mimetype' |
|
47 | 51 | ATTR_KEY = 'key' |
|
48 | 52 | ATTR_REF = 'ref' |
|
49 | 53 | ATTR_URL = 'url' |
|
50 | 54 | ATTR_ID_TYPE = 'id-type' |
|
51 | 55 | |
|
52 | 56 | ID_TYPE_MD5 = 'md5' |
|
53 | 57 | ID_TYPE_URL = 'url' |
|
54 | 58 | |
|
55 | 59 | STATUS_SUCCESS = 'success' |
|
56 | 60 | |
|
57 | 61 | |
|
58 | 62 | logger = logging.getLogger('boards.sync') |
|
59 | 63 | |
|
60 | 64 | |
|
61 | 65 | class SyncManager: |
|
62 | 66 | @staticmethod |
|
63 | 67 | def generate_response_get(model_list: list): |
|
64 | 68 | response = et.Element(TAG_RESPONSE) |
|
65 | 69 | |
|
66 | 70 | status = et.SubElement(response, TAG_STATUS) |
|
67 | 71 | status.text = STATUS_SUCCESS |
|
68 | 72 | |
|
69 | 73 | models = et.SubElement(response, TAG_MODELS) |
|
70 | 74 | |
|
71 | 75 | for post in model_list: |
|
72 | 76 | model = et.SubElement(models, TAG_MODEL) |
|
73 | 77 | model.set(ATTR_NAME, 'post') |
|
74 | 78 | |
|
75 | 79 | global_id = post.global_id |
|
76 | 80 | |
|
77 | 81 | attachments = post.attachments.all() |
|
78 | 82 | if global_id.content: |
|
79 | 83 | model.append(et.fromstring(global_id.content)) |
|
80 | 84 | if len(attachments) > 0: |
|
81 | 85 | internal_attachments = False |
|
82 | 86 | for attachment in attachments: |
|
83 | 87 | if attachment.is_internal(): |
|
84 | 88 | internal_attachments = True |
|
85 | 89 | break |
|
86 | 90 | |
|
87 | 91 | if internal_attachments: |
|
88 | 92 | attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS) |
|
89 | 93 | for file in attachments: |
|
90 | 94 | SyncManager._attachment_to_xml( |
|
91 | 95 | None, attachment_refs, file) |
|
92 | 96 | else: |
|
93 | 97 | content_tag = et.SubElement(model, TAG_CONTENT) |
|
94 | 98 | |
|
95 | 99 | tag_id = et.SubElement(content_tag, TAG_ID) |
|
96 | 100 | global_id.to_xml_element(tag_id) |
|
97 | 101 | |
|
98 | 102 | title = et.SubElement(content_tag, TAG_TITLE) |
|
99 | 103 | title.text = post.title |
|
100 | 104 | |
|
101 | 105 | text = et.SubElement(content_tag, TAG_TEXT) |
|
102 | 106 | text.text = post.get_sync_text() |
|
103 | 107 | |
|
104 | 108 | thread = post.get_thread() |
|
105 | 109 | if post.is_opening(): |
|
106 | 110 | tag_tags = et.SubElement(content_tag, TAG_TAGS) |
|
107 | 111 | for tag in thread.get_tags(): |
|
108 | 112 | tag_tag = et.SubElement(tag_tags, TAG_TAG) |
|
109 | 113 | tag_tag.text = tag.name |
|
110 | 114 | else: |
|
111 | 115 | tag_thread = et.SubElement(content_tag, TAG_THREAD) |
|
112 | 116 | thread_id = et.SubElement(tag_thread, TAG_ID) |
|
113 | 117 | thread.get_opening_post().global_id.to_xml_element(thread_id) |
|
114 | 118 | |
|
115 | 119 | pub_time = et.SubElement(content_tag, TAG_PUB_TIME) |
|
116 | 120 | pub_time.text = str(post.get_pub_time_str()) |
|
117 | 121 | |
|
118 | 122 | if post.tripcode: |
|
119 | 123 | tripcode = et.SubElement(content_tag, TAG_TRIPCODE) |
|
120 | 124 | tripcode.text = post.tripcode |
|
121 | 125 | |
|
122 | 126 | if len(attachments) > 0: |
|
123 | 127 | attachments_tag = et.SubElement(content_tag, TAG_ATTACHMENTS) |
|
124 | 128 | |
|
125 | 129 | internal_attachments = False |
|
126 | 130 | for attachment in attachments: |
|
127 | 131 | if attachment.is_internal(): |
|
128 | 132 | internal_attachments = True |
|
129 | 133 | break |
|
130 | 134 | |
|
131 | 135 | if internal_attachments: |
|
132 | 136 | attachment_refs = et.SubElement(model, TAG_ATTACHMENT_REFS) |
|
133 | 137 | else: |
|
134 | 138 | attachment_refs = None |
|
135 | 139 | |
|
136 | 140 | for file in attachments: |
|
137 | 141 | SyncManager._attachment_to_xml( |
|
138 | 142 | attachments_tag, attachment_refs, file) |
|
139 | 143 | version_tag = et.SubElement(content_tag, TAG_VERSION) |
|
140 | 144 | version_tag.text = str(post.version) |
|
141 | 145 | |
|
142 | 146 | global_id.content = et.tostring(content_tag, ENCODING_UNICODE) |
|
143 | 147 | global_id.save() |
|
144 | 148 | |
|
145 | 149 | signatures_tag = et.SubElement(model, TAG_SIGNATURES) |
|
146 | 150 | post_signatures = global_id.signature_set.all() |
|
147 | 151 | if post_signatures: |
|
148 | 152 | signatures = post_signatures |
|
149 | 153 | else: |
|
150 | 154 | key = KeyPair.objects.get(public_key=global_id.key) |
|
151 | 155 | signature = Signature( |
|
152 | 156 | key_type=key.key_type, |
|
153 | 157 | key=key.public_key, |
|
154 | 158 | signature=key.sign(global_id.content), |
|
155 | 159 | global_id=global_id, |
|
156 | 160 | ) |
|
157 | 161 | signature.save() |
|
158 | 162 | signatures = [signature] |
|
159 | 163 | for signature in signatures: |
|
160 | 164 | signature_tag = et.SubElement(signatures_tag, TAG_SIGNATURE) |
|
161 | 165 | signature_tag.set(ATTR_TYPE, signature.key_type) |
|
162 | 166 | signature_tag.set(ATTR_VALUE, signature.signature) |
|
163 | 167 | signature_tag.set(ATTR_KEY, signature.key) |
|
164 | 168 | |
|
165 | 169 | return et.tostring(response, ENCODING_UNICODE) |
|
166 | 170 | |
|
167 | 171 | @staticmethod |
|
168 | 172 | def parse_response_get(response_xml, hostname): |
|
169 | 173 | tag_root = et.fromstring(response_xml) |
|
170 | 174 | tag_status = tag_root.find(TAG_STATUS) |
|
171 | 175 | if STATUS_SUCCESS == tag_status.text: |
|
172 | 176 | tag_models = tag_root.find(TAG_MODELS) |
|
173 | 177 | for tag_model in tag_models: |
|
174 | 178 | SyncManager.parse_post(tag_model, hostname) |
|
175 | 179 | else: |
|
176 | 180 | raise SyncException(EXCEPTION_NODE.format(tag_status.text)) |
|
177 | 181 | |
|
178 | 182 | @staticmethod |
|
179 | 183 | @transaction.atomic |
|
180 | 184 | def parse_post(tag_model, hostname): |
|
181 | 185 | tag_content = tag_model.find(TAG_CONTENT) |
|
182 | 186 | |
|
183 | 187 | content_str = et.tostring(tag_content, ENCODING_UNICODE) |
|
184 | 188 | |
|
185 | 189 | tag_id = tag_content.find(TAG_ID) |
|
186 | 190 | global_id, exists = GlobalId.from_xml_element(tag_id) |
|
187 | 191 | signatures = SyncManager._verify_model(global_id, content_str, tag_model) |
|
188 | 192 | |
|
189 | 193 | version = int(tag_content.find(TAG_VERSION).text) |
|
190 | 194 | is_old = exists and global_id.post.version < version |
|
191 | 195 | if exists and not is_old: |
|
192 | 196 | print('Post with same ID exists and is up to date.') |
|
193 | 197 | else: |
|
194 | 198 | global_id.content = content_str |
|
195 | 199 | global_id.save() |
|
196 | 200 | for signature in signatures: |
|
197 | 201 | signature.global_id = global_id |
|
198 | 202 | signature.save() |
|
199 | 203 | |
|
200 | 204 | title = tag_content.find(TAG_TITLE).text or '' |
|
201 | 205 | text = tag_content.find(TAG_TEXT).text or '' |
|
202 | 206 | pub_time = tag_content.find(TAG_PUB_TIME).text |
|
203 | 207 | tripcode_tag = tag_content.find(TAG_TRIPCODE) |
|
204 | 208 | if tripcode_tag is not None: |
|
205 | 209 | tripcode = tripcode_tag.text or '' |
|
206 | 210 | else: |
|
207 | 211 | tripcode = '' |
|
208 | 212 | |
|
209 | 213 | thread = tag_content.find(TAG_THREAD) |
|
210 | 214 | tags = [] |
|
211 | 215 | if thread: |
|
212 | 216 | thread_id = thread.find(TAG_ID) |
|
213 | 217 | op_global_id, exists = GlobalId.from_xml_element(thread_id) |
|
214 | 218 | if exists: |
|
215 | 219 | opening_post = Post.objects.get(global_id=op_global_id) |
|
216 | 220 | else: |
|
217 | 221 | logger.debug('No thread exists for post {}'.format(global_id)) |
|
218 | 222 | else: |
|
219 | 223 | opening_post = None |
|
220 | 224 | tag_tags = tag_content.find(TAG_TAGS) |
|
221 | 225 | for tag_tag in tag_tags: |
|
222 | 226 | tag, created = Tag.objects.get_or_create( |
|
223 | 227 | name=tag_tag.text) |
|
224 | 228 | tags.append(tag) |
|
225 | 229 | |
|
226 | 230 | # TODO Check that the replied posts are already present |
|
227 | 231 | # before adding new ones |
|
228 | 232 | |
|
229 | 233 | files = [] |
|
230 | 234 | urls = [] |
|
231 | 235 | tag_attachments = tag_content.find(TAG_ATTACHMENTS) or list() |
|
232 | 236 | tag_refs = tag_model.find(TAG_ATTACHMENT_REFS) |
|
233 | 237 | for attachment in tag_attachments: |
|
234 | 238 | if attachment.get(ATTR_ID_TYPE) == ID_TYPE_URL: |
|
235 | 239 | urls.append(attachment.text) |
|
236 | 240 | else: |
|
237 | 241 | tag_ref = tag_refs.find("{}[@ref='{}']".format( |
|
238 | 242 | TAG_ATTACHMENT_REF, attachment.text)) |
|
239 | 243 | url = tag_ref.get(ATTR_URL) |
|
240 | 244 | try: |
|
241 | attached_file = download(hostname + url) | |
|
245 | attached_file = download(hostname + url, validate=False) | |
|
242 | 246 | |
|
243 | 247 | if attached_file is None: |
|
244 | 248 | raise SyncException(EXCEPTION_DOWNLOAD) |
|
245 | 249 | |
|
246 | 250 | hash = get_file_hash(attached_file) |
|
247 | 251 | if hash != attachment.text: |
|
248 | 252 | raise SyncException(EXCEPTION_HASH) |
|
249 | 253 | |
|
250 | 254 | files.append(attached_file) |
|
251 | 255 | except forms.ValidationError: |
|
252 | 256 | urls.append(hostname+url) |
|
253 | 257 | |
|
254 | 258 | |
|
255 | 259 | if is_old: |
|
256 | 260 | post = global_id.post |
|
257 | 261 | Post.objects.update_post( |
|
258 | 262 | post, title=title, text=text, pub_time=pub_time, |
|
259 | 263 | tags=tags, files=files, file_urls=urls, |
|
260 | 264 | tripcode=tripcode, version=version) |
|
261 | 265 | logger.debug('Parsed updated post {}'.format(global_id)) |
|
262 | 266 | else: |
|
263 | 267 | Post.objects.import_post( |
|
264 | 268 | title=title, text=text, pub_time=pub_time, |
|
265 | 269 | opening_post=opening_post, tags=tags, |
|
266 | 270 | global_id=global_id, files=files, |
|
267 | 271 | file_urls=urls, tripcode=tripcode, |
|
268 | 272 | version=version) |
|
269 | 273 | logger.debug('Parsed new post {}'.format(global_id)) |
|
270 | 274 | |
|
271 | 275 | @staticmethod |
|
272 | def generate_response_list(): | |
|
276 | def generate_response_list(filters): | |
|
273 | 277 | response = et.Element(TAG_RESPONSE) |
|
274 | 278 | |
|
275 | 279 | status = et.SubElement(response, TAG_STATUS) |
|
276 | 280 | status.text = STATUS_SUCCESS |
|
277 | 281 | |
|
278 | 282 | models = et.SubElement(response, TAG_MODELS) |
|
279 | 283 | |
|
280 |
|
|
|
284 | posts = Post.objects.prefetch_related('global_id') | |
|
285 | for post_filter in filters: | |
|
286 | posts = post_filter.filter(posts) | |
|
287 | ||
|
288 | for post in posts: | |
|
281 | 289 | tag_model = et.SubElement(models, TAG_MODEL) |
|
282 | 290 | tag_id = et.SubElement(tag_model, TAG_ID) |
|
283 | 291 | post.global_id.to_xml_element(tag_id) |
|
284 | 292 | tag_version = et.SubElement(tag_model, TAG_VERSION) |
|
285 | 293 | tag_version.text = str(post.version) |
|
286 | 294 | |
|
287 | 295 | return et.tostring(response, ENCODING_UNICODE) |
|
288 | 296 | |
|
289 | 297 | @staticmethod |
|
290 | 298 | def _verify_model(global_id, content_str, tag_model): |
|
291 | 299 | """ |
|
292 | 300 | Verifies all signatures for a single model. |
|
293 | 301 | """ |
|
294 | 302 | |
|
295 | 303 | signatures = [] |
|
296 | 304 | |
|
297 | 305 | tag_signatures = tag_model.find(TAG_SIGNATURES) |
|
298 | 306 | has_author_signature = False |
|
299 | 307 | for tag_signature in tag_signatures: |
|
300 | 308 | signature_type = tag_signature.get(ATTR_TYPE) |
|
301 | 309 | signature_value = tag_signature.get(ATTR_VALUE) |
|
302 | 310 | signature_key = tag_signature.get(ATTR_KEY) |
|
303 | 311 | |
|
304 | 312 | if global_id.key_type == signature_type and\ |
|
305 | 313 | global_id.key == signature_key: |
|
306 | 314 | has_author_signature = True |
|
307 | 315 | |
|
308 | 316 | signature = Signature(key_type=signature_type, |
|
309 | 317 | key=signature_key, |
|
310 | 318 | signature=signature_value) |
|
311 | 319 | |
|
312 | 320 | if not KeyPair.objects.verify(signature, content_str): |
|
313 | 321 | raise SyncException(EXCEPTION_SIGNATURE.format(content_str)) |
|
314 | 322 | |
|
315 | 323 | signatures.append(signature) |
|
316 | 324 | if not has_author_signature: |
|
317 | 325 | raise SyncException(EXCEPTION_AUTHOR_SIGNATURE.format(content_str)) |
|
318 | 326 | |
|
319 | 327 | return signatures |
|
320 | 328 | |
|
321 | 329 | @staticmethod |
|
322 | 330 | def _attachment_to_xml(tag_attachments, tag_refs, attachment): |
|
323 | 331 | if tag_attachments is not None: |
|
324 | 332 | attachment_tag = et.SubElement(tag_attachments, TAG_ATTACHMENT) |
|
325 | 333 | if attachment.is_internal(): |
|
326 | 334 | mimetype = get_file_mimetype(attachment.file.file) |
|
327 | 335 | attachment_tag.set(ATTR_MIMETYPE, mimetype) |
|
328 | 336 | attachment_tag.set(ATTR_ID_TYPE, ID_TYPE_MD5) |
|
329 | 337 | attachment_tag.text = attachment.hash |
|
330 | 338 | else: |
|
331 | 339 | attachment_tag.set(ATTR_ID_TYPE, ID_TYPE_URL) |
|
332 | 340 | attachment_tag.text = attachment.url |
|
333 | 341 | |
|
334 | 342 | if tag_refs is not None: |
|
335 | 343 | attachment_ref = et.SubElement(tag_refs, TAG_ATTACHMENT_REF) |
|
336 | 344 | attachment_ref.set(ATTR_REF, attachment.hash) |
|
337 | attachment_ref.set(ATTR_URL, attachment.file.url) | |
|
345 | attachment_ref.set(ATTR_URL, attachment.file.url) |
@@ -1,158 +1,125 b'' | |||
|
1 | 1 | import xml.etree.ElementTree as et |
|
2 | ||
|
2 | 3 | from django.db import models |
|
4 | ||
|
3 | 5 | from boards.models import KeyPair |
|
4 | 6 | |
|
5 | ||
|
6 | 7 | TAG_MODEL = 'model' |
|
7 | 8 | TAG_REQUEST = 'request' |
|
8 | 9 | TAG_ID = 'id' |
|
9 | 10 | |
|
10 | 11 | TYPE_GET = 'get' |
|
11 | 12 | TYPE_LIST = 'list' |
|
12 | 13 | |
|
13 | 14 | ATTR_VERSION = 'version' |
|
14 | 15 | ATTR_TYPE = 'type' |
|
15 | 16 | ATTR_NAME = 'name' |
|
16 | 17 | |
|
17 | 18 | ATTR_KEY = 'key' |
|
18 | 19 | ATTR_KEY_TYPE = 'type' |
|
19 | 20 | ATTR_LOCAL_ID = 'local-id' |
|
20 | 21 | |
|
21 | 22 | |
|
22 | 23 | class GlobalIdManager(models.Manager): |
|
23 | def generate_request_get(self, global_id_list: list): | |
|
24 | """ | |
|
25 | Form a get request from a list of ModelId objects. | |
|
26 | """ | |
|
27 | ||
|
28 | request = et.Element(TAG_REQUEST) | |
|
29 | request.set(ATTR_TYPE, TYPE_GET) | |
|
30 | request.set(ATTR_VERSION, '1.0') | |
|
31 | ||
|
32 | model = et.SubElement(request, TAG_MODEL) | |
|
33 | model.set(ATTR_VERSION, '1.0') | |
|
34 | model.set(ATTR_NAME, 'post') | |
|
35 | ||
|
36 | for global_id in global_id_list: | |
|
37 | tag_id = et.SubElement(model, TAG_ID) | |
|
38 | global_id.to_xml_element(tag_id) | |
|
39 | ||
|
40 | return et.tostring(request, 'unicode') | |
|
41 | ||
|
42 | def generate_request_list(self): | |
|
43 | """ | |
|
44 | Form a pull request from a list of ModelId objects. | |
|
45 | """ | |
|
46 | ||
|
47 | request = et.Element(TAG_REQUEST) | |
|
48 | request.set(ATTR_TYPE, TYPE_LIST) | |
|
49 | request.set(ATTR_VERSION, '1.0') | |
|
50 | ||
|
51 | model = et.SubElement(request, TAG_MODEL) | |
|
52 | model.set(ATTR_VERSION, '1.0') | |
|
53 | model.set(ATTR_NAME, 'post') | |
|
54 | ||
|
55 | return et.tostring(request, 'unicode') | |
|
56 | ||
|
57 | 24 | def global_id_exists(self, global_id): |
|
58 | 25 | """ |
|
59 | 26 | Checks if the same global id already exists in the system. |
|
60 | 27 | """ |
|
61 | 28 | |
|
62 | 29 | return self.filter(key=global_id.key, |
|
63 | 30 | key_type=global_id.key_type, |
|
64 | 31 | local_id=global_id.local_id).exists() |
|
65 | 32 | |
|
66 | 33 | |
|
67 | 34 | class GlobalId(models.Model): |
|
68 | 35 | """ |
|
69 | 36 | Global model ID and cache. |
|
70 | 37 | Key, key type and local ID make a single global identificator of the model. |
|
71 | 38 | Content is an XML cache of the model that can be passed along between nodes |
|
72 | 39 | without manual serialization each time. |
|
73 | 40 | """ |
|
74 | 41 | class Meta: |
|
75 | 42 | app_label = 'boards' |
|
76 | 43 | |
|
77 | 44 | objects = GlobalIdManager() |
|
78 | 45 | |
|
79 | 46 | def __init__(self, *args, **kwargs): |
|
80 | 47 | models.Model.__init__(self, *args, **kwargs) |
|
81 | 48 | |
|
82 | 49 | if 'key' in kwargs and 'key_type' in kwargs and 'local_id' in kwargs: |
|
83 | 50 | self.key = kwargs['key'] |
|
84 | 51 | self.key_type = kwargs['key_type'] |
|
85 | 52 | self.local_id = kwargs['local_id'] |
|
86 | 53 | |
|
87 | 54 | key = models.TextField() |
|
88 | 55 | key_type = models.TextField() |
|
89 | 56 | local_id = models.IntegerField() |
|
90 | 57 | content = models.TextField(blank=True, null=True) |
|
91 | 58 | |
|
92 | 59 | def __str__(self): |
|
93 | 60 | return '%s::%s::%d' % (self.key_type, self.key, self.local_id) |
|
94 | 61 | |
|
95 | 62 | def to_xml_element(self, element: et.Element): |
|
96 | 63 | """ |
|
97 | 64 | Exports global id to an XML element. |
|
98 | 65 | """ |
|
99 | 66 | |
|
100 | 67 | element.set(ATTR_KEY, self.key) |
|
101 | 68 | element.set(ATTR_KEY_TYPE, self.key_type) |
|
102 | 69 | element.set(ATTR_LOCAL_ID, str(self.local_id)) |
|
103 | 70 | |
|
104 | 71 | @staticmethod |
|
105 | 72 | def from_xml_element(element: et.Element): |
|
106 | 73 | """ |
|
107 | 74 | Parses XML id tag and gets global id from it. |
|
108 | 75 | |
|
109 | 76 | Arguments: |
|
110 | 77 | element -- the XML 'id' element |
|
111 | 78 | |
|
112 | 79 | Returns: |
|
113 | 80 | global_id -- id itself |
|
114 | 81 | exists -- True if the global id was taken from database, False if it |
|
115 | 82 | did not exist and was created. |
|
116 | 83 | """ |
|
117 | 84 | |
|
118 | 85 | try: |
|
119 | 86 | return GlobalId.objects.get(key=element.get(ATTR_KEY), |
|
120 | 87 | key_type=element.get(ATTR_KEY_TYPE), |
|
121 | 88 | local_id=int(element.get( |
|
122 | 89 | ATTR_LOCAL_ID))), True |
|
123 | 90 | except GlobalId.DoesNotExist: |
|
124 | 91 | return GlobalId(key=element.get(ATTR_KEY), |
|
125 | 92 | key_type=element.get(ATTR_KEY_TYPE), |
|
126 | 93 | local_id=int(element.get(ATTR_LOCAL_ID))), False |
|
127 | 94 | |
|
128 | 95 | def is_local(self): |
|
129 | 96 | """Checks fo the ID is local model's""" |
|
130 | 97 | return KeyPair.objects.filter( |
|
131 | 98 | key_type=self.key_type, public_key=self.key).exists() |
|
132 | 99 | |
|
133 | 100 | def clear_cache(self): |
|
134 | 101 | """ |
|
135 | 102 | Removes content cache and signatures. |
|
136 | 103 | """ |
|
137 | 104 | self.content = None |
|
138 | 105 | self.save() |
|
139 | 106 | self.signature_set.all().delete() |
|
140 | 107 | |
|
141 | 108 | |
|
142 | 109 | class Signature(models.Model): |
|
143 | 110 | class Meta: |
|
144 | 111 | app_label = 'boards' |
|
145 | 112 | |
|
146 | 113 | def __init__(self, *args, **kwargs): |
|
147 | 114 | models.Model.__init__(self, *args, **kwargs) |
|
148 | 115 | |
|
149 | 116 | if 'key' in kwargs and 'key_type' in kwargs and 'signature' in kwargs: |
|
150 | 117 | self.key_type = kwargs['key_type'] |
|
151 | 118 | self.key = kwargs['key'] |
|
152 | 119 | self.signature = kwargs['signature'] |
|
153 | 120 | |
|
154 | 121 | key_type = models.TextField() |
|
155 | 122 | key = models.TextField() |
|
156 | 123 | signature = models.TextField() |
|
157 | 124 | |
|
158 | 125 | global_id = models.ForeignKey('GlobalId') |
@@ -1,132 +1,135 b'' | |||
|
1 | 1 | import re |
|
2 | 2 | import os |
|
3 | 3 | |
|
4 | 4 | from django.db.models.signals import post_save, pre_save, pre_delete, \ |
|
5 | 5 | post_delete |
|
6 | 6 | from django.dispatch import receiver |
|
7 | 7 | from django.utils import timezone |
|
8 | 8 | |
|
9 | 9 | from boards import thumbs |
|
10 | 10 | from boards.mdx_neboard import get_parser |
|
11 | 11 | |
|
12 | 12 | from boards.models import Post, GlobalId, Attachment |
|
13 | 13 | from boards.models.attachment.viewers import FILE_TYPES_IMAGE |
|
14 | 14 | from boards.models.post import REGEX_NOTIFICATION, REGEX_REPLY,\ |
|
15 | 15 | REGEX_GLOBAL_REPLY |
|
16 | 16 | from boards.models.post.manager import post_import_deps |
|
17 | 17 | from boards.models.user import Notification |
|
18 | 18 | from neboard.settings import MEDIA_ROOT |
|
19 | 19 | |
|
20 | 20 | |
|
21 | 21 | THUMB_SIZES = ((200, 150),) |
|
22 | 22 | |
|
23 | 23 | |
|
24 | 24 | @receiver(post_save, sender=Post) |
|
25 | 25 | def connect_replies(instance, **kwargs): |
|
26 | for reply_number in re.finditer(REGEX_REPLY, instance.get_raw_text()): | |
|
27 | post_id = reply_number.group(1) | |
|
26 | if not kwargs['update_fields']: | |
|
27 | for reply_number in re.finditer(REGEX_REPLY, instance.get_raw_text()): | |
|
28 | post_id = reply_number.group(1) | |
|
28 | 29 | |
|
29 | try: | |
|
30 | referenced_post = Post.objects.get(id=post_id) | |
|
30 | try: | |
|
31 | referenced_post = Post.objects.get(id=post_id) | |
|
31 | 32 | |
|
32 | if not referenced_post.referenced_posts.filter( | |
|
33 | id=instance.id).exists(): | |
|
34 | referenced_post.referenced_posts.add(instance) | |
|
35 | referenced_post.last_edit_time = instance.pub_time | |
|
36 | referenced_post.build_refmap() | |
|
37 | referenced_post.save(update_fields=['refmap', 'last_edit_time']) | |
|
38 | except Post.DoesNotExist: | |
|
39 | pass | |
|
33 | if not referenced_post.referenced_posts.filter( | |
|
34 | id=instance.id).exists(): | |
|
35 | referenced_post.referenced_posts.add(instance) | |
|
36 | referenced_post.last_edit_time = instance.pub_time | |
|
37 | referenced_post.build_refmap() | |
|
38 | referenced_post.save(update_fields=['refmap', 'last_edit_time']) | |
|
39 | except Post.DoesNotExist: | |
|
40 | pass | |
|
40 | 41 | |
|
41 | 42 | |
|
42 | 43 | @receiver(post_save, sender=Post) |
|
43 | 44 | @receiver(post_import_deps, sender=Post) |
|
44 | 45 | def connect_global_replies(instance, **kwargs): |
|
45 | for reply_number in re.finditer(REGEX_GLOBAL_REPLY, instance.get_raw_text()): | |
|
46 | key_type = reply_number.group(1) | |
|
47 |
key = reply_number.group( |
|
|
48 |
|
|
|
46 | if not kwargs['update_fields']: | |
|
47 | for reply_number in re.finditer(REGEX_GLOBAL_REPLY, instance.get_raw_text()): | |
|
48 | key_type = reply_number.group(1) | |
|
49 | key = reply_number.group(2) | |
|
50 | local_id = reply_number.group(3) | |
|
49 | 51 | |
|
50 | try: | |
|
51 | global_id = GlobalId.objects.get(key_type=key_type, key=key, | |
|
52 | local_id=local_id) | |
|
53 | referenced_post = Post.objects.get(global_id=global_id) | |
|
54 | referenced_post.referenced_posts.add(instance) | |
|
55 | referenced_post.last_edit_time = instance.pub_time | |
|
56 | referenced_post.build_refmap() | |
|
57 | referenced_post.save(update_fields=['refmap', 'last_edit_time']) | |
|
58 | except (GlobalId.DoesNotExist, Post.DoesNotExist): | |
|
59 | pass | |
|
52 | try: | |
|
53 | global_id = GlobalId.objects.get(key_type=key_type, key=key, | |
|
54 | local_id=local_id) | |
|
55 | referenced_post = Post.objects.get(global_id=global_id) | |
|
56 | referenced_post.referenced_posts.add(instance) | |
|
57 | referenced_post.last_edit_time = instance.pub_time | |
|
58 | referenced_post.build_refmap() | |
|
59 | referenced_post.save(update_fields=['refmap', 'last_edit_time']) | |
|
60 | except (GlobalId.DoesNotExist, Post.DoesNotExist): | |
|
61 | pass | |
|
60 | 62 | |
|
61 | 63 | |
|
62 | 64 | @receiver(post_save, sender=Post) |
|
63 | 65 | def connect_notifications(instance, **kwargs): |
|
64 | for reply_number in re.finditer(REGEX_NOTIFICATION, instance.get_raw_text()): | |
|
65 | user_name = reply_number.group(1).lower() | |
|
66 | Notification.objects.get_or_create(name=user_name, post=instance) | |
|
66 | if not kwargs['update_fields']: | |
|
67 | for reply_number in re.finditer(REGEX_NOTIFICATION, instance.get_raw_text()): | |
|
68 | user_name = reply_number.group(1).lower() | |
|
69 | Notification.objects.get_or_create(name=user_name, post=instance) | |
|
67 | 70 | |
|
68 | 71 | |
|
69 | 72 | @receiver(pre_save, sender=Post) |
|
70 | 73 | @receiver(post_import_deps, sender=Post) |
|
71 | 74 | def parse_text(instance, **kwargs): |
|
72 | 75 | instance._text_rendered = get_parser().parse(instance.get_raw_text()) |
|
73 | 76 | |
|
74 | 77 | |
|
75 | 78 | @receiver(pre_delete, sender=Post) |
|
76 | 79 | def delete_attachments(instance, **kwargs): |
|
77 | 80 | for attachment in instance.attachments.all(): |
|
78 | 81 | attachment_refs_count = attachment.attachment_posts.count() |
|
79 | 82 | if attachment_refs_count == 1: |
|
80 | 83 | attachment.delete() |
|
81 | 84 | |
|
82 | 85 | |
|
83 | 86 | @receiver(post_delete, sender=Post) |
|
84 | 87 | def update_thread_on_delete(instance, **kwargs): |
|
85 | 88 | thread = instance.get_thread() |
|
86 | 89 | thread.last_edit_time = timezone.now() |
|
87 | 90 | thread.save() |
|
88 | 91 | |
|
89 | 92 | |
|
90 | 93 | @receiver(post_delete, sender=Post) |
|
91 | 94 | def delete_global_id(instance, **kwargs): |
|
92 | 95 | if instance.global_id and instance.global_id.id: |
|
93 | 96 | instance.global_id.delete() |
|
94 | 97 | |
|
95 | 98 | |
|
96 | 99 | @receiver(post_save, sender=Attachment) |
|
97 | 100 | def generate_thumb(instance, **kwargs): |
|
98 | 101 | if instance.mimetype in FILE_TYPES_IMAGE: |
|
99 | 102 | for size in THUMB_SIZES: |
|
100 | 103 | (w, h) = size |
|
101 | 104 | split = instance.file.name.rsplit('.', 1) |
|
102 | 105 | thumb_name = '%s.%sx%s.%s' % (split[0], w, h, split[1]) |
|
103 | 106 | |
|
104 | 107 | if not instance.file.storage.exists(thumb_name): |
|
105 | 108 | # you can use another thumbnailing function if you like |
|
106 | 109 | thumb_content = thumbs.generate_thumb(instance.file, size, split[1]) |
|
107 | 110 | |
|
108 | 111 | thumb_name_ = instance.file.storage.save(thumb_name, thumb_content) |
|
109 | 112 | |
|
110 | 113 | if not thumb_name == thumb_name_: |
|
111 | 114 | raise ValueError( |
|
112 | 115 | 'There is already a file named %s' % thumb_name_) |
|
113 | 116 | |
|
114 | 117 | |
|
115 | 118 | @receiver(pre_delete, sender=Post) |
|
116 | 119 | def rebuild_refmap(instance, **kwargs): |
|
117 | 120 | for referenced_post in instance.refposts.all(): |
|
118 | 121 | referenced_post.build_refmap(excluded_ids=[instance.id]) |
|
119 | 122 | referenced_post.save(update_fields=['refmap']) |
|
120 | 123 | |
|
121 | 124 | |
|
122 | 125 | @receiver(post_delete, sender=Attachment) |
|
123 | 126 | def delete_file(instance, **kwargs): |
|
124 | 127 | if instance.is_internal(): |
|
125 | 128 | file = MEDIA_ROOT + instance.file.name |
|
126 | 129 | os.remove(file) |
|
127 | 130 | if instance.mimetype in FILE_TYPES_IMAGE: |
|
128 | 131 | for size in THUMB_SIZES: |
|
129 | 132 | file_name_parts = instance.file.name.split('.') |
|
130 | 133 | thumb_file = MEDIA_ROOT + '{}.{}x{}.{}'.format(file_name_parts[0], size[0], size[1], file_name_parts[1]) |
|
131 | 134 | os.remove(thumb_file) |
|
132 | 135 |
@@ -1,160 +1,155 b'' | |||
|
1 | 1 | /* |
|
2 | 2 | @licstart The following is the entire license notice for the |
|
3 | 3 | JavaScript code in this page. |
|
4 | 4 | |
|
5 | 5 | |
|
6 | 6 | Copyright (C) 2013 neko259 |
|
7 | 7 | |
|
8 | 8 | The JavaScript code in this page is free software: you can |
|
9 | 9 | redistribute it and/or modify it under the terms of the GNU |
|
10 | 10 | General Public License (GNU GPL) as published by the Free Software |
|
11 | 11 | Foundation, either version 3 of the License, or (at your option) |
|
12 | 12 | any later version. The code is distributed WITHOUT ANY WARRANTY; |
|
13 | 13 | without even the implied warranty of MERCHANTABILITY or FITNESS |
|
14 | 14 | FOR A PARTICULAR PURPOSE. See the GNU GPL for more details. |
|
15 | 15 | |
|
16 | 16 | As additional permission under GNU GPL version 3 section 7, you |
|
17 | 17 | may distribute non-source (e.g., minimized or compacted) forms of |
|
18 | 18 | that code without the copy of the GNU GPL normally required by |
|
19 | 19 | section 4, provided you include this license notice and a URL |
|
20 | 20 | through which recipients can access the Corresponding Source. |
|
21 | 21 | |
|
22 | 22 | @licend The above is the entire license notice |
|
23 | 23 | for the JavaScript code in this page. |
|
24 | 24 | */ |
|
25 | 25 | |
|
26 | 26 | var CLOSE_BUTTON = '#form-close-button'; |
|
27 | 27 | var REPLY_TO_MSG = '.reply-to-message'; |
|
28 | 28 | var REPLY_TO_MSG_ID = '#reply-to-message-id'; |
|
29 | 29 | |
|
30 | 30 | var $html = $("html, body"); |
|
31 | 31 | |
|
32 | 32 | function moveCaretToEnd(el) { |
|
33 | 33 | var newPos = el.val().length; |
|
34 | 34 | el[0].setSelectionRange(newPos, newPos); |
|
35 | 35 | } |
|
36 | 36 | |
|
37 | 37 | function getForm() { |
|
38 | 38 | return $('.post-form-w'); |
|
39 | 39 | } |
|
40 | 40 | |
|
41 | 41 | function resetFormPosition() { |
|
42 | 42 | var form = getForm(); |
|
43 | 43 | form.insertAfter($('.thread')); |
|
44 | 44 | |
|
45 | 45 | $(CLOSE_BUTTON).hide(); |
|
46 | 46 | $(REPLY_TO_MSG).hide(); |
|
47 | 47 | } |
|
48 | 48 | |
|
49 | 49 | function showFormAfter(blockToInsertAfter) { |
|
50 | 50 | var form = getForm(); |
|
51 | 51 | form.insertAfter(blockToInsertAfter); |
|
52 | 52 | |
|
53 | 53 | $(CLOSE_BUTTON).show(); |
|
54 | 54 | form.show(); |
|
55 | 55 | $(REPLY_TO_MSG_ID).text(blockToInsertAfter.attr('id')); |
|
56 | 56 | $(REPLY_TO_MSG).show(); |
|
57 | 57 | } |
|
58 | 58 | |
|
59 | 59 | function addQuickReply(postId) { |
|
60 | // If we click "reply" on the same post, it means "cancel" | |
|
61 | if (getForm().prev().attr('id') == postId) { | |
|
62 | resetFormPosition(); | |
|
63 | } else { | |
|
64 | var blockToInsert = null; | |
|
65 | var textAreaJq = getPostTextarea(); | |
|
66 | var postLinkRaw = '[post]' + postId + '[/post]' | |
|
67 | var textToAdd = ''; | |
|
60 | var blockToInsert = null; | |
|
61 | var textAreaJq = getPostTextarea(); | |
|
62 | var postLinkRaw = '[post]' + postId + '[/post]' | |
|
63 | var textToAdd = ''; | |
|
68 | 64 | |
|
69 |
|
|
|
70 |
|
|
|
65 | if (postId != null) { | |
|
66 | var post = $('#' + postId); | |
|
71 | 67 | |
|
72 |
|
|
|
73 |
|
|
|
74 |
|
|
|
75 |
|
|
|
76 |
|
|
|
77 |
|
|
|
78 |
|
|
|
79 | } | |
|
80 | textToAdd += postLinkRaw + '\n'; | |
|
68 | // If this is not OP, add reflink to the post. If there already is | |
|
69 | // the same reflink, don't add it again. | |
|
70 | var postText = textAreaJq.val(); | |
|
71 | if (!post.is(':first-child') && postText.indexOf(postLinkRaw) < 0) { | |
|
72 | // Insert line break if none is present. | |
|
73 | if (postText.length > 0 && !postText.endsWith('\n') && !postText.endsWith('\r')) { | |
|
74 | textToAdd += '\n'; | |
|
81 | 75 | } |
|
76 | textToAdd += postLinkRaw + '\n'; | |
|
77 | } | |
|
82 | 78 | |
|
83 |
|
|
|
84 |
|
|
|
85 |
|
|
|
86 |
|
|
|
87 |
|
|
|
88 |
|
|
|
89 | ||
|
90 | textAreaJq.focus(); | |
|
79 | textAreaJq.val(textAreaJq.val()+ textToAdd); | |
|
80 | blockToInsert = post; | |
|
81 | } else { | |
|
82 | blockToInsert = $('.thread'); | |
|
83 | } | |
|
84 | showFormAfter(blockToInsert); | |
|
91 | 85 | |
|
92 | moveCaretToEnd(textAreaJq); | |
|
93 | } | |
|
86 | textAreaJq.focus(); | |
|
87 | ||
|
88 | moveCaretToEnd(textAreaJq); | |
|
94 | 89 | } |
|
95 | 90 | |
|
96 | 91 | function addQuickQuote() { |
|
97 | 92 | var textAreaJq = getPostTextarea(); |
|
98 | 93 | |
|
99 | 94 | var quoteButton = $("#quote-button"); |
|
100 | 95 | var postId = quoteButton.attr('data-post-id'); |
|
101 | if (postId != null && getForm().prev().attr('id') != postId) { | |
|
96 | if (postId != null) { | |
|
102 | 97 | addQuickReply(postId); |
|
103 | 98 | } |
|
104 | 99 | |
|
105 | 100 | var textToAdd = ''; |
|
106 | 101 | var selection = window.getSelection().toString(); |
|
107 | 102 | if (selection.length == 0) { |
|
108 | 103 | selection = quoteButton.attr('data-text'); |
|
109 | 104 | } |
|
110 | 105 | if (selection.length > 0) { |
|
111 | 106 | textToAdd += '[quote]' + selection + '[/quote]\n'; |
|
112 | 107 | } |
|
113 | 108 | |
|
114 | 109 | textAreaJq.val(textAreaJq.val() + textToAdd); |
|
115 | 110 | |
|
116 | 111 | textAreaJq.focus(); |
|
117 | 112 | |
|
118 | 113 | moveCaretToEnd(textAreaJq); |
|
119 | 114 | } |
|
120 | 115 | |
|
121 | 116 | function scrollToBottom() { |
|
122 | 117 | $html.animate({scrollTop: $html.height()}, "fast"); |
|
123 | 118 | } |
|
124 | 119 | |
|
125 | 120 | function showQuoteButton() { |
|
126 | 121 | var selection = window.getSelection().getRangeAt(0).getBoundingClientRect(); |
|
127 | 122 | var quoteButton = $("#quote-button"); |
|
128 | 123 | if (selection.width > 0) { |
|
129 | 124 | // quoteButton.offset({ top: selection.top - selection.height, left: selection.left }); |
|
130 | 125 | quoteButton.css({top: selection.top + $(window).scrollTop() - 30, left: selection.left}); |
|
131 | 126 | quoteButton.show(); |
|
132 | 127 | |
|
133 | 128 | var text = window.getSelection().toString(); |
|
134 | 129 | quoteButton.attr('data-text', text); |
|
135 | 130 | |
|
136 | 131 | var rect = window.getSelection().getRangeAt(0).getBoundingClientRect(); |
|
137 | 132 | var element = $(document.elementFromPoint(rect.x, rect.y)); |
|
138 | 133 | var postId = null; |
|
139 | 134 | if (element.hasClass('post')) { |
|
140 | 135 | postId = element.attr('id'); |
|
141 | 136 | } else { |
|
142 | 137 | var postParent = element.parents('.post'); |
|
143 | 138 | if (postParent.length > 0) { |
|
144 | 139 | postId = postParent.attr('id'); |
|
145 | 140 | } |
|
146 | 141 | } |
|
147 | 142 | quoteButton.attr('data-post-id', postId); |
|
148 | 143 | } else { |
|
149 | 144 | quoteButton.hide(); |
|
150 | 145 | } |
|
151 | 146 | } |
|
152 | 147 | |
|
153 | 148 | $(document).ready(function() { |
|
154 | 149 | $('body').on('mouseup', function() { |
|
155 | 150 | showQuoteButton(); |
|
156 | 151 | }); |
|
157 | 152 | $("#quote-button").click(function() { |
|
158 | 153 | addQuickQuote(); |
|
159 | 154 | }) |
|
160 | 155 | }); |
@@ -1,91 +1,90 b'' | |||
|
1 | from base64 import b64encode | |
|
2 | 1 |
|
|
3 | 2 | |
|
4 | 3 | from django.test import TestCase |
|
5 | 4 | from boards.models import KeyPair, GlobalId, Post, Signature |
|
6 | 5 | from boards.models.post.sync import SyncManager |
|
7 | 6 | |
|
8 | 7 | logger = logging.getLogger(__name__) |
|
9 | 8 | |
|
10 | 9 | |
|
11 | 10 | class KeyTest(TestCase): |
|
12 | 11 | def test_create_key(self): |
|
13 | 12 | key = KeyPair.objects.generate_key('ecdsa') |
|
14 | 13 | |
|
15 | 14 | self.assertIsNotNone(key, 'The key was not created.') |
|
16 | 15 | |
|
17 | 16 | def test_validation(self): |
|
18 | 17 | key = KeyPair.objects.generate_key(key_type='ecdsa') |
|
19 | 18 | message = 'msg' |
|
20 | 19 | signature_value = key.sign(message) |
|
21 | 20 | |
|
22 | 21 | signature = Signature(key_type='ecdsa', key=key.public_key, |
|
23 | 22 | signature=signature_value) |
|
24 | 23 | valid = KeyPair.objects.verify(signature, message) |
|
25 | 24 | |
|
26 | 25 | self.assertTrue(valid, 'Message verification failed.') |
|
27 | 26 | |
|
28 | 27 | def test_primary_constraint(self): |
|
29 | 28 | KeyPair.objects.generate_key(key_type='ecdsa', primary=True) |
|
30 | 29 | |
|
31 | 30 | with self.assertRaises(Exception): |
|
32 | 31 | KeyPair.objects.generate_key(key_type='ecdsa', primary=True) |
|
33 | 32 | |
|
34 | 33 | def test_model_id_save(self): |
|
35 | 34 | model_id = GlobalId(key_type='test', key='test key', local_id='1') |
|
36 | 35 | model_id.save() |
|
37 | 36 | |
|
38 | 37 | def test_request_get(self): |
|
39 | 38 | post = self._create_post_with_key() |
|
40 | 39 | |
|
41 |
request = |
|
|
40 | request = SyncManager.generate_request_get([post.global_id]) | |
|
42 | 41 | logger.debug(request) |
|
43 | 42 | |
|
44 | 43 | key = KeyPair.objects.get(primary=True) |
|
45 | 44 | self.assertTrue('<request type="get" version="1.0">' |
|
46 | 45 | '<model name="post" version="1.0">' |
|
47 | 46 | '<id key="%s" local-id="1" type="%s" />' |
|
48 | 47 | '</model>' |
|
49 | 48 | '</request>' % ( |
|
50 | 49 | key.public_key, |
|
51 | 50 | key.key_type, |
|
52 | 51 | ) in request, |
|
53 | 52 | 'Wrong XML generated for the GET request.') |
|
54 | 53 | |
|
55 | 54 | def test_response_get(self): |
|
56 | 55 | post = self._create_post_with_key() |
|
57 | 56 | reply_post = Post.objects.create_post(title='test_title', |
|
58 | 57 | text='[post]%d[/post]' % post.id, |
|
59 | 58 | thread=post.get_thread()) |
|
60 | 59 | |
|
61 | 60 | response = SyncManager.generate_response_get([reply_post]) |
|
62 | 61 | logger.debug(response) |
|
63 | 62 | |
|
64 | 63 | key = KeyPair.objects.get(primary=True) |
|
65 | 64 | self.assertTrue('<status>success</status>' |
|
66 | 65 | '<models>' |
|
67 | 66 | '<model name="post">' |
|
68 | 67 | '<content>' |
|
69 | 68 | '<id key="%s" local-id="%d" type="%s" />' |
|
70 | 69 | '<title>test_title</title>' |
|
71 | 70 | '<text>[post]%s[/post]</text>' |
|
72 | 71 | '<thread><id key="%s" local-id="%d" type="%s" /></thread>' |
|
73 | 72 | '<pub-time>%s</pub-time>' |
|
74 | 73 | '<version>%s</version>' |
|
75 | 74 | '</content>' % ( |
|
76 | 75 | key.public_key, |
|
77 | 76 | reply_post.id, |
|
78 | 77 | key.key_type, |
|
79 | 78 | str(post.global_id), |
|
80 | 79 | key.public_key, |
|
81 | 80 | post.id, |
|
82 | 81 | key.key_type, |
|
83 | 82 | str(reply_post.get_pub_time_str()), |
|
84 | 83 | post.version, |
|
85 | 84 | ) in response, |
|
86 | 85 | 'Wrong XML generated for the GET response.') |
|
87 | 86 | |
|
88 | 87 | def _create_post_with_key(self): |
|
89 | 88 | KeyPair.objects.generate_key(primary=True) |
|
90 | 89 | |
|
91 | 90 | return Post.objects.create_post(title='test_title', text='test_text') |
@@ -1,105 +1,214 b'' | |||
|
1 | 1 | from django.test import TestCase |
|
2 | 2 | |
|
3 | 3 | from boards.models import KeyPair, Post, Tag |
|
4 | 4 | from boards.models.post.sync import SyncManager |
|
5 | 5 | from boards.tests.mocks import MockRequest |
|
6 | from boards.views.sync import response_get | |
|
6 | from boards.views.sync import response_get, response_list | |
|
7 | 7 | |
|
8 | 8 | __author__ = 'neko259' |
|
9 | 9 | |
|
10 | 10 | |
|
11 | 11 | class SyncTest(TestCase): |
|
12 | 12 | def test_get(self): |
|
13 | 13 | """ |
|
14 | 14 | Forms a GET request of a post and checks the response. |
|
15 | 15 | """ |
|
16 | 16 | |
|
17 | 17 | key = KeyPair.objects.generate_key(primary=True) |
|
18 | 18 | tag = Tag.objects.create(name='tag1') |
|
19 | 19 | post = Post.objects.create_post(title='test_title', |
|
20 | 20 | text='test_text\rline two', |
|
21 | 21 | tags=[tag]) |
|
22 | 22 | |
|
23 | 23 | request = MockRequest() |
|
24 | 24 | request.body = ( |
|
25 | 25 | '<request type="get" version="1.0">' |
|
26 | 26 | '<model name="post" version="1.0">' |
|
27 | 27 | '<id key="%s" local-id="%d" type="%s" />' |
|
28 | 28 | '</model>' |
|
29 | 29 | '</request>' % (post.global_id.key, |
|
30 | 30 | post.id, |
|
31 | 31 | post.global_id.key_type) |
|
32 | 32 | ) |
|
33 | 33 | |
|
34 | 34 | response = response_get(request).content.decode() |
|
35 | 35 | self.assertTrue( |
|
36 | 36 | '<status>success</status>' |
|
37 | 37 | '<models>' |
|
38 | 38 | '<model name="post">' |
|
39 | 39 | '<content>' |
|
40 | 40 | '<id key="%s" local-id="%d" type="%s" />' |
|
41 | 41 | '<title>%s</title>' |
|
42 | 42 | '<text>%s</text>' |
|
43 | 43 | '<tags><tag>%s</tag></tags>' |
|
44 | 44 | '<pub-time>%s</pub-time>' |
|
45 | 45 | '<version>%s</version>' |
|
46 | 46 | '</content>' % ( |
|
47 | 47 | post.global_id.key, |
|
48 | 48 | post.global_id.local_id, |
|
49 | 49 | post.global_id.key_type, |
|
50 | 50 | post.title, |
|
51 | 51 | post.get_sync_text(), |
|
52 | 52 | post.get_thread().get_tags().first().name, |
|
53 | 53 | post.get_pub_time_str(), |
|
54 | 54 | post.version, |
|
55 | 55 | ) in response, |
|
56 | 56 | 'Wrong response generated for the GET request.') |
|
57 | 57 | |
|
58 | 58 | post.delete() |
|
59 | 59 | key.delete() |
|
60 | 60 | |
|
61 | 61 | KeyPair.objects.generate_key(primary=True) |
|
62 | 62 | |
|
63 | 63 | SyncManager.parse_response_get(response, None) |
|
64 | 64 | self.assertEqual(1, Post.objects.count(), |
|
65 | 65 | 'Post was not created from XML response.') |
|
66 | 66 | |
|
67 | 67 | parsed_post = Post.objects.first() |
|
68 | 68 | self.assertEqual('tag1', |
|
69 | 69 | parsed_post.get_thread().get_tags().first().name, |
|
70 | 70 | 'Invalid tag was parsed.') |
|
71 | 71 | |
|
72 | 72 | SyncManager.parse_response_get(response, None) |
|
73 | 73 | self.assertEqual(1, Post.objects.count(), |
|
74 | 74 | 'The same post was imported twice.') |
|
75 | 75 | |
|
76 | 76 | self.assertEqual(1, parsed_post.global_id.signature_set.count(), |
|
77 | 77 | 'Signature was not saved.') |
|
78 | 78 | |
|
79 | 79 | post = parsed_post |
|
80 | 80 | |
|
81 | 81 | # Trying to sync the same once more |
|
82 | 82 | response = response_get(request).content.decode() |
|
83 | 83 | |
|
84 | 84 | self.assertTrue( |
|
85 | 85 | '<status>success</status>' |
|
86 | 86 | '<models>' |
|
87 | 87 | '<model name="post">' |
|
88 | 88 | '<content>' |
|
89 | 89 | '<id key="%s" local-id="%d" type="%s" />' |
|
90 | 90 | '<title>%s</title>' |
|
91 | 91 | '<text>%s</text>' |
|
92 | 92 | '<tags><tag>%s</tag></tags>' |
|
93 | 93 | '<pub-time>%s</pub-time>' |
|
94 | 94 | '<version>%s</version>' |
|
95 | 95 | '</content>' % ( |
|
96 | 96 | post.global_id.key, |
|
97 | 97 | post.global_id.local_id, |
|
98 | 98 | post.global_id.key_type, |
|
99 | 99 | post.title, |
|
100 | 100 | post.get_sync_text(), |
|
101 | 101 | post.get_thread().get_tags().first().name, |
|
102 | 102 | post.get_pub_time_str(), |
|
103 | 103 | post.version, |
|
104 | 104 | ) in response, |
|
105 | 105 | 'Wrong response generated for the GET request.') |
|
106 | ||
|
107 | def test_list_all(self): | |
|
108 | key = KeyPair.objects.generate_key(primary=True) | |
|
109 | tag = Tag.objects.create(name='tag1') | |
|
110 | post = Post.objects.create_post(title='test_title', | |
|
111 | text='test_text\rline two', | |
|
112 | tags=[tag]) | |
|
113 | post2 = Post.objects.create_post(title='test title 2', | |
|
114 | text='test text 2', | |
|
115 | tags=[tag]) | |
|
116 | ||
|
117 | request_all = MockRequest() | |
|
118 | request_all.body = ( | |
|
119 | '<request type="list" version="1.0">' | |
|
120 | '<model name="post" version="1.0">' | |
|
121 | '</model>' | |
|
122 | '</request>' | |
|
123 | ) | |
|
124 | ||
|
125 | response_all = response_list(request_all).content.decode() | |
|
126 | self.assertTrue( | |
|
127 | '<status>success</status>' | |
|
128 | '<models>' | |
|
129 | '<model>' | |
|
130 | '<id key="{}" local-id="{}" type="{}" />' | |
|
131 | '<version>{}</version>' | |
|
132 | '</model>' | |
|
133 | '<model>' | |
|
134 | '<id key="{}" local-id="{}" type="{}" />' | |
|
135 | '<version>{}</version>' | |
|
136 | '</model>' | |
|
137 | '</models>'.format( | |
|
138 | post.global_id.key, | |
|
139 | post.global_id.local_id, | |
|
140 | post.global_id.key_type, | |
|
141 | post.version, | |
|
142 | post2.global_id.key, | |
|
143 | post2.global_id.local_id, | |
|
144 | post2.global_id.key_type, | |
|
145 | post2.version, | |
|
146 | ) in response_all, | |
|
147 | 'Wrong response generated for the LIST request for all posts.') | |
|
148 | ||
|
149 | def test_list_existing_thread(self): | |
|
150 | key = KeyPair.objects.generate_key(primary=True) | |
|
151 | tag = Tag.objects.create(name='tag1') | |
|
152 | post = Post.objects.create_post(title='test_title', | |
|
153 | text='test_text\rline two', | |
|
154 | tags=[tag]) | |
|
155 | post2 = Post.objects.create_post(title='test title 2', | |
|
156 | text='test text 2', | |
|
157 | tags=[tag]) | |
|
158 | ||
|
159 | request_thread = MockRequest() | |
|
160 | request_thread.body = ( | |
|
161 | '<request type="list" version="1.0">' | |
|
162 | '<model name="post" version="1.0">' | |
|
163 | '<thread>{}</thread>' | |
|
164 | '</model>' | |
|
165 | '</request>'.format( | |
|
166 | post.id, | |
|
167 | ) | |
|
168 | ) | |
|
169 | ||
|
170 | response_thread = response_list(request_thread).content.decode() | |
|
171 | self.assertTrue( | |
|
172 | '<status>success</status>' | |
|
173 | '<models>' | |
|
174 | '<model>' | |
|
175 | '<id key="{}" local-id="{}" type="{}" />' | |
|
176 | '<version>{}</version>' | |
|
177 | '</model>' | |
|
178 | '</models>'.format( | |
|
179 | post.global_id.key, | |
|
180 | post.global_id.local_id, | |
|
181 | post.global_id.key_type, | |
|
182 | post.version, | |
|
183 | ) in response_thread, | |
|
184 | 'Wrong response generated for the LIST request for posts of ' | |
|
185 | 'existing thread.') | |
|
186 | ||
|
187 | def test_list_non_existing_thread(self): | |
|
188 | key = KeyPair.objects.generate_key(primary=True) | |
|
189 | tag = Tag.objects.create(name='tag1') | |
|
190 | post = Post.objects.create_post(title='test_title', | |
|
191 | text='test_text\rline two', | |
|
192 | tags=[tag]) | |
|
193 | post2 = Post.objects.create_post(title='test title 2', | |
|
194 | text='test text 2', | |
|
195 | tags=[tag]) | |
|
196 | ||
|
197 | request_thread = MockRequest() | |
|
198 | request_thread.body = ( | |
|
199 | '<request type="list" version="1.0">' | |
|
200 | '<model name="post" version="1.0">' | |
|
201 | '<thread>{}</thread>' | |
|
202 | '</model>' | |
|
203 | '</request>'.format( | |
|
204 | 0, | |
|
205 | ) | |
|
206 | ) | |
|
207 | ||
|
208 | response_thread = response_list(request_thread).content.decode() | |
|
209 | self.assertTrue( | |
|
210 | '<status>success</status>' | |
|
211 | '<models />' | |
|
212 | in response_thread, | |
|
213 | 'Wrong response generated for the LIST request for posts of ' | |
|
214 | 'non-existing thread.') |
@@ -1,55 +1,79 b'' | |||
|
1 | import logging | |
|
2 | ||
|
1 | 3 | import xml.etree.ElementTree as et |
|
2 | 4 | |
|
3 | 5 | from django.http import HttpResponse, Http404 |
|
6 | ||
|
7 | from boards.abstracts.sync_filters import ThreadFilter, TAG_THREAD | |
|
4 | 8 | from boards.models import GlobalId, Post |
|
5 | 9 | from boards.models.post.sync import SyncManager |
|
6 | 10 | |
|
7 | 11 | |
|
12 | logger = logging.getLogger('boards.sync') | |
|
13 | ||
|
14 | ||
|
15 | FILTERS = { | |
|
16 | TAG_THREAD: ThreadFilter, | |
|
17 | } | |
|
18 | ||
|
19 | ||
|
8 | 20 | def response_list(request): |
|
9 | 21 | request_xml = request.body |
|
10 | 22 | |
|
23 | filters = [] | |
|
24 | ||
|
11 | 25 | if request_xml is None or len(request_xml) == 0: |
|
12 | 26 | return HttpResponse(content='Use the API') |
|
27 | else: | |
|
28 | root_tag = et.fromstring(request_xml) | |
|
29 | model_tag = root_tag[0] | |
|
13 | 30 | |
|
14 | response_xml = SyncManager.generate_response_list() | |
|
31 | for tag_filter in model_tag: | |
|
32 | filter_name = tag_filter.tag | |
|
33 | model_filter = FILTERS.get(filter_name)(tag_filter) | |
|
34 | if not model_filter: | |
|
35 | logger.warning('Unavailable filter: {}'.format(filter_name)) | |
|
36 | filters.append(model_filter) | |
|
37 | ||
|
38 | response_xml = SyncManager.generate_response_list(filters) | |
|
15 | 39 | |
|
16 | 40 | return HttpResponse(content=response_xml) |
|
17 | 41 | |
|
18 | 42 | |
|
19 | 43 | def response_get(request): |
|
20 | 44 | """ |
|
21 | 45 | Processes a GET request with post ID list and returns the posts XML list. |
|
22 | 46 | Request should contain an 'xml' post attribute with the actual request XML. |
|
23 | 47 | """ |
|
24 | 48 | |
|
25 | 49 | request_xml = request.body |
|
26 | 50 | |
|
27 | 51 | if request_xml is None or len(request_xml) == 0: |
|
28 | 52 | return HttpResponse(content='Use the API') |
|
29 | 53 | |
|
30 | 54 | posts = [] |
|
31 | 55 | |
|
32 | 56 | root_tag = et.fromstring(request_xml) |
|
33 | 57 | model_tag = root_tag[0] |
|
34 | 58 | for id_tag in model_tag: |
|
35 | 59 | global_id, exists = GlobalId.from_xml_element(id_tag) |
|
36 | 60 | if exists: |
|
37 | 61 | posts.append(Post.objects.get(global_id=global_id)) |
|
38 | 62 | |
|
39 | 63 | response_xml = SyncManager.generate_response_get(posts) |
|
40 | 64 | |
|
41 | 65 | return HttpResponse(content=response_xml) |
|
42 | 66 | |
|
43 | 67 | |
|
44 | 68 | def get_post_sync_data(request, post_id): |
|
45 | 69 | try: |
|
46 | 70 | post = Post.objects.get(id=post_id) |
|
47 | 71 | except Post.DoesNotExist: |
|
48 | 72 | raise Http404() |
|
49 | 73 | |
|
50 | 74 | xml_str = SyncManager.generate_response_get([post]) |
|
51 | 75 | |
|
52 | 76 | return HttpResponse( |
|
53 | 77 | content_type='text/xml; charset=utf-8', |
|
54 | 78 | content=xml_str, |
|
55 | 79 | ) |
@@ -1,66 +1,66 b'' | |||
|
1 | 1 | # INTRO # |
|
2 | 2 | |
|
3 | 3 | This project aims to create centralized forum-like discussion platform with |
|
4 | 4 | anonymity in mind. |
|
5 | 5 | |
|
6 | 6 | Main repository: https://bitbucket.org/neko259/neboard/ |
|
7 | 7 | |
|
8 | 8 | Site: http://neboard.me/ |
|
9 | 9 | |
|
10 | 10 | # INSTALLATION # |
|
11 | 11 | |
|
12 | 12 | 1. Download application and move inside it: |
|
13 | 13 | |
|
14 | 14 | `hg clone https://bitbucket.org/neko259/neboard` |
|
15 | 15 | |
|
16 | 16 | `cd neboard` |
|
17 | 17 | |
|
18 | 18 | 2. Install all application dependencies: |
|
19 | 19 | |
|
20 | 20 | Some minimal system-wide depenencies: |
|
21 | 21 | |
|
22 | 22 | * python3 |
|
23 | 23 | * pip/pip3 |
|
24 | 24 | * jpeg |
|
25 | 25 | |
|
26 | 26 | Python dependencies: |
|
27 | 27 | |
|
28 | 28 | `pip3 install -r requirements.txt` |
|
29 | 29 | |
|
30 | 30 | You can use virtualenv to speed up the process or avoid conflicts. |
|
31 | 31 | |
|
32 | 32 | 3. Setup a database in `neboard/settings.py`. You can also change other settings like search engine. |
|
33 | 33 | |
|
34 | 34 | Depending on configured database and search engine, you need to install corresponding dependencies manually. |
|
35 | 35 | |
|
36 | Default database is *sqlite*, default search engine is *simple*. | |
|
36 | Default database is *sqlite*. If you want to change the database backend, refer to the django documentation for the correct settings. Please note that sqlite accepts only one connection at a time, so you won't be able to run 2 servers or a server and a sync at the same time. | |
|
37 | 37 | |
|
38 | 38 | 4. Setup SECRET_KEY to a secret value in `neboard/settings.py |
|
39 | 39 | 5. Run `./manage.py migrate` to apply all migrations |
|
40 |
6. Apply config changes to `boards/config/ |
|
|
40 | 6. Apply config changes to `boards/config/settings.ini`. You can see the default settings in `boards/config/default_config.ini`(do not delete or overwrite it). | |
|
41 | 41 | 7. If you want to use decetral engine, run `./manage.py generate_keypair` to generate keys |
|
42 | 42 | |
|
43 | 43 | # RUNNING # |
|
44 | 44 | |
|
45 | 45 | You can run the server using django default embedded webserver by running: |
|
46 | 46 | |
|
47 | 47 | ./manage.py runserver <address>:<port> |
|
48 | 48 | |
|
49 | 49 | See django-admin command help for details. |
|
50 | 50 | |
|
51 | 51 | Also consider using wsgi or fcgi interfaces on production servers. |
|
52 | 52 | |
|
53 | 53 | When running for the first time, you need to setup at least one section tag. |
|
54 | 54 | Go to the admin page and manually create one tag with "required" property set. |
|
55 | 55 | |
|
56 | 56 | # UPGRADE # |
|
57 | 57 | |
|
58 | 58 | 1. Backup your project data. |
|
59 | 59 | 2. Copy the project contents over the old project directory |
|
60 | 60 | 3. Run migrations by `./manage.py migrate` |
|
61 | 61 | |
|
62 | 62 | You can also just clone the mercurial project and pull it to update |
|
63 | 63 | |
|
64 | 64 | # CONCLUSION # |
|
65 | 65 | |
|
66 | 66 | Enjoy our software and thank you! |
General Comments 0
You need to be logged in to leave comments.
Login now