##// END OF EJS Templates
Use tripcode from settings when fetching posts from sources
neko259 -
r1973:399be5a4 default
parent child Browse files
Show More
@@ -1,591 +1,590 b''
1 1 import hashlib
2 2 import logging
3 3 import re
4 4 import time
5 5
6 6 import pytz
7 7
8 8 from PIL import Image
9 9
10 10 from django import forms
11 11 from django.core.files.uploadedfile import SimpleUploadedFile, UploadedFile
12 12 from django.forms.utils import ErrorList
13 13 from django.utils.translation import ugettext_lazy as _, ungettext_lazy
14 14 from django.core.files.images import get_image_dimensions
15 15 from django.core.cache import cache
16 16
17 17 import boards.settings as board_settings
18 18 import neboard
19 19 from boards import utils
20 20 from boards.abstracts.constants import REGEX_TAGS
21 21 from boards.abstracts.sticker_factory import get_attachment_by_alias
22 22 from boards.abstracts.settingsmanager import get_settings_manager
23 23 from boards.forms.fields import UrlFileField
24 24 from boards.mdx_neboard import formatters
25 25 from boards.models import Attachment
26 26 from boards.models import Tag
27 27 from boards.models.attachment import StickerPack
28 28 from boards.models.attachment.downloaders import download, REGEX_MAGNET
29 29 from boards.models.post import TITLE_MAX_LENGTH
30 30 from boards.utils import validate_file_size, get_file_mimetype, \
31 FILE_EXTENSION_DELIMITER
31 FILE_EXTENSION_DELIMITER, get_tripcode_from_text
32 32 from boards.models.attachment.viewers import FILE_TYPES_IMAGE
33 33 from neboard import settings
34 34
35 35 SECTION_FORMS = 'Forms'
36 36
37 37 POW_HASH_LENGTH = 16
38 38 POW_LIFE_MINUTES = 5
39 39
40 40 REGEX_USERNAMES = re.compile(r'^[\w\s\d,]+$', re.UNICODE)
41 41 REGEX_URL = re.compile(r'^(http|https|ftp):\/\/', re.UNICODE)
42 42
43 43 VETERAN_POSTING_DELAY = 5
44 44
45 45 ATTRIBUTE_PLACEHOLDER = 'placeholder'
46 46 ATTRIBUTE_ROWS = 'rows'
47 47
48 48 LAST_POST_TIME = 'last_post_time'
49 49 LAST_LOGIN_TIME = 'last_login_time'
50 50 TEXT_PLACEHOLDER = _('Type message here. Use formatting panel for more advanced usage.')
51 51 TAGS_PLACEHOLDER = _('music images i_dont_like_tags')
52 52
53 53 LABEL_TITLE = _('Title')
54 54 LABEL_TEXT = _('Text')
55 55 LABEL_TAG = _('Tag')
56 56 LABEL_SEARCH = _('Search')
57 57 LABEL_FILE = _('File')
58 58 LABEL_DUPLICATES = _('Check for duplicates')
59 59 LABEL_URL = _('Do not download URLs')
60 60
61 61 ERROR_SPEED = 'Please wait %(delay)d second before sending message'
62 62 ERROR_SPEED_PLURAL = 'Please wait %(delay)d seconds before sending message'
63 63 ERROR_MANY_FILES = 'You can post no more than %(files)d file.'
64 64 ERROR_MANY_FILES_PLURAL = 'You can post no more than %(files)d files.'
65 65 ERROR_DUPLICATES = 'Some files are already present on the board.'
66 66
67 67 TAG_MAX_LENGTH = 20
68 68
69 69 TEXTAREA_ROWS = 4
70 70
71 71 TRIPCODE_DELIM = '##'
72 72
73 73 # TODO Maybe this may be converted into the database table?
74 74 MIMETYPE_EXTENSIONS = {
75 75 'image/jpeg': 'jpeg',
76 76 'image/png': 'png',
77 77 'image/gif': 'gif',
78 78 'video/webm': 'webm',
79 79 'application/pdf': 'pdf',
80 80 'x-diff': 'diff',
81 81 'image/svg+xml': 'svg',
82 82 'application/x-shockwave-flash': 'swf',
83 83 'image/x-ms-bmp': 'bmp',
84 84 'image/bmp': 'bmp',
85 85 }
86 86
87 87 DOWN_MODE_DOWNLOAD = 'DOWNLOAD'
88 88 DOWN_MODE_DOWNLOAD_UNIQUE = 'DOWNLOAD_UNIQUE'
89 89 DOWN_MODE_URL = 'URL'
90 90 DOWN_MODE_TRY = 'TRY'
91 91
92 92
93 93 logger = logging.getLogger('boards.forms')
94 94
95 95
96 96 def get_timezones():
97 97 timezones = []
98 98 for tz in pytz.common_timezones:
99 99 timezones.append((tz, tz),)
100 100 return timezones
101 101
102 102
103 103 class FormatPanel(forms.Textarea):
104 104 """
105 105 Panel for text formatting. Consists of buttons to add different tags to the
106 106 form text area.
107 107 """
108 108
109 109 def render(self, name, value, attrs=None):
110 110 output = '<div id="mark-panel">'
111 111 for formatter in formatters:
112 112 output += '<span class="mark_btn"' + \
113 113 ' onClick="addMarkToMsg(\'' + formatter.format_left + \
114 114 '\', \'' + formatter.format_right + '\')">' + \
115 115 formatter.preview_left + formatter.name + \
116 116 formatter.preview_right + '</span>'
117 117
118 118 output += '</div>'
119 119 output += super(FormatPanel, self).render(name, value, attrs=attrs)
120 120
121 121 return output
122 122
123 123
124 124 class PlainErrorList(ErrorList):
125 125 def __unicode__(self):
126 126 return self.as_text()
127 127
128 128 def as_text(self):
129 129 return ''.join(['(!) %s ' % e for e in self])
130 130
131 131
132 132 class NeboardForm(forms.Form):
133 133 """
134 134 Form with neboard-specific formatting.
135 135 """
136 136 required_css_class = 'required-field'
137 137
138 138 def as_div(self):
139 139 """
140 140 Returns this form rendered as HTML <as_div>s.
141 141 """
142 142
143 143 return self._html_output(
144 144 # TODO Do not show hidden rows in the list here
145 145 normal_row='<div class="form-row">'
146 146 '<div class="form-label">'
147 147 '%(label)s'
148 148 '</div>'
149 149 '<div class="form-input">'
150 150 '%(field)s'
151 151 '</div>'
152 152 '</div>'
153 153 '<div class="form-row">'
154 154 '%(help_text)s'
155 155 '</div>',
156 156 error_row='<div class="form-row">'
157 157 '<div class="form-label"></div>'
158 158 '<div class="form-errors">%s</div>'
159 159 '</div>',
160 160 row_ender='</div>',
161 161 help_text_html='%s',
162 162 errors_on_separate_row=True)
163 163
164 164 def as_json_errors(self):
165 165 errors = []
166 166
167 167 for name, field in list(self.fields.items()):
168 168 if self[name].errors:
169 169 errors.append({
170 170 'field': name,
171 171 'errors': self[name].errors.as_text(),
172 172 })
173 173
174 174 return errors
175 175
176 176
177 177 class PostForm(NeboardForm):
178 178
179 179 title = forms.CharField(max_length=TITLE_MAX_LENGTH, required=False,
180 180 label=LABEL_TITLE,
181 181 widget=forms.TextInput(
182 182 attrs={ATTRIBUTE_PLACEHOLDER: 'Title{}tripcode'.format(TRIPCODE_DELIM)}))
183 183 text = forms.CharField(
184 184 widget=FormatPanel(attrs={
185 185 ATTRIBUTE_PLACEHOLDER: TEXT_PLACEHOLDER,
186 186 ATTRIBUTE_ROWS: TEXTAREA_ROWS,
187 187 }),
188 188 required=False, label=LABEL_TEXT)
189 189 download_mode = forms.ChoiceField(
190 190 choices=(
191 191 (DOWN_MODE_TRY, _('Download or insert as URLs')),
192 192 (DOWN_MODE_DOWNLOAD, _('Download')),
193 193 (DOWN_MODE_DOWNLOAD_UNIQUE, _('Download and check for uniqueness')),
194 194 (DOWN_MODE_URL, _('Insert as URLs')),
195 195 ),
196 196 initial=DOWN_MODE_TRY,
197 197 label=_('File process mode'))
198 198 file = UrlFileField(required=False, label=LABEL_FILE)
199 199
200 200 # This field is for spam prevention only
201 201 email = forms.CharField(max_length=100, required=False, label=_('e-mail'),
202 202 widget=forms.TextInput(attrs={
203 203 'class': 'form-email'}))
204 204 subscribe = forms.BooleanField(required=False, label=_('Subscribe to thread'))
205 205
206 206 guess = forms.CharField(widget=forms.HiddenInput(), required=False)
207 207 timestamp = forms.CharField(widget=forms.HiddenInput(), required=False)
208 208 iteration = forms.CharField(widget=forms.HiddenInput(), required=False)
209 209
210 210 session = None
211 211 need_to_ban = False
212 212
213 213 def clean_title(self):
214 214 title = self.cleaned_data['title']
215 215 if title:
216 216 if len(title) > TITLE_MAX_LENGTH:
217 217 raise forms.ValidationError(_('Title must have less than %s '
218 218 'characters') %
219 219 str(TITLE_MAX_LENGTH))
220 220 return title
221 221
222 222 def clean_text(self):
223 223 text = self.cleaned_data['text'].strip()
224 224 if text:
225 225 max_length = board_settings.get_int(SECTION_FORMS, 'MaxTextLength')
226 226 if len(text) > max_length:
227 227 raise forms.ValidationError(_('Text must have less than %s '
228 228 'characters') % str(max_length))
229 229 return text
230 230
231 231 def clean_file(self):
232 232 return self._clean_files(self.cleaned_data['file'])
233 233
234 234 def clean(self):
235 235 cleaned_data = super(PostForm, self).clean()
236 236
237 237 if cleaned_data['email']:
238 238 if board_settings.get_bool(SECTION_FORMS, 'Autoban'):
239 239 self.need_to_ban = True
240 240 raise forms.ValidationError('A human cannot enter a hidden field')
241 241
242 242 if not self.errors:
243 243 self._clean_text_file()
244 244
245 245 limit_speed = board_settings.get_bool(SECTION_FORMS, 'LimitPostingSpeed')
246 246 limit_first = board_settings.get_bool(SECTION_FORMS, 'LimitFirstPosting')
247 247
248 248 settings_manager = get_settings_manager(self)
249 249 if not self.errors and limit_speed or (limit_first and not settings_manager.get_setting('confirmed_user')):
250 250 pow_difficulty = board_settings.get_int(SECTION_FORMS, 'PowDifficulty')
251 251 if pow_difficulty > 0:
252 252 # PoW-based
253 253 if cleaned_data['timestamp'] \
254 254 and cleaned_data['iteration'] and cleaned_data['guess'] \
255 255 and not settings_manager.get_setting('confirmed_user'):
256 256 self._validate_hash(cleaned_data['timestamp'], cleaned_data['iteration'], cleaned_data['guess'], cleaned_data['text'])
257 257 else:
258 258 # Time-based
259 259 self._validate_posting_speed()
260 260 settings_manager.set_setting('confirmed_user', True)
261 261 if self.cleaned_data['download_mode'] == DOWN_MODE_DOWNLOAD_UNIQUE:
262 262 self._check_file_duplicates(self.get_files())
263 263
264 264 return cleaned_data
265 265
266 266 def get_files(self):
267 267 """
268 268 Gets file from form or URL.
269 269 """
270 270
271 271 files = []
272 272 for file in self.cleaned_data['file']:
273 273 if isinstance(file, UploadedFile):
274 274 files.append(file)
275 275
276 276 return files
277 277
278 278 def get_file_urls(self):
279 279 files = []
280 280 for file in self.cleaned_data['file']:
281 281 if type(file) == str:
282 282 files.append(file)
283 283
284 284 return files
285 285
286 286 def get_tripcode(self):
287 287 title = self.cleaned_data['title']
288 288 if title is not None and TRIPCODE_DELIM in title:
289 code = title.split(TRIPCODE_DELIM, maxsplit=1)[1] + neboard.settings.SECRET_KEY
290 tripcode = hashlib.md5(code.encode()).hexdigest()
289 tripcode = get_tripcode_from_text(title.split(TRIPCODE_DELIM, maxsplit=1)[1])
291 290 else:
292 291 tripcode = ''
293 292 return tripcode
294 293
295 294 def get_title(self):
296 295 title = self.cleaned_data['title']
297 296 if title is not None and TRIPCODE_DELIM in title:
298 297 return title.split(TRIPCODE_DELIM, maxsplit=1)[0]
299 298 else:
300 299 return title
301 300
302 301 def get_images(self):
303 302 return self.cleaned_data.get('stickers', [])
304 303
305 304 def is_subscribe(self):
306 305 return self.cleaned_data['subscribe']
307 306
308 307 def _update_file_extension(self, file):
309 308 if file:
310 309 mimetype = get_file_mimetype(file)
311 310 extension = MIMETYPE_EXTENSIONS.get(mimetype)
312 311 if extension:
313 312 filename = file.name.split(FILE_EXTENSION_DELIMITER, 1)[0]
314 313 new_filename = filename + FILE_EXTENSION_DELIMITER + extension
315 314
316 315 file.name = new_filename
317 316 else:
318 317 logger.info('Unrecognized file mimetype: {}'.format(mimetype))
319 318
320 319 def _clean_files(self, inputs):
321 320 files = []
322 321
323 322 max_file_count = board_settings.get_int(SECTION_FORMS, 'MaxFileCount')
324 323 if len(inputs) > max_file_count:
325 324 raise forms.ValidationError(
326 325 ungettext_lazy(ERROR_MANY_FILES, ERROR_MANY_FILES,
327 326 max_file_count) % {'files': max_file_count})
328 327 for file_input in inputs:
329 328 if isinstance(file_input, UploadedFile):
330 329 files.append(self._clean_file_file(file_input))
331 330 else:
332 331 files.append(self._clean_file_url(file_input))
333 332
334 333 for file in files:
335 334 self._validate_image_dimensions(file)
336 335
337 336 return files
338 337
339 338 def _validate_image_dimensions(self, file):
340 339 if isinstance(file, UploadedFile):
341 340 mimetype = get_file_mimetype(file)
342 341 if mimetype.split('/')[-1] in FILE_TYPES_IMAGE:
343 342 Image.warnings.simplefilter('error', Image.DecompressionBombWarning)
344 343 try:
345 344 print(get_image_dimensions(file))
346 345 except Exception:
347 346 raise forms.ValidationError('Possible decompression bomb or large image.')
348 347
349 348 def _clean_file_file(self, file):
350 349 validate_file_size(file.size)
351 350 self._update_file_extension(file)
352 351
353 352 return file
354 353
355 354 def _clean_file_url(self, url):
356 355 file = None
357 356
358 357 if url:
359 358 mode = self.cleaned_data['download_mode']
360 359 if mode == DOWN_MODE_URL:
361 360 return url
362 361
363 362 try:
364 363 image = get_attachment_by_alias(url, self.session)
365 364 if image is not None:
366 365 if 'stickers' not in self.cleaned_data:
367 366 self.cleaned_data['stickers'] = []
368 367 self.cleaned_data['stickers'].append(image)
369 368 return
370 369
371 370 if file is None:
372 371 file = self._get_file_from_url(url)
373 372 if not file:
374 373 raise forms.ValidationError(_('Invalid URL'))
375 374 else:
376 375 validate_file_size(file.size)
377 376 self._update_file_extension(file)
378 377 except forms.ValidationError as e:
379 378 # Assume we will get the plain URL instead of a file and save it
380 379 if mode == DOWN_MODE_TRY and (REGEX_URL.match(url) or REGEX_MAGNET.match(url)):
381 380 logger.info('Error in forms: {}'.format(e))
382 381 return url
383 382 else:
384 383 raise e
385 384
386 385 return file
387 386
388 387 def _clean_text_file(self):
389 388 text = self.cleaned_data.get('text')
390 389 file = self.get_files()
391 390 file_url = self.get_file_urls()
392 391 images = self.get_images()
393 392
394 393 if (not text) and (not file) and (not file_url) and len(images) == 0:
395 394 error_message = _('Either text or file must be entered.')
396 395 self._add_general_error(error_message)
397 396
398 397 def _get_cache_key(self, key):
399 398 return '{}_{}'.format(self.session.session_key, key)
400 399
401 400 def _set_session_cache(self, key, value):
402 401 cache.set(self._get_cache_key(key), value)
403 402
404 403 def _get_session_cache(self, key):
405 404 return cache.get(self._get_cache_key(key))
406 405
407 406 def _get_last_post_time(self):
408 407 last = self._get_session_cache(LAST_POST_TIME)
409 408 if last is None:
410 409 last = self.session.get(LAST_POST_TIME)
411 410 return last
412 411
413 412 def _validate_posting_speed(self):
414 413 can_post = True
415 414
416 415 posting_delay = board_settings.get_int(SECTION_FORMS, 'PostingDelay')
417 416
418 417 if board_settings.get_bool(SECTION_FORMS, 'LimitPostingSpeed'):
419 418 now = time.time()
420 419
421 420 current_delay = 0
422 421
423 422 if LAST_POST_TIME not in self.session:
424 423 self.session[LAST_POST_TIME] = now
425 424
426 425 need_delay = True
427 426 else:
428 427 last_post_time = self._get_last_post_time()
429 428 current_delay = int(now - last_post_time)
430 429
431 430 need_delay = current_delay < posting_delay
432 431
433 432 self._set_session_cache(LAST_POST_TIME, now)
434 433
435 434 if need_delay:
436 435 delay = posting_delay - current_delay
437 436 error_message = ungettext_lazy(ERROR_SPEED, ERROR_SPEED_PLURAL,
438 437 delay) % {'delay': delay}
439 438 self._add_general_error(error_message)
440 439
441 440 can_post = False
442 441
443 442 if can_post:
444 443 self.session[LAST_POST_TIME] = now
445 444 else:
446 445 # Reset the time since posting failed
447 446 self._set_session_cache(LAST_POST_TIME, self.session[LAST_POST_TIME])
448 447
449 448 def _get_file_from_url(self, url: str) -> SimpleUploadedFile:
450 449 """
451 450 Gets an file file from URL.
452 451 """
453 452
454 453 try:
455 454 return download(url)
456 455 except forms.ValidationError as e:
457 456 raise e
458 457 except Exception as e:
459 458 raise forms.ValidationError(e)
460 459
461 460 def _validate_hash(self, timestamp: str, iteration: str, guess: str, message: str):
462 461 payload = timestamp + message.replace('\r\n', '\n')
463 462 difficulty = board_settings.get_int(SECTION_FORMS, 'PowDifficulty')
464 463 target = str(int(2 ** (POW_HASH_LENGTH * 3) / difficulty))
465 464 if len(target) < POW_HASH_LENGTH:
466 465 target = '0' * (POW_HASH_LENGTH - len(target)) + target
467 466
468 467 computed_guess = hashlib.sha256((payload + iteration).encode())\
469 468 .hexdigest()[0:POW_HASH_LENGTH]
470 469 if guess != computed_guess or guess > target:
471 470 self._add_general_error(_('Invalid PoW.'))
472 471
473 472 def _check_file_duplicates(self, files):
474 473 for file in files:
475 474 file_hash = utils.get_file_hash(file)
476 475 if Attachment.objects.get_existing_duplicate(file_hash, file):
477 476 self._add_general_error(_(ERROR_DUPLICATES))
478 477
479 478 def _add_general_error(self, message):
480 479 self.add_error('text', forms.ValidationError(message))
481 480
482 481
483 482 class ThreadForm(PostForm):
484 483
485 484 tags = forms.CharField(
486 485 widget=forms.TextInput(attrs={ATTRIBUTE_PLACEHOLDER: TAGS_PLACEHOLDER}),
487 486 max_length=100, label=_('Tags'), required=True)
488 487 monochrome = forms.BooleanField(label=_('Monochrome'), required=False)
489 488 stickerpack = forms.BooleanField(label=_('Sticker Pack'), required=False)
490 489
491 490 def clean_tags(self):
492 491 tags = self.cleaned_data['tags'].strip()
493 492
494 493 if not tags or not REGEX_TAGS.match(tags):
495 494 raise forms.ValidationError(
496 495 _('Inappropriate characters in tags.'))
497 496
498 497 default_tag_name = board_settings.get(SECTION_FORMS, 'DefaultTag')\
499 498 .strip().lower()
500 499
501 500 required_tag_exists = False
502 501 tag_set = set()
503 502 for tag_string in tags.split():
504 503 tag_name = tag_string.strip().lower()
505 504 if tag_name == default_tag_name:
506 505 required_tag_exists = True
507 506 tag, created = Tag.objects.get_or_create_with_alias(
508 507 name=tag_name, required=True)
509 508 else:
510 509 tag, created = Tag.objects.get_or_create_with_alias(name=tag_name)
511 510 tag_set.add(tag)
512 511
513 512 # If this is a new tag, don't check for its parents because nobody
514 513 # added them yet
515 514 if not created:
516 515 tag_set |= set(tag.get_all_parents())
517 516
518 517 for tag in tag_set:
519 518 if tag.required:
520 519 required_tag_exists = True
521 520 break
522 521
523 522 # Use default tag if no section exists
524 523 if not required_tag_exists:
525 524 default_tag, created = Tag.objects.get_or_create_with_alias(
526 525 name=default_tag_name, required=True)
527 526 tag_set.add(default_tag)
528 527
529 528 return tag_set
530 529
531 530 def clean(self):
532 531 cleaned_data = super(ThreadForm, self).clean()
533 532
534 533 return cleaned_data
535 534
536 535 def is_monochrome(self):
537 536 return self.cleaned_data['monochrome']
538 537
539 538 def clean_stickerpack(self):
540 539 stickerpack = self.cleaned_data['stickerpack']
541 540 if stickerpack:
542 541 tripcode = self.get_tripcode()
543 542 if not tripcode:
544 543 raise forms.ValidationError(_(
545 544 'Tripcode should be specified to own a stickerpack.'))
546 545 title = self.get_title()
547 546 if not title:
548 547 raise forms.ValidationError(_(
549 548 'Title should be specified as a stickerpack name.'))
550 549 if not REGEX_TAGS.match(title):
551 550 raise forms.ValidationError(_('Inappropriate sticker pack name.'))
552 551
553 552 existing_pack = StickerPack.objects.filter(name=title).first()
554 553 if existing_pack:
555 554 if existing_pack.tripcode != tripcode:
556 555 raise forms.ValidationError(_(
557 556 'A sticker pack with this name already exists and is'
558 557 ' owned by another tripcode.'))
559 558 if not existing_pack.tripcode:
560 559 raise forms.ValidationError(_(
561 560 'This sticker pack can only be updated by an '
562 561 'administrator.'))
563 562
564 563 return stickerpack
565 564
566 565 def is_stickerpack(self):
567 566 return self.cleaned_data['stickerpack']
568 567
569 568
570 569 class SettingsForm(NeboardForm):
571 570
572 571 theme = forms.ChoiceField(
573 572 choices=board_settings.get_list_dict('View', 'Themes'),
574 573 label=_('Theme'))
575 574 image_viewer = forms.ChoiceField(
576 575 choices=board_settings.get_list_dict('View', 'ImageViewers'),
577 576 label=_('Image view mode'))
578 577 username = forms.CharField(label=_('User name'), required=False)
579 578 timezone = forms.ChoiceField(choices=get_timezones(), label=_('Time zone'))
580 579
581 580 def clean_username(self):
582 581 username = self.cleaned_data['username']
583 582
584 583 if username and not REGEX_USERNAMES.match(username):
585 584 raise forms.ValidationError(_('Inappropriate characters.'))
586 585
587 586 return username
588 587
589 588
590 589 class SearchForm(NeboardForm):
591 590 query = forms.CharField(max_length=500, label=LABEL_SEARCH, required=False)
@@ -1,68 +1,74 b''
1 1 import feedparser
2 2 import logging
3 3 import calendar
4 4
5 5 from time import mktime
6 6 from datetime import datetime
7 7
8 8 from django.db import models, transaction
9 9 from django.utils.dateparse import parse_datetime
10 10 from django.utils.timezone import utc
11 11 from django.utils import timezone
12
12 13 from boards.models import Post
13 14 from boards.models.post import TITLE_MAX_LENGTH
15 from boards.utils import get_tripcode_from_text
16 from boards import settings
14 17
15 18
16 19 SOURCE_TYPE_MAX_LENGTH = 100
17 20 SOURCE_TYPE_RSS = 'RSS'
18 21 TYPE_CHOICES = (
19 22 (SOURCE_TYPE_RSS, SOURCE_TYPE_RSS),
20 23 )
21 24
22 25
23 26 class ThreadSource(models.Model):
24 27 class Meta:
25 28 app_label = 'boards'
26 29
27 30 name = models.TextField()
28 31 thread = models.ForeignKey('Thread')
29 32 timestamp = models.DateTimeField()
30 33 source = models.TextField()
31 34 source_type = models.CharField(max_length=SOURCE_TYPE_MAX_LENGTH,
32 35 choices=TYPE_CHOICES)
33 36
34 37 def __str__(self):
35 38 return self.name
36 39
37 40 @transaction.atomic
38 41 def fetch_latest_posts(self):
39 42 """Creates new posts with the info fetched since the timestamp."""
40 43 logger = logging.getLogger('boards.source')
41 44
42 45 if self.thread.is_archived():
43 46 logger.error('The thread {} is archived, please try another one'.format(self.thread))
44 47 else:
48 tripcode = get_tripcode_from_text(
49 settings.get('External', 'SourceFetcherTripcode'))
45 50 start_timestamp = self.timestamp
46 51 last_timestamp = start_timestamp
47 52 logger.info('Start timestamp is {}'.format(start_timestamp))
48 53 if self.thread.is_bumplimit():
49 54 logger.warn('The thread {} has reached its bumplimit, please create a new one'.format(self.thread))
50 55 if self.source_type == SOURCE_TYPE_RSS:
51 56 feed = feedparser.parse(self.source)
52 57 items = sorted(feed.entries, key=lambda entry: entry.published_parsed)
53 58 for item in items:
54 59 title = item.title[:TITLE_MAX_LENGTH]
55 60 timestamp = datetime.fromtimestamp(calendar.timegm(item.published_parsed), tz=utc)
56 61 if not timestamp:
57 62 logger.error('Invalid timestamp {} for {}'.format(item.published, title))
58 63 else:
59 64 if timestamp > last_timestamp:
60 65 last_timestamp = timestamp
61 66 if timestamp > start_timestamp:
62 Post.objects.create_post(title=title, text=item.description, thread=self.thread, file_urls=[item.link])
67 Post.objects.create_post(title=title, text=item.description,
68 thread=self.thread, file_urls=[item.link], tripcode=tripcode)
63 69 logger.info('Fetched item {} from {} into thread {}'.format(
64 70 title, self.name, self.thread))
65 71 logger.info('New timestamp is {}'.format(last_timestamp))
66 72 self.timestamp = last_timestamp
67 73 self.save(update_fields=['timestamp'])
68 74
@@ -1,143 +1,152 b''
1 1 """
2 2 This module contains helper functions and helper classes.
3 3 """
4 4 import time
5 5 import uuid
6 6
7 7 import hashlib
8 8 import magic
9 9 import os
10 10 from django import forms
11 11 from django.core.cache import cache
12 12 from django.db.models import Model
13 13 from django.template.defaultfilters import filesizeformat
14 14 from django.utils import timezone
15 15 from django.utils.translation import ugettext_lazy as _
16 16
17 17 import boards
18 from neboard import settings
18 19 from boards.abstracts.constants import FILE_DIRECTORY
19 20 from boards.settings import get_bool
20 21
21 22 CACHE_KEY_DELIMITER = '_'
22 23
23 24 HTTP_FORWARDED = 'HTTP_X_FORWARDED_FOR'
24 25 META_REMOTE_ADDR = 'REMOTE_ADDR'
25 26
26 27 SETTING_MESSAGES = 'Messages'
27 28 SETTING_ANON_MODE = 'AnonymousMode'
28 29
29 30 ANON_IP = '127.0.0.1'
30 31
31 32 FILE_EXTENSION_DELIMITER = '.'
32 33
33 34
34 35 def is_anonymous_mode():
35 36 return get_bool(SETTING_MESSAGES, SETTING_ANON_MODE)
36 37
37 38
38 39 def get_client_ip(request):
39 40 if is_anonymous_mode():
40 41 ip = ANON_IP
41 42 else:
42 43 x_forwarded_for = request.META.get(HTTP_FORWARDED)
43 44 if x_forwarded_for:
44 45 ip = x_forwarded_for.split(',')[-1].strip()
45 46 else:
46 47 ip = request.META.get(META_REMOTE_ADDR)
47 48 return ip
48 49
49 50
50 51 # TODO The output format is not epoch because it includes microseconds
51 52 def datetime_to_epoch(datetime):
52 53 return int(time.mktime(timezone.localtime(
53 54 datetime,timezone.get_current_timezone()).timetuple())
54 55 * 1000000 + datetime.microsecond)
55 56
56 57
57 58 # TODO Test this carefully
58 59 def cached_result(key_method=None):
59 60 """
60 61 Caches method result in the Django's cache system, persisted by object name,
61 62 object name, model id if object is a Django model, args and kwargs if any.
62 63 """
63 64 def _cached_result(function):
64 65 def inner_func(obj, *args, **kwargs):
65 66 cache_key_params = [obj.__class__.__name__, function.__name__]
66 67
67 68 cache_key_params += args
68 69 for key, value in kwargs:
69 70 cache_key_params.append(key + ':' + value)
70 71
71 72 if isinstance(obj, Model):
72 73 cache_key_params.append(str(obj.id))
73 74
74 75 if key_method is not None:
75 76 cache_key_params += [str(arg) for arg in key_method(obj)]
76 77
77 78 cache_key = CACHE_KEY_DELIMITER.join(cache_key_params)
78 79
79 80 persisted_result = cache.get(cache_key)
80 81 if persisted_result is not None:
81 82 result = persisted_result
82 83 else:
83 84 result = function(obj, *args, **kwargs)
84 85 if result is not None:
85 86 cache.set(cache_key, result)
86 87
87 88 return result
88 89
89 90 return inner_func
90 91 return _cached_result
91 92
92 93
93 94 def get_file_hash(file) -> str:
94 95 md5 = hashlib.md5()
95 96 for chunk in file.chunks():
96 97 md5.update(chunk)
97 98 return md5.hexdigest()
98 99
99 100
100 101 def validate_file_size(size: int):
101 102 max_size = boards.settings.get_int('Forms', 'MaxFileSize')
102 103 if 0 < max_size < size:
103 104 raise forms.ValidationError(
104 105 _('File must be less than %s but is %s.')
105 106 % (filesizeformat(max_size), filesizeformat(size)))
106 107
107 108
108 109 def get_extension(filename):
109 110 return filename.split(FILE_EXTENSION_DELIMITER)[-1:][0]
110 111
111 112
112 113 def get_upload_filename(model_instance, old_filename):
113 114 extension = get_extension(old_filename)
114 115 new_name = '{}.{}'.format(uuid.uuid4(), extension)
115 116
116 117 return os.path.join(FILE_DIRECTORY, new_name)
117 118
118 119
119 120 def get_file_mimetype(file) -> str:
120 121 buf = b''
121 122 for chunk in file.chunks():
122 123 buf += chunk
123 124
124 125 file_type = magic.from_buffer(buf, mime=True)
125 126 if file_type is None:
126 127 file_type = 'application/octet-stream'
127 128 elif type(file_type) == bytes:
128 129 file_type = file_type.decode()
129 130 return file_type
130 131
131 132
132 133 def get_domain(url: str) -> str:
133 134 """
134 135 Gets domain from an URL with random number of domain levels.
135 136 """
136 137 domain_parts = url.split('/')
137 138 if len(domain_parts) >= 2:
138 139 full_domain = domain_parts[2]
139 140 else:
140 141 full_domain = ''
141 142
142 143 return full_domain
143 144
145
146 def get_tripcode_from_text(text: str) -> str:
147 tripcode = ''
148 if text:
149 code = text + settings.SECRET_KEY
150 tripcode = hashlib.md5(code.encode()).hexdigest()
151 return tripcode
152
General Comments 0
You need to be logged in to leave comments. Login now