|
|
import re
|
|
|
import logging
|
|
|
import xml.etree.ElementTree as ET
|
|
|
|
|
|
import httplib2
|
|
|
from django.core.management import BaseCommand
|
|
|
from django.utils.dateparse import parse_datetime
|
|
|
|
|
|
from boards.models import GlobalId
|
|
|
from boards.models.post.sync import SyncManager, TAG_ID, TAG_UPDATE_TIME
|
|
|
|
|
|
__author__ = 'neko259'
|
|
|
|
|
|
|
|
|
REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)')
|
|
|
|
|
|
|
|
|
class Command(BaseCommand):
|
|
|
help = 'Send a sync or get request to the server.'
|
|
|
|
|
|
def add_arguments(self, parser):
|
|
|
parser.add_argument('url', type=str, help='Server root url')
|
|
|
parser.add_argument('--global-id', type=str, default='',
|
|
|
help='Post global ID')
|
|
|
parser.add_argument('--split-query', type=int, default=1,
|
|
|
help='Split GET query into separate by the given'
|
|
|
' number of posts in one')
|
|
|
parser.add_argument('--thread', type=int,
|
|
|
help='Get posts of one specific thread')
|
|
|
parser.add_argument('--tags', type=str,
|
|
|
help='Get posts of the tags, comma-separated')
|
|
|
parser.add_argument('--time-from', type=str,
|
|
|
help='Get posts from the given timestamp')
|
|
|
|
|
|
def handle(self, *args, **options):
|
|
|
logger = logging.getLogger('boards.sync')
|
|
|
|
|
|
url = options.get('url')
|
|
|
|
|
|
list_url = url + 'api/sync/list/'
|
|
|
get_url = url + 'api/sync/get/'
|
|
|
file_url = url[:-1]
|
|
|
|
|
|
global_id_str = options.get('global_id')
|
|
|
if global_id_str:
|
|
|
match = REGEX_GLOBAL_ID.match(global_id_str)
|
|
|
if match:
|
|
|
key_type = match.group(1)
|
|
|
key = match.group(2)
|
|
|
local_id = match.group(3)
|
|
|
|
|
|
global_id = GlobalId(key_type=key_type, key=key,
|
|
|
local_id=local_id)
|
|
|
|
|
|
xml = SyncManager.generate_request_get([global_id])
|
|
|
h = httplib2.Http()
|
|
|
response, content = h.request(get_url, method="POST", body=xml)
|
|
|
|
|
|
SyncManager.parse_response_get(content, file_url)
|
|
|
else:
|
|
|
raise Exception('Invalid global ID')
|
|
|
else:
|
|
|
logger.info('Running LIST request...')
|
|
|
h = httplib2.Http()
|
|
|
|
|
|
tags = []
|
|
|
tags_str = options.get('tags')
|
|
|
if tags_str:
|
|
|
tags = tags_str.split(',')
|
|
|
|
|
|
xml = SyncManager.generate_request_list(
|
|
|
opening_post=options.get('thread'), tags=tags,
|
|
|
timestamp_from=options.get('time_from')).encode()
|
|
|
response, content = h.request(list_url, method="POST", body=xml)
|
|
|
if response.status != 200:
|
|
|
raise Exception('Server returned error {}'.format(response.status))
|
|
|
|
|
|
logger.info('Processing response...')
|
|
|
|
|
|
root = ET.fromstring(content)
|
|
|
status = root.findall('status')[0].text
|
|
|
if status == 'success':
|
|
|
ids_to_sync = list()
|
|
|
|
|
|
models = root.findall('models')[0]
|
|
|
for model in models:
|
|
|
tag_id = model.find(TAG_ID)
|
|
|
global_id, exists = GlobalId.from_xml_element(tag_id)
|
|
|
tag_update_time = model.find(TAG_UPDATE_TIME)
|
|
|
if tag_update_time:
|
|
|
update_time = tag_update_time.text
|
|
|
else:
|
|
|
update_time = None
|
|
|
if not exists or update_time is None or global_id.post.last_edit_time < parse_datetime(update_time):
|
|
|
logger.debug('Processed (+) post {}'.format(global_id))
|
|
|
ids_to_sync.append(global_id)
|
|
|
else:
|
|
|
logger.debug('* Processed (-) post {}'.format(global_id))
|
|
|
logger.info('Starting sync...')
|
|
|
|
|
|
if len(ids_to_sync) > 0:
|
|
|
limit = options.get('split_query', len(ids_to_sync))
|
|
|
for offset in range(0, len(ids_to_sync), limit):
|
|
|
xml = SyncManager.generate_request_get(ids_to_sync[offset:offset + limit])
|
|
|
h = httplib2.Http()
|
|
|
logger.info('Running GET request...')
|
|
|
response, content = h.request(get_url, method="POST", body=xml)
|
|
|
logger.info('Processing response...')
|
|
|
|
|
|
SyncManager.parse_response_get(content, file_url)
|
|
|
|
|
|
logger.info('Sync completed successfully')
|
|
|
else:
|
|
|
logger.info('Nothing to get, everything synced')
|
|
|
else:
|
|
|
raise Exception('Invalid response status')
|
|
|
|
