import re import logging import xml.etree.ElementTree as ET import httplib2 from django.core.management import BaseCommand from django.utils.dateparse import parse_datetime from boards.models import GlobalId from boards.models.post.sync import SyncManager, TAG_ID, TAG_UPDATE_TIME __author__ = 'neko259' REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)') class Command(BaseCommand): help = 'Send a sync or get request to the server.' def add_arguments(self, parser): parser.add_argument('url', type=str, help='Server root url') parser.add_argument('--global-id', type=str, default='', help='Post global ID') parser.add_argument('--split-query', type=int, default=1, help='Split GET query into separate by the given' ' number of posts in one') parser.add_argument('--thread', type=int, help='Get posts of one specific thread') parser.add_argument('--tags', type=str, help='Get posts of the tags, comma-separated') parser.add_argument('--time-from', type=str, help='Get posts from the given timestamp') def handle(self, *args, **options): logger = logging.getLogger('boards.sync') url = options.get('url') list_url = url + 'api/sync/list/' get_url = url + 'api/sync/get/' file_url = url[:-1] global_id_str = options.get('global_id') if global_id_str: match = REGEX_GLOBAL_ID.match(global_id_str) if match: key_type = match.group(1) key = match.group(2) local_id = match.group(3) global_id = GlobalId(key_type=key_type, key=key, local_id=local_id) xml = SyncManager.generate_request_get([global_id]) h = httplib2.Http() response, content = h.request(get_url, method="POST", body=xml) SyncManager.parse_response_get(content, file_url) else: raise Exception('Invalid global ID') else: logger.info('Running LIST request...') h = httplib2.Http() tags = [] tags_str = options.get('tags') if tags_str: tags = tags_str.split(',') xml = SyncManager.generate_request_list( opening_post=options.get('thread'), tags=tags, timestamp_from=options.get('time_from')).encode() response, content = h.request(list_url, method="POST", body=xml) if response.status != 200: raise Exception('Server returned error {}'.format(response.status)) logger.info('Processing response...') root = ET.fromstring(content) status = root.findall('status')[0].text if status == 'success': ids_to_sync = list() models = root.findall('models')[0] for model in models: tag_id = model.find(TAG_ID) global_id, exists = GlobalId.from_xml_element(tag_id) tag_update_time = model.find(TAG_UPDATE_TIME) if tag_update_time: update_time = tag_update_time.text else: update_time = None if not exists or update_time is None or global_id.post.last_edit_time < parse_datetime(update_time): logger.debug('Processed (+) post {}'.format(global_id)) ids_to_sync.append(global_id) else: logger.debug('* Processed (-) post {}'.format(global_id)) logger.info('Starting sync...') if len(ids_to_sync) > 0: limit = options.get('split_query', len(ids_to_sync)) for offset in range(0, len(ids_to_sync), limit): xml = SyncManager.generate_request_get(ids_to_sync[offset:offset + limit]) h = httplib2.Http() logger.info('Running GET request...') response, content = h.request(get_url, method="POST", body=xml) logger.info('Processing response...') SyncManager.parse_response_get(content, file_url) logger.info('Sync completed successfully') else: logger.info('Nothing to get, everything synced') else: raise Exception('Invalid response status')