boards/management/commands/sync_with_server.py

import re
import logging
import xml.etree.ElementTree as ET

import httplib2
from django.core.management import BaseCommand
from django.utils.dateparse import parse_datetime

from boards.models import GlobalId, KeyPair
from boards.models.post.sync import SyncManager, TAG_ID, TAG_UPDATE_TIME

__author__ = 'neko259'


REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)')


class Command(BaseCommand):
    help = 'Send a sync or get request to the server.'

    def add_arguments(self, parser):
        parser.add_argument('url', type=str, help='Server root url')
        parser.add_argument('--global-id', type=str, default='',
                            help='Post global ID')
        parser.add_argument('--split-query', type=int, default=1,
                            help='Split GET query into separate by the given'
                                 ' number of posts in one')
        parser.add_argument('--thread', type=int,
                            help='Get posts of one specific thread')
        parser.add_argument('--tags', type=str,
                help='Get posts of the tags, comma-separated')
        parser.add_argument('--time-from', type=str,
                help='Get posts from the given timestamp')

    def handle(self, *args, **options):
        logger = logging.getLogger('boards.sync')

        url = options.get('url')

        list_url = url + 'api/sync/list/'
        get_url = url + 'api/sync/get/'
        file_url = url[:-1]

        global_id_str = options.get('global_id')
        if global_id_str:
            match = REGEX_GLOBAL_ID.match(global_id_str)
            if match:
                key_type = match.group(1)
                key = match.group(2)
                local_id = match.group(3)

                global_id = GlobalId(key_type=key_type, key=key,
                                     local_id=local_id)

                xml = SyncManager.generate_request_get([global_id])
                h = httplib2.Http()
                response, content = h.request(get_url, method="POST", body=xml)

                SyncManager.parse_response_get(content, file_url)
            else:
                raise Exception('Invalid global ID')
        else:
            logger.info('Running LIST request...')
            h = httplib2.Http()

            tags = []
            tags_str = options.get('tags')
            if tags_str:
                tags = tags_str.split(',')

            timestamp_str = options.get('time_from')
            timestamp = None
            if timestamp_str:
                timestamp = parse_datetime(timestamp_str)

            xml = SyncManager.generate_request_list(
                opening_post=options.get('thread'), tags=tags,
                timestamp_from=timestamp).encode()
            response, content = h.request(list_url, method="POST", body=xml)
            if response.status != 200:
                raise Exception('Server returned error {}'.format(response.status))

            logger.info('Processing response...')

            root = ET.fromstring(content)
            status = root.findall('status')[0].text
            if status == 'success':
                ids_to_sync = list()

                models = root.findall('models')[0]
                for model in models:
                    self.add_to_sync_list(ids_to_sync, logger, model)
                logger.info('Starting sync...')

                if len(ids_to_sync) > 0:
                    limit = options.get('split_query', len(ids_to_sync))
                    for offset in range(0, len(ids_to_sync), limit):
                        xml = SyncManager.generate_request_get(ids_to_sync[offset:offset + limit])
                        h = httplib2.Http()
                        logger.info('Running GET request...')
                        response, content = h.request(get_url, method="POST", body=xml)
                        logger.info('Processing response...')

                        SyncManager.parse_response_get(content, file_url)

                        logger.info('Sync completed successfully')
                else:
                    logger.info('Nothing to get, everything synced')
            else:
                raise Exception('Invalid response status')

    def add_to_sync_list(self, ids_to_sync, logger, model):
        tag_id = model.find(TAG_ID)
        global_id, exists = GlobalId.from_xml_element(tag_id)
        from_this_board = self._is_from_this_board(global_id)
        if from_this_board:
            # If the post originates from this board, no need to process
            # it again, nobody else could modify it
            logger.debug('NO SYNC Processed post {}'.format(global_id))
        else:
            tag_update_time = model.find(TAG_UPDATE_TIME)
            if tag_update_time:
                update_time = tag_update_time.text
            else:
                update_time = None
            if not exists or update_time is None or global_id.post.last_edit_time < parse_datetime(
                    update_time):
                logger.debug('SYNC Processed post {}'.format(global_id))
                ids_to_sync.append(global_id)
            else:
                logger.debug('NO SYNC Processed post {}'.format(global_id))

    def _is_from_this_board(self, global_id):
        from_this_board = KeyPair.objects.filter(
            key_type=global_id.key_type,
            public_key=global_id.key).exists()
        return from_this_board