##// END OF EJS Templates
Fixed sync without specifying tags filter
neko259 -
r1845:b8a95ef7 default
parent child Browse files
Show More
@@ -1,104 +1,110 b''
1 import re
1 import re
2 import logging
2 import logging
3 import xml.etree.ElementTree as ET
3 import xml.etree.ElementTree as ET
4
4
5 import httplib2
5 import httplib2
6 from django.core.management import BaseCommand
6 from django.core.management import BaseCommand
7
7
8 from boards.models import GlobalId
8 from boards.models import GlobalId
9 from boards.models.post.sync import SyncManager, TAG_ID, TAG_VERSION
9 from boards.models.post.sync import SyncManager, TAG_ID, TAG_VERSION
10
10
11 __author__ = 'neko259'
11 __author__ = 'neko259'
12
12
13
13
14 REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)')
14 REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)')
15
15
16
16
17 class Command(BaseCommand):
17 class Command(BaseCommand):
18 help = 'Send a sync or get request to the server.'
18 help = 'Send a sync or get request to the server.'
19
19
20 def add_arguments(self, parser):
20 def add_arguments(self, parser):
21 parser.add_argument('url', type=str, help='Server root url')
21 parser.add_argument('url', type=str, help='Server root url')
22 parser.add_argument('--global-id', type=str, default='',
22 parser.add_argument('--global-id', type=str, default='',
23 help='Post global ID')
23 help='Post global ID')
24 parser.add_argument('--split-query', type=int, default=1,
24 parser.add_argument('--split-query', type=int, default=1,
25 help='Split GET query into separate by the given'
25 help='Split GET query into separate by the given'
26 ' number of posts in one')
26 ' number of posts in one')
27 parser.add_argument('--thread', type=int,
27 parser.add_argument('--thread', type=int,
28 help='Get posts of one specific thread')
28 help='Get posts of one specific thread')
29 parser.add_argument('--tags', type=str,
29 parser.add_argument('--tags', type=str,
30 help='Get posts of the tags, comma-separated')
30 help='Get posts of the tags, comma-separated')
31
31
32 def handle(self, *args, **options):
32 def handle(self, *args, **options):
33 logger = logging.getLogger('boards.sync')
33 logger = logging.getLogger('boards.sync')
34
34
35 url = options.get('url')
35 url = options.get('url')
36
36
37 list_url = url + 'api/sync/list/'
37 list_url = url + 'api/sync/list/'
38 get_url = url + 'api/sync/get/'
38 get_url = url + 'api/sync/get/'
39 file_url = url[:-1]
39 file_url = url[:-1]
40
40
41 global_id_str = options.get('global_id')
41 global_id_str = options.get('global_id')
42 if global_id_str:
42 if global_id_str:
43 match = REGEX_GLOBAL_ID.match(global_id_str)
43 match = REGEX_GLOBAL_ID.match(global_id_str)
44 if match:
44 if match:
45 key_type = match.group(1)
45 key_type = match.group(1)
46 key = match.group(2)
46 key = match.group(2)
47 local_id = match.group(3)
47 local_id = match.group(3)
48
48
49 global_id = GlobalId(key_type=key_type, key=key,
49 global_id = GlobalId(key_type=key_type, key=key,
50 local_id=local_id)
50 local_id=local_id)
51
51
52 xml = SyncManager.generate_request_get([global_id])
52 xml = SyncManager.generate_request_get([global_id])
53 h = httplib2.Http()
53 h = httplib2.Http()
54 response, content = h.request(get_url, method="POST", body=xml)
54 response, content = h.request(get_url, method="POST", body=xml)
55
55
56 SyncManager.parse_response_get(content, file_url)
56 SyncManager.parse_response_get(content, file_url)
57 else:
57 else:
58 raise Exception('Invalid global ID')
58 raise Exception('Invalid global ID')
59 else:
59 else:
60 logger.info('Running LIST request...')
60 logger.info('Running LIST request...')
61 h = httplib2.Http()
61 h = httplib2.Http()
62
63 tags = []
64 tags_str = options.get('tags')
65 if tags_str:
66 tags = tags_str.split(',')
67
62 xml = SyncManager.generate_request_list(
68 xml = SyncManager.generate_request_list(
63 opening_post=options.get('thread'), tags=options.get('tags').split(',')).encode()
69 opening_post=options.get('thread'), tags=tags).encode()
64 response, content = h.request(list_url, method="POST", body=xml)
70 response, content = h.request(list_url, method="POST", body=xml)
65 if response.status != 200:
71 if response.status != 200:
66 raise Exception('Server returned error {}'.format(response.status))
72 raise Exception('Server returned error {}'.format(response.status))
67
73
68 logger.info('Processing response...')
74 logger.info('Processing response...')
69
75
70 root = ET.fromstring(content)
76 root = ET.fromstring(content)
71 status = root.findall('status')[0].text
77 status = root.findall('status')[0].text
72 if status == 'success':
78 if status == 'success':
73 ids_to_sync = list()
79 ids_to_sync = list()
74
80
75 models = root.findall('models')[0]
81 models = root.findall('models')[0]
76 for model in models:
82 for model in models:
77 tag_id = model.find(TAG_ID)
83 tag_id = model.find(TAG_ID)
78 global_id, exists = GlobalId.from_xml_element(tag_id)
84 global_id, exists = GlobalId.from_xml_element(tag_id)
79 tag_version = model.find(TAG_VERSION)
85 tag_version = model.find(TAG_VERSION)
80 if tag_version is not None:
86 if tag_version is not None:
81 version = int(tag_version.text) or 1
87 version = int(tag_version.text) or 1
82 else:
88 else:
83 version = 1
89 version = 1
84 if not exists or global_id.post.version < version:
90 if not exists or global_id.post.version < version:
85 logger.debug('Processed (+) post {}'.format(global_id))
91 logger.debug('Processed (+) post {}'.format(global_id))
86 ids_to_sync.append(global_id)
92 ids_to_sync.append(global_id)
87 else:
93 else:
88 logger.debug('* Processed (-) post {}'.format(global_id))
94 logger.debug('* Processed (-) post {}'.format(global_id))
89 logger.info('Starting sync...')
95 logger.info('Starting sync...')
90
96
91 if len(ids_to_sync) > 0:
97 if len(ids_to_sync) > 0:
92 limit = options.get('split_query', len(ids_to_sync))
98 limit = options.get('split_query', len(ids_to_sync))
93 for offset in range(0, len(ids_to_sync), limit):
99 for offset in range(0, len(ids_to_sync), limit):
94 xml = SyncManager.generate_request_get(ids_to_sync[offset:offset + limit])
100 xml = SyncManager.generate_request_get(ids_to_sync[offset:offset + limit])
95 h = httplib2.Http()
101 h = httplib2.Http()
96 logger.info('Running GET request...')
102 logger.info('Running GET request...')
97 response, content = h.request(get_url, method="POST", body=xml)
103 response, content = h.request(get_url, method="POST", body=xml)
98 logger.info('Processing response...')
104 logger.info('Processing response...')
99
105
100 SyncManager.parse_response_get(content, file_url)
106 SyncManager.parse_response_get(content, file_url)
101 else:
107 else:
102 logger.info('Nothing to get, everything synced')
108 logger.info('Nothing to get, everything synced')
103 else:
109 else:
104 raise Exception('Invalid response status')
110 raise Exception('Invalid response status')
General Comments 0
You need to be logged in to leave comments. Login now