Show More
@@ -1,113 +1,115 b'' | |||||
1 | import re |
|
1 | import re | |
2 | import logging |
|
2 | import logging | |
3 | import xml.etree.ElementTree as ET |
|
3 | import xml.etree.ElementTree as ET | |
4 |
|
4 | |||
5 | import httplib2 |
|
5 | import httplib2 | |
6 | from django.core.management import BaseCommand |
|
6 | from django.core.management import BaseCommand | |
7 |
|
7 | |||
8 | from boards.models import GlobalId |
|
8 | from boards.models import GlobalId | |
9 | from boards.models.post.sync import SyncManager, TAG_ID, TAG_VERSION |
|
9 | from boards.models.post.sync import SyncManager, TAG_ID, TAG_VERSION | |
10 |
|
10 | |||
11 | __author__ = 'neko259' |
|
11 | __author__ = 'neko259' | |
12 |
|
12 | |||
13 |
|
13 | |||
14 | REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)') |
|
14 | REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)') | |
15 |
|
15 | |||
16 |
|
16 | |||
17 | class Command(BaseCommand): |
|
17 | class Command(BaseCommand): | |
18 | help = 'Send a sync or get request to the server.' |
|
18 | help = 'Send a sync or get request to the server.' | |
19 |
|
19 | |||
20 | def add_arguments(self, parser): |
|
20 | def add_arguments(self, parser): | |
21 | parser.add_argument('url', type=str, help='Server root url') |
|
21 | parser.add_argument('url', type=str, help='Server root url') | |
22 | parser.add_argument('--global-id', type=str, default='', |
|
22 | parser.add_argument('--global-id', type=str, default='', | |
23 | help='Post global ID') |
|
23 | help='Post global ID') | |
24 | parser.add_argument('--split-query', type=int, default=1, |
|
24 | parser.add_argument('--split-query', type=int, default=1, | |
25 | help='Split GET query into separate by the given' |
|
25 | help='Split GET query into separate by the given' | |
26 | ' number of posts in one') |
|
26 | ' number of posts in one') | |
27 | parser.add_argument('--thread', type=int, |
|
27 | parser.add_argument('--thread', type=int, | |
28 | help='Get posts of one specific thread') |
|
28 | help='Get posts of one specific thread') | |
29 | parser.add_argument('--tags', type=str, |
|
29 | parser.add_argument('--tags', type=str, | |
30 | help='Get posts of the tags, comma-separated') |
|
30 | help='Get posts of the tags, comma-separated') | |
31 | parser.add_argument('--time-from', type=str, |
|
31 | parser.add_argument('--time-from', type=str, | |
32 | help='Get posts from the given timestamp') |
|
32 | help='Get posts from the given timestamp') | |
33 |
|
33 | |||
34 | def handle(self, *args, **options): |
|
34 | def handle(self, *args, **options): | |
35 | logger = logging.getLogger('boards.sync') |
|
35 | logger = logging.getLogger('boards.sync') | |
36 |
|
36 | |||
37 | url = options.get('url') |
|
37 | url = options.get('url') | |
38 |
|
38 | |||
39 | list_url = url + 'api/sync/list/' |
|
39 | list_url = url + 'api/sync/list/' | |
40 | get_url = url + 'api/sync/get/' |
|
40 | get_url = url + 'api/sync/get/' | |
41 | file_url = url[:-1] |
|
41 | file_url = url[:-1] | |
42 |
|
42 | |||
43 | global_id_str = options.get('global_id') |
|
43 | global_id_str = options.get('global_id') | |
44 | if global_id_str: |
|
44 | if global_id_str: | |
45 | match = REGEX_GLOBAL_ID.match(global_id_str) |
|
45 | match = REGEX_GLOBAL_ID.match(global_id_str) | |
46 | if match: |
|
46 | if match: | |
47 | key_type = match.group(1) |
|
47 | key_type = match.group(1) | |
48 | key = match.group(2) |
|
48 | key = match.group(2) | |
49 | local_id = match.group(3) |
|
49 | local_id = match.group(3) | |
50 |
|
50 | |||
51 | global_id = GlobalId(key_type=key_type, key=key, |
|
51 | global_id = GlobalId(key_type=key_type, key=key, | |
52 | local_id=local_id) |
|
52 | local_id=local_id) | |
53 |
|
53 | |||
54 | xml = SyncManager.generate_request_get([global_id]) |
|
54 | xml = SyncManager.generate_request_get([global_id]) | |
55 | h = httplib2.Http() |
|
55 | h = httplib2.Http() | |
56 | response, content = h.request(get_url, method="POST", body=xml) |
|
56 | response, content = h.request(get_url, method="POST", body=xml) | |
57 |
|
57 | |||
58 | SyncManager.parse_response_get(content, file_url) |
|
58 | SyncManager.parse_response_get(content, file_url) | |
59 | else: |
|
59 | else: | |
60 | raise Exception('Invalid global ID') |
|
60 | raise Exception('Invalid global ID') | |
61 | else: |
|
61 | else: | |
62 | logger.info('Running LIST request...') |
|
62 | logger.info('Running LIST request...') | |
63 | h = httplib2.Http() |
|
63 | h = httplib2.Http() | |
64 |
|
64 | |||
65 | tags = [] |
|
65 | tags = [] | |
66 | tags_str = options.get('tags') |
|
66 | tags_str = options.get('tags') | |
67 | if tags_str: |
|
67 | if tags_str: | |
68 | tags = tags_str.split(',') |
|
68 | tags = tags_str.split(',') | |
69 |
|
69 | |||
70 | xml = SyncManager.generate_request_list( |
|
70 | xml = SyncManager.generate_request_list( | |
71 | opening_post=options.get('thread'), tags=tags, |
|
71 | opening_post=options.get('thread'), tags=tags, | |
72 | timestamp_from=options.get('time_from')).encode() |
|
72 | timestamp_from=options.get('time_from')).encode() | |
73 | response, content = h.request(list_url, method="POST", body=xml) |
|
73 | response, content = h.request(list_url, method="POST", body=xml) | |
74 | if response.status != 200: |
|
74 | if response.status != 200: | |
75 | raise Exception('Server returned error {}'.format(response.status)) |
|
75 | raise Exception('Server returned error {}'.format(response.status)) | |
76 |
|
76 | |||
77 | logger.info('Processing response...') |
|
77 | logger.info('Processing response...') | |
78 |
|
78 | |||
79 | root = ET.fromstring(content) |
|
79 | root = ET.fromstring(content) | |
80 | status = root.findall('status')[0].text |
|
80 | status = root.findall('status')[0].text | |
81 | if status == 'success': |
|
81 | if status == 'success': | |
82 | ids_to_sync = list() |
|
82 | ids_to_sync = list() | |
83 |
|
83 | |||
84 | models = root.findall('models')[0] |
|
84 | models = root.findall('models')[0] | |
85 | for model in models: |
|
85 | for model in models: | |
86 | tag_id = model.find(TAG_ID) |
|
86 | tag_id = model.find(TAG_ID) | |
87 | global_id, exists = GlobalId.from_xml_element(tag_id) |
|
87 | global_id, exists = GlobalId.from_xml_element(tag_id) | |
88 | tag_version = model.find(TAG_VERSION) |
|
88 | tag_version = model.find(TAG_VERSION) | |
89 | if tag_version is not None: |
|
89 | if tag_version is not None: | |
90 | version = int(tag_version.text) or 1 |
|
90 | version = int(tag_version.text) or 1 | |
91 | else: |
|
91 | else: | |
92 | version = 1 |
|
92 | version = 1 | |
93 | if not exists or global_id.post.version < version: |
|
93 | if not exists or global_id.post.version < version: | |
94 | logger.debug('Processed (+) post {}'.format(global_id)) |
|
94 | logger.debug('Processed (+) post {}'.format(global_id)) | |
95 | ids_to_sync.append(global_id) |
|
95 | ids_to_sync.append(global_id) | |
96 | else: |
|
96 | else: | |
97 | logger.debug('* Processed (-) post {}'.format(global_id)) |
|
97 | logger.debug('* Processed (-) post {}'.format(global_id)) | |
98 | logger.info('Starting sync...') |
|
98 | logger.info('Starting sync...') | |
99 |
|
99 | |||
100 | if len(ids_to_sync) > 0: |
|
100 | if len(ids_to_sync) > 0: | |
101 | limit = options.get('split_query', len(ids_to_sync)) |
|
101 | limit = options.get('split_query', len(ids_to_sync)) | |
102 | for offset in range(0, len(ids_to_sync), limit): |
|
102 | for offset in range(0, len(ids_to_sync), limit): | |
103 | xml = SyncManager.generate_request_get(ids_to_sync[offset:offset + limit]) |
|
103 | xml = SyncManager.generate_request_get(ids_to_sync[offset:offset + limit]) | |
104 | h = httplib2.Http() |
|
104 | h = httplib2.Http() | |
105 | logger.info('Running GET request...') |
|
105 | logger.info('Running GET request...') | |
106 | response, content = h.request(get_url, method="POST", body=xml) |
|
106 | response, content = h.request(get_url, method="POST", body=xml) | |
107 | logger.info('Processing response...') |
|
107 | logger.info('Processing response...') | |
108 |
|
108 | |||
109 | SyncManager.parse_response_get(content, file_url) |
|
109 | SyncManager.parse_response_get(content, file_url) | |
|
110 | ||||
|
111 | logger.info('Sync completed successfully') | |||
110 | else: |
|
112 | else: | |
111 | logger.info('Nothing to get, everything synced') |
|
113 | logger.info('Nothing to get, everything synced') | |
112 | else: |
|
114 | else: | |
113 | raise Exception('Invalid response status') |
|
115 | raise Exception('Invalid response status') |
General Comments 0
You need to be logged in to leave comments.
Login now