##// END OF EJS Templates
Added completion log message in the sync
neko259 -
r1847:dcbd9cc7 default
parent child Browse files
Show More
@@ -1,113 +1,115 b''
1 import re
1 import re
2 import logging
2 import logging
3 import xml.etree.ElementTree as ET
3 import xml.etree.ElementTree as ET
4
4
5 import httplib2
5 import httplib2
6 from django.core.management import BaseCommand
6 from django.core.management import BaseCommand
7
7
8 from boards.models import GlobalId
8 from boards.models import GlobalId
9 from boards.models.post.sync import SyncManager, TAG_ID, TAG_VERSION
9 from boards.models.post.sync import SyncManager, TAG_ID, TAG_VERSION
10
10
11 __author__ = 'neko259'
11 __author__ = 'neko259'
12
12
13
13
14 REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)')
14 REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)')
15
15
16
16
17 class Command(BaseCommand):
17 class Command(BaseCommand):
18 help = 'Send a sync or get request to the server.'
18 help = 'Send a sync or get request to the server.'
19
19
20 def add_arguments(self, parser):
20 def add_arguments(self, parser):
21 parser.add_argument('url', type=str, help='Server root url')
21 parser.add_argument('url', type=str, help='Server root url')
22 parser.add_argument('--global-id', type=str, default='',
22 parser.add_argument('--global-id', type=str, default='',
23 help='Post global ID')
23 help='Post global ID')
24 parser.add_argument('--split-query', type=int, default=1,
24 parser.add_argument('--split-query', type=int, default=1,
25 help='Split GET query into separate by the given'
25 help='Split GET query into separate by the given'
26 ' number of posts in one')
26 ' number of posts in one')
27 parser.add_argument('--thread', type=int,
27 parser.add_argument('--thread', type=int,
28 help='Get posts of one specific thread')
28 help='Get posts of one specific thread')
29 parser.add_argument('--tags', type=str,
29 parser.add_argument('--tags', type=str,
30 help='Get posts of the tags, comma-separated')
30 help='Get posts of the tags, comma-separated')
31 parser.add_argument('--time-from', type=str,
31 parser.add_argument('--time-from', type=str,
32 help='Get posts from the given timestamp')
32 help='Get posts from the given timestamp')
33
33
34 def handle(self, *args, **options):
34 def handle(self, *args, **options):
35 logger = logging.getLogger('boards.sync')
35 logger = logging.getLogger('boards.sync')
36
36
37 url = options.get('url')
37 url = options.get('url')
38
38
39 list_url = url + 'api/sync/list/'
39 list_url = url + 'api/sync/list/'
40 get_url = url + 'api/sync/get/'
40 get_url = url + 'api/sync/get/'
41 file_url = url[:-1]
41 file_url = url[:-1]
42
42
43 global_id_str = options.get('global_id')
43 global_id_str = options.get('global_id')
44 if global_id_str:
44 if global_id_str:
45 match = REGEX_GLOBAL_ID.match(global_id_str)
45 match = REGEX_GLOBAL_ID.match(global_id_str)
46 if match:
46 if match:
47 key_type = match.group(1)
47 key_type = match.group(1)
48 key = match.group(2)
48 key = match.group(2)
49 local_id = match.group(3)
49 local_id = match.group(3)
50
50
51 global_id = GlobalId(key_type=key_type, key=key,
51 global_id = GlobalId(key_type=key_type, key=key,
52 local_id=local_id)
52 local_id=local_id)
53
53
54 xml = SyncManager.generate_request_get([global_id])
54 xml = SyncManager.generate_request_get([global_id])
55 h = httplib2.Http()
55 h = httplib2.Http()
56 response, content = h.request(get_url, method="POST", body=xml)
56 response, content = h.request(get_url, method="POST", body=xml)
57
57
58 SyncManager.parse_response_get(content, file_url)
58 SyncManager.parse_response_get(content, file_url)
59 else:
59 else:
60 raise Exception('Invalid global ID')
60 raise Exception('Invalid global ID')
61 else:
61 else:
62 logger.info('Running LIST request...')
62 logger.info('Running LIST request...')
63 h = httplib2.Http()
63 h = httplib2.Http()
64
64
65 tags = []
65 tags = []
66 tags_str = options.get('tags')
66 tags_str = options.get('tags')
67 if tags_str:
67 if tags_str:
68 tags = tags_str.split(',')
68 tags = tags_str.split(',')
69
69
70 xml = SyncManager.generate_request_list(
70 xml = SyncManager.generate_request_list(
71 opening_post=options.get('thread'), tags=tags,
71 opening_post=options.get('thread'), tags=tags,
72 timestamp_from=options.get('time_from')).encode()
72 timestamp_from=options.get('time_from')).encode()
73 response, content = h.request(list_url, method="POST", body=xml)
73 response, content = h.request(list_url, method="POST", body=xml)
74 if response.status != 200:
74 if response.status != 200:
75 raise Exception('Server returned error {}'.format(response.status))
75 raise Exception('Server returned error {}'.format(response.status))
76
76
77 logger.info('Processing response...')
77 logger.info('Processing response...')
78
78
79 root = ET.fromstring(content)
79 root = ET.fromstring(content)
80 status = root.findall('status')[0].text
80 status = root.findall('status')[0].text
81 if status == 'success':
81 if status == 'success':
82 ids_to_sync = list()
82 ids_to_sync = list()
83
83
84 models = root.findall('models')[0]
84 models = root.findall('models')[0]
85 for model in models:
85 for model in models:
86 tag_id = model.find(TAG_ID)
86 tag_id = model.find(TAG_ID)
87 global_id, exists = GlobalId.from_xml_element(tag_id)
87 global_id, exists = GlobalId.from_xml_element(tag_id)
88 tag_version = model.find(TAG_VERSION)
88 tag_version = model.find(TAG_VERSION)
89 if tag_version is not None:
89 if tag_version is not None:
90 version = int(tag_version.text) or 1
90 version = int(tag_version.text) or 1
91 else:
91 else:
92 version = 1
92 version = 1
93 if not exists or global_id.post.version < version:
93 if not exists or global_id.post.version < version:
94 logger.debug('Processed (+) post {}'.format(global_id))
94 logger.debug('Processed (+) post {}'.format(global_id))
95 ids_to_sync.append(global_id)
95 ids_to_sync.append(global_id)
96 else:
96 else:
97 logger.debug('* Processed (-) post {}'.format(global_id))
97 logger.debug('* Processed (-) post {}'.format(global_id))
98 logger.info('Starting sync...')
98 logger.info('Starting sync...')
99
99
100 if len(ids_to_sync) > 0:
100 if len(ids_to_sync) > 0:
101 limit = options.get('split_query', len(ids_to_sync))
101 limit = options.get('split_query', len(ids_to_sync))
102 for offset in range(0, len(ids_to_sync), limit):
102 for offset in range(0, len(ids_to_sync), limit):
103 xml = SyncManager.generate_request_get(ids_to_sync[offset:offset + limit])
103 xml = SyncManager.generate_request_get(ids_to_sync[offset:offset + limit])
104 h = httplib2.Http()
104 h = httplib2.Http()
105 logger.info('Running GET request...')
105 logger.info('Running GET request...')
106 response, content = h.request(get_url, method="POST", body=xml)
106 response, content = h.request(get_url, method="POST", body=xml)
107 logger.info('Processing response...')
107 logger.info('Processing response...')
108
108
109 SyncManager.parse_response_get(content, file_url)
109 SyncManager.parse_response_get(content, file_url)
110
111 logger.info('Sync completed successfully')
110 else:
112 else:
111 logger.info('Nothing to get, everything synced')
113 logger.info('Nothing to get, everything synced')
112 else:
114 else:
113 raise Exception('Invalid response status')
115 raise Exception('Invalid response status')
General Comments 0
You need to be logged in to leave comments. Login now