##// END OF EJS Templates
Use logging during sync. Log list response processing
neko259 -
r1605:25856b69 default
parent child Browse files
Show More
@@ -1,90 +1,96 b''
1 import re
1 import re
2 import logging
2 import xml.etree.ElementTree as ET
3 import xml.etree.ElementTree as ET
3
4
4 import httplib2
5 import httplib2
5 from django.core.management import BaseCommand
6 from django.core.management import BaseCommand
6
7
7 from boards.models import GlobalId
8 from boards.models import GlobalId
8 from boards.models.post.sync import SyncManager, TAG_ID, TAG_VERSION
9 from boards.models.post.sync import SyncManager, TAG_ID, TAG_VERSION
9
10
10 __author__ = 'neko259'
11 __author__ = 'neko259'
11
12
12
13
13 REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)')
14 REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)')
14
15
15
16
16 class Command(BaseCommand):
17 class Command(BaseCommand):
17 help = 'Send a sync or get request to the server.'
18 help = 'Send a sync or get request to the server.'
18
19
19 def add_arguments(self, parser):
20 def add_arguments(self, parser):
20 parser.add_argument('url', type=str, help='Server root url')
21 parser.add_argument('url', type=str, help='Server root url')
21 parser.add_argument('--global-id', type=str, default='',
22 parser.add_argument('--global-id', type=str, default='',
22 help='Post global ID')
23 help='Post global ID')
23 parser.add_argument('--split-query', type=int,
24 parser.add_argument('--split-query', type=int,
24 help='Split GET query into separate by the given'
25 help='Split GET query into separate by the given'
25 ' number of posts in one')
26 ' number of posts in one')
26
27
27 def handle(self, *args, **options):
28 def handle(self, *args, **options):
29 logger = logging.getLogger('boards.sync')
30
28 url = options.get('url')
31 url = options.get('url')
29
32
30 list_url = url + 'api/sync/list/'
33 list_url = url + 'api/sync/list/'
31 get_url = url + 'api/sync/get/'
34 get_url = url + 'api/sync/get/'
32 file_url = url[:-1]
35 file_url = url[:-1]
33
36
34 global_id_str = options.get('global_id')
37 global_id_str = options.get('global_id')
35 if global_id_str:
38 if global_id_str:
36 match = REGEX_GLOBAL_ID.match(global_id_str)
39 match = REGEX_GLOBAL_ID.match(global_id_str)
37 if match:
40 if match:
38 key_type = match.group(1)
41 key_type = match.group(1)
39 key = match.group(2)
42 key = match.group(2)
40 local_id = match.group(3)
43 local_id = match.group(3)
41
44
42 global_id = GlobalId(key_type=key_type, key=key,
45 global_id = GlobalId(key_type=key_type, key=key,
43 local_id=local_id)
46 local_id=local_id)
44
47
45 xml = GlobalId.objects.generate_request_get([global_id])
48 xml = GlobalId.objects.generate_request_get([global_id])
46 h = httplib2.Http()
49 h = httplib2.Http()
47 response, content = h.request(get_url, method="POST", body=xml)
50 response, content = h.request(get_url, method="POST", body=xml)
48
51
49 SyncManager.parse_response_get(content, file_url)
52 SyncManager.parse_response_get(content, file_url)
50 else:
53 else:
51 raise Exception('Invalid global ID')
54 raise Exception('Invalid global ID')
52 else:
55 else:
53 print('Running LIST request...')
56 logger.info('Running LIST request...')
54 h = httplib2.Http()
57 h = httplib2.Http()
55 xml = GlobalId.objects.generate_request_list()
58 xml = GlobalId.objects.generate_request_list()
56 response, content = h.request(list_url, method="POST", body=xml)
59 response, content = h.request(list_url, method="POST", body=xml)
57 print('Processing response...')
60 logger.info('Processing response...')
58
61
59 root = ET.fromstring(content)
62 root = ET.fromstring(content)
60 status = root.findall('status')[0].text
63 status = root.findall('status')[0].text
61 if status == 'success':
64 if status == 'success':
62 ids_to_sync = list()
65 ids_to_sync = list()
63
66
64 models = root.findall('models')[0]
67 models = root.findall('models')[0]
65 for model in models:
68 for model in models:
66 tag_id = model.find(TAG_ID)
69 tag_id = model.find(TAG_ID)
67 global_id, exists = GlobalId.from_xml_element(tag_id)
70 global_id, exists = GlobalId.from_xml_element(tag_id)
68 tag_version = model.find(TAG_VERSION)
71 tag_version = model.find(TAG_VERSION)
69 if tag_version is not None:
72 if tag_version is not None:
70 version = int(tag_version.text) or 1
73 version = int(tag_version.text) or 1
71 else:
74 else:
72 version = 1
75 version = 1
73 if not exists or global_id.post.version < version:
76 if not exists or global_id.post.version < version:
77 logger.debug('* Processed (+) post {}'.format(global_id))
74 ids_to_sync.append(global_id)
78 ids_to_sync.append(global_id)
75 print('Starting sync...')
79 else:
80 logger.debug('* Processed (-) post {}'.format(global_id))
81 logger.info('Starting sync...')
76
82
77 if len(ids_to_sync) > 0:
83 if len(ids_to_sync) > 0:
78 limit = options.get('split_query', len(ids_to_sync))
84 limit = options.get('split_query', len(ids_to_sync))
79 for offset in range(0, len(ids_to_sync), limit):
85 for offset in range(0, len(ids_to_sync), limit):
80 xml = GlobalId.objects.generate_request_get(ids_to_sync[offset:offset+limit])
86 xml = GlobalId.objects.generate_request_get(ids_to_sync[offset:offset+limit])
81 h = httplib2.Http()
87 h = httplib2.Http()
82 print('Running GET request...')
88 logger.info('Running GET request...')
83 response, content = h.request(get_url, method="POST", body=xml)
89 response, content = h.request(get_url, method="POST", body=xml)
84 print('Processing response...')
90 logger.info('Processing response...')
85
91
86 SyncManager.parse_response_get(content, file_url)
92 SyncManager.parse_response_get(content, file_url)
87 else:
93 else:
88 print('Nothing to get, everything synced')
94 logger.info('Nothing to get, everything synced')
89 else:
95 else:
90 raise Exception('Invalid response status')
96 raise Exception('Invalid response status')
General Comments 0
You need to be logged in to leave comments. Login now