##// END OF EJS Templates
Log requests during sync. Remove request body from log cause it is too large
neko259 -
r1581:8b72916c default
parent child Browse files
Show More
@@ -1,90 +1,88 b''
1 import re
1 import re
2 import xml.etree.ElementTree as ET
2 import xml.etree.ElementTree as ET
3
3
4 import httplib2
4 import httplib2
5 from django.core.management import BaseCommand
5 from django.core.management import BaseCommand
6
6
7 from boards.models import GlobalId
7 from boards.models import GlobalId
8 from boards.models.post.sync import SyncManager, TAG_ID, TAG_VERSION
8 from boards.models.post.sync import SyncManager, TAG_ID, TAG_VERSION
9
9
10 __author__ = 'neko259'
10 __author__ = 'neko259'
11
11
12
12
13 REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)')
13 REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)')
14
14
15
15
16 class Command(BaseCommand):
16 class Command(BaseCommand):
17 help = 'Send a sync or get request to the server.'
17 help = 'Send a sync or get request to the server.'
18
18
19 def add_arguments(self, parser):
19 def add_arguments(self, parser):
20 parser.add_argument('url', type=str, help='Server root url')
20 parser.add_argument('url', type=str, help='Server root url')
21 parser.add_argument('--global-id', type=str, default='',
21 parser.add_argument('--global-id', type=str, default='',
22 help='Post global ID')
22 help='Post global ID')
23 parser.add_argument('--split-query', type=int,
23 parser.add_argument('--split-query', type=int,
24 help='Split GET query into separate by the given'
24 help='Split GET query into separate by the given'
25 ' number of posts in one')
25 ' number of posts in one')
26
26
27 def handle(self, *args, **options):
27 def handle(self, *args, **options):
28 url = options.get('url')
28 url = options.get('url')
29
29
30 list_url = url + 'api/sync/list/'
30 list_url = url + 'api/sync/list/'
31 get_url = url + 'api/sync/get/'
31 get_url = url + 'api/sync/get/'
32 file_url = url[:-1]
32 file_url = url[:-1]
33
33
34 global_id_str = options.get('global_id')
34 global_id_str = options.get('global_id')
35 if global_id_str:
35 if global_id_str:
36 match = REGEX_GLOBAL_ID.match(global_id_str)
36 match = REGEX_GLOBAL_ID.match(global_id_str)
37 if match:
37 if match:
38 key_type = match.group(1)
38 key_type = match.group(1)
39 key = match.group(2)
39 key = match.group(2)
40 local_id = match.group(3)
40 local_id = match.group(3)
41
41
42 global_id = GlobalId(key_type=key_type, key=key,
42 global_id = GlobalId(key_type=key_type, key=key,
43 local_id=local_id)
43 local_id=local_id)
44
44
45 xml = GlobalId.objects.generate_request_get([global_id])
45 xml = GlobalId.objects.generate_request_get([global_id])
46 # body = urllib.parse.urlencode(data)
47 h = httplib2.Http()
46 h = httplib2.Http()
48 response, content = h.request(get_url, method="POST", body=xml)
47 response, content = h.request(get_url, method="POST", body=xml)
49
48
50 SyncManager.parse_response_get(content, file_url)
49 SyncManager.parse_response_get(content, file_url)
51 else:
50 else:
52 raise Exception('Invalid global ID')
51 raise Exception('Invalid global ID')
53 else:
52 else:
53 print('Running LIST request')
54 h = httplib2.Http()
54 h = httplib2.Http()
55 xml = GlobalId.objects.generate_request_list()
55 xml = GlobalId.objects.generate_request_list()
56 response, content = h.request(list_url, method="POST", body=xml)
56 response, content = h.request(list_url, method="POST", body=xml)
57
57
58 print(content.decode() + '\n')
59
60 root = ET.fromstring(content)
58 root = ET.fromstring(content)
61 status = root.findall('status')[0].text
59 status = root.findall('status')[0].text
62 if status == 'success':
60 if status == 'success':
63 ids_to_sync = list()
61 ids_to_sync = list()
64
62
65 models = root.findall('models')[0]
63 models = root.findall('models')[0]
66 for model in models:
64 for model in models:
67 tag_id = model.find(TAG_ID)
65 tag_id = model.find(TAG_ID)
68 global_id, exists = GlobalId.from_xml_element(tag_id)
66 global_id, exists = GlobalId.from_xml_element(tag_id)
69 tag_version = model.find(TAG_VERSION)
67 tag_version = model.find(TAG_VERSION)
70 if tag_version is not None:
68 if tag_version is not None:
71 version = int(tag_version.text) or 1
69 version = int(tag_version.text) or 1
72 else:
70 else:
73 version = 1
71 version = 1
74 if not exists or global_id.post.version < version:
72 if not exists or global_id.post.version < version:
75 ids_to_sync.append(global_id)
73 ids_to_sync.append(global_id)
76 print('Starting sync...')
74 print('Starting sync...')
77
75
78 if len(ids_to_sync) > 0:
76 if len(ids_to_sync) > 0:
79 limit = options.get('split_query', len(ids_to_sync))
77 limit = options.get('split_query', len(ids_to_sync))
80 for offset in range(0, len(ids_to_sync), limit):
78 for offset in range(0, len(ids_to_sync), limit):
81 xml = GlobalId.objects.generate_request_get(ids_to_sync[offset:offset+limit])
79 xml = GlobalId.objects.generate_request_get(ids_to_sync[offset:offset+limit])
82 # body = urllib.parse.urlencode(data)
83 h = httplib2.Http()
80 h = httplib2.Http()
81 print('Running GET request')
84 response, content = h.request(get_url, method="POST", body=xml)
82 response, content = h.request(get_url, method="POST", body=xml)
85
83
86 SyncManager.parse_response_get(content, file_url)
84 SyncManager.parse_response_get(content, file_url)
87 else:
85 else:
88 print('Nothing to get, everything synced')
86 print('Nothing to get, everything synced')
89 else:
87 else:
90 raise Exception('Invalid response status')
88 raise Exception('Invalid response status')
General Comments 0
You need to be logged in to leave comments. Login now