Parse url attachments by the sync manager. Parse each post in a separate transaction. Default number of posts in one GET request is 1

boards/management/commands/sync_with_server.py

@@ -21,7 +21,7 b' class Command(BaseCommand):'
         parser.add_argument('url', type=str, help='Server root url')
         parser.add_argument('--global-id', type=str, default='',
                             help='Post global ID')
-        parser.add_argument('--split-query', type=int, default=10,
+        parser.add_argument('--split-query', type=int, default=1,
                             help='Split GET query into separate by the given'
                                  ' number of posts in one')
 

boards/models/post/manager.py

@@ -138,7 +138,7 b' class PostManager(models.Manager):'
     @transaction.atomic
     def import_post(self, title: str, text: str, pub_time: str, global_id,
                     opening_post=None, tags=list(), files=list(),
-                    tripcode=None, version=1):
+                    file_urls=list(), tripcode=None, version=1):
         is_opening = opening_post is None
         if is_opening:
             thread = boards.models.thread.Thread.objects.create(
@@ -160,6 +160,8 b' class PostManager(models.Manager):'
 
         for file in files:
             self._add_file_to_post(file, post)
+        for file_url in file_urls:
+            post.attachments.add(Attachment.objects.create_from_url(file_url))
 
         url_to_post = '[post]{}[/post]'.format(str(global_id))
         replies = self.filter(text__contains=url_to_post)
@@ -168,7 +170,7 b' class PostManager(models.Manager):'
 
     @transaction.atomic
     def update_post(self, post, title: str, text: str, pub_time: str,
-                    tags=list(), files=list(), tripcode=None, version=1):
+                    tags=list(), files=list(), file_urls=list(), tripcode=None, version=1):
         post.title = title
         post.text = text
         post.pub_time = pub_time
@@ -181,6 +183,8 b' class PostManager(models.Manager):'
         post.attachments.clear()
         for file in files:
             self._add_file_to_post(file, post)
+        for file_url in file_urls:
+            post.attachments.add(Attachment.objects.create_from_url(file_url))
 
         thread = post.get_thread()
         thread.tags.clear()

boards/models/post/sync.py

@@ -8,7 +8,6 b' from boards.utils import get_file_mimety'
 from django.db import transaction
 
 EXCEPTION_NODE = 'Sync node returned an error: {}.'
-EXCEPTION_OP = 'Load the OP first.'
 EXCEPTION_DOWNLOAD = 'File was not downloaded.'
 EXCEPTION_HASH = 'File hash does not match attachment hash.'
 EXCEPTION_SIGNATURE = 'Invalid model signature for {}.'
@@ -165,96 +164,105 b' class SyncManager:'
         return et.tostring(response, ENCODING_UNICODE)
 
     @staticmethod
-    @transaction.atomic
     def parse_response_get(response_xml, hostname):
         tag_root = et.fromstring(response_xml)
         tag_status = tag_root.find(TAG_STATUS)
         if STATUS_SUCCESS == tag_status.text:
             tag_models = tag_root.find(TAG_MODELS)
             for tag_model in tag_models:
-                tag_content = tag_model.find(TAG_CONTENT)
-
-                content_str = et.tostring(tag_content, ENCODING_UNICODE)
-
-                tag_id = tag_content.find(TAG_ID)
-                global_id, exists = GlobalId.from_xml_element(tag_id)
-                signatures = SyncManager._verify_model(global_id, content_str, tag_model)
-
-                version = int(tag_content.find(TAG_VERSION).text)
-                is_old = exists and global_id.post.version < version
-                if exists and not is_old:
-                    print('Post with same ID exists and is up to date.')
-                else:
-                    global_id.content = content_str
-                    global_id.save()
-                    for signature in signatures:
-                        signature.global_id = global_id
-                        signature.save()
-
-                    title = tag_content.find(TAG_TITLE).text or ''
-                    text = tag_content.find(TAG_TEXT).text or ''
-                    pub_time = tag_content.find(TAG_PUB_TIME).text
-                    tripcode_tag = tag_content.find(TAG_TRIPCODE)
-                    if tripcode_tag is not None:
-                        tripcode = tripcode_tag.text or ''
-                    else:
-                        tripcode = ''
-
-                    thread = tag_content.find(TAG_THREAD)
-                    tags = []
-                    if thread:
-                        thread_id = thread.find(TAG_ID)
-                        op_global_id, exists = GlobalId.from_xml_element(thread_id)
-                        if exists:
-                            opening_post = Post.objects.get(global_id=op_global_id)
-                        else:
-                            raise SyncException(EXCEPTION_OP)
-                    else:
-                        opening_post = None
-                        tag_tags = tag_content.find(TAG_TAGS)
-                        for tag_tag in tag_tags:
-                            tag, created = Tag.objects.get_or_create(
-                                name=tag_tag.text)
-                            tags.append(tag)
-
-                    # TODO Check that the replied posts are already present
-                    # before adding new ones
-
-                    files = []
-                    tag_attachments = tag_content.find(TAG_ATTACHMENTS) or list()
-                    tag_refs = tag_model.find(TAG_ATTACHMENT_REFS)
-                    for attachment in tag_attachments:
-                        tag_ref = tag_refs.find("{}[@ref='{}']".format(
-                            TAG_ATTACHMENT_REF, attachment.text))
-                        url = tag_ref.get(ATTR_URL)
-                        attached_file = download(hostname + url)
-                        if attached_file is None:
-                            raise SyncException(EXCEPTION_DOWNLOAD)
-
-                        hash = get_file_hash(attached_file)
-                        if hash != attachment.text:
-                            raise SyncException(EXCEPTION_HASH)
-
-                        files.append(attached_file)
-
-                    if is_old:
-                        post = global_id.post
-                        Post.objects.update_post(
-                            post, title=title, text=text, pub_time=pub_time,
-                            tags=tags, files=files, tripcode=tripcode,
-                            version=version)
-                        logger.debug('Parsed updated post {}'.format(global_id))
-                    else:
-                        Post.objects.import_post(
-                            title=title, text=text, pub_time=pub_time,
-                            opening_post=opening_post, tags=tags,
-                            global_id=global_id, files=files, tripcode=tripcode,
-                            version=version)
-                        logger.debug('Parsed new post {}'.format(global_id))
+                SyncManager.parse_post(tag_model, hostname)
         else:
             raise SyncException(EXCEPTION_NODE.format(tag_status.text))
 
     @staticmethod
+    @transaction.atomic
+    def parse_post(tag_model, hostname):
+        tag_content = tag_model.find(TAG_CONTENT)
+
+        content_str = et.tostring(tag_content, ENCODING_UNICODE)
+
+        tag_id = tag_content.find(TAG_ID)
+        global_id, exists = GlobalId.from_xml_element(tag_id)
+        signatures = SyncManager._verify_model(global_id, content_str, tag_model)
+
+        version = int(tag_content.find(TAG_VERSION).text)
+        is_old = exists and global_id.post.version < version
+        if exists and not is_old:
+            print('Post with same ID exists and is up to date.')
+        else:
+            global_id.content = content_str
+            global_id.save()
+            for signature in signatures:
+                signature.global_id = global_id
+                signature.save()
+
+            title = tag_content.find(TAG_TITLE).text or ''
+            text = tag_content.find(TAG_TEXT).text or ''
+            pub_time = tag_content.find(TAG_PUB_TIME).text
+            tripcode_tag = tag_content.find(TAG_TRIPCODE)
+            if tripcode_tag is not None:
+                tripcode = tripcode_tag.text or ''
+            else:
+                tripcode = ''
+
+            thread = tag_content.find(TAG_THREAD)
+            tags = []
+            if thread:
+                thread_id = thread.find(TAG_ID)
+                op_global_id, exists = GlobalId.from_xml_element(thread_id)
+                if exists:
+                    opening_post = Post.objects.get(global_id=op_global_id)
+                else:
+                    logger.debug('No thread exists for post {}'.format(global_id))
+            else:
+                opening_post = None
+                tag_tags = tag_content.find(TAG_TAGS)
+                for tag_tag in tag_tags:
+                    tag, created = Tag.objects.get_or_create(
+                        name=tag_tag.text)
+                    tags.append(tag)
+
+            # TODO Check that the replied posts are already present
+            # before adding new ones
+
+            files = []
+            urls = []
+            tag_attachments = tag_content.find(TAG_ATTACHMENTS) or list()
+            tag_refs = tag_model.find(TAG_ATTACHMENT_REFS)
+            for attachment in tag_attachments:
+                if attachment.get(ATTR_ID_TYPE) == ID_TYPE_URL:
+                    urls.append(attachment.text)
+                else:
+                    tag_ref = tag_refs.find("{}[@ref='{}']".format(
+                        TAG_ATTACHMENT_REF, attachment.text))
+                    url = tag_ref.get(ATTR_URL)
+                    attached_file = download(hostname + url)
+                    if attached_file is None:
+                        raise SyncException(EXCEPTION_DOWNLOAD)
+
+                    hash = get_file_hash(attached_file)
+                    if hash != attachment.text:
+                        raise SyncException(EXCEPTION_HASH)
+
+                    files.append(attached_file)
+
+            if is_old:
+                post = global_id.post
+                Post.objects.update_post(
+                    post, title=title, text=text, pub_time=pub_time,
+                    tags=tags, files=files, file_urls=urls,
+                    tripcode=tripcode, version=version)
+                logger.debug('Parsed updated post {}'.format(global_id))
+            else:
+                Post.objects.import_post(
+                    title=title, text=text, pub_time=pub_time,
+                    opening_post=opening_post, tags=tags,
+                    global_id=global_id, files=files,
+                    file_urls=urls, tripcode=tripcode,
+                    version=version)
+                logger.debug('Parsed new post {}'.format(global_id))
+
+    @staticmethod
     def generate_response_list():
         response = et.Element(TAG_RESPONSE)
 

docs/dip-1.markdown

@@ -152,6 +152,7 b' Sample response:'
                     -->
                     <attachments>
                         <attachment mimetype="image/png" id-type="md5">TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5I</attachment>
+                        <attachment id-type="url">http://example.com/</attachment>
                     </attachments>
                 </content>
                 <!--