Some progress in the sync

boards/management/commands/sync_with_server.py

@@ -1,46 +1,61 b''
 import re
 import urllib.parse
 import httplib2
+import xml.etree.ElementTree as ET
+
 from django.core.management import BaseCommand
 from boards.models import GlobalId
 
 __author__ = 'neko259'
 
 
 REGEX_GLOBAL_ID = re.compile(r'(\w+)::([\w\+/]+)::(\d+)')
 
 
 class Command(BaseCommand):
     help = 'Send a sync or get request to the server.' + \
            'sync_with_server <server_url> [post_global_id]'
 
     def add_arguments(self, parser):
         parser.add_argument('url', type=str)
-        parser.add_argument('global_id', type=str)
+        #parser.add_argument('global_id', type=str)  # TODO Implement this
 
     def handle(self, *args, **options):
         url = options.get('url')
         global_id_str = options.get('global_id')
         if global_id_str:
             match = REGEX_GLOBAL_ID.match(global_id_str)
             if match:
                 key_type = match.group(1)
                 key = match.group(2)
                 local_id = match.group(3)
 
                 global_id = GlobalId(key_type=key_type, key=key,
                                      local_id=local_id)
 
                 xml = GlobalId.objects.generate_request_get([global_id])
                 data = {'xml': xml}
                 body = urllib.parse.urlencode(data)
                 h = httplib2.Http()
                 response, content = h.request(url, method="POST", body=body)
 
                 # TODO Parse content and get the model list
 
                 print(content)
             else:
                 raise Exception('Invalid global ID')
         else:
-            raise Exception('Full sync is not supported yet.')
+            h = httplib2.Http()
+            response, content = h.request(url, method="POST")
+
+            print(content)
+
+            root = ET.fromstring(content)
+            status = root.findall('status')[0].text
+            if status == 'success':
+                models = root.findall('models')[0]
+                for model in models:
+                    model_content = model[0]
+                    print(model_content.findall('text')[0].text)
+            else:
+                raise Exception('Invalid response status')

boards/urls.py

@@ -1,85 +1,89 b''
 from django.conf.urls import patterns, url
 from django.views.i18n import javascript_catalog
 
 from boards import views
 from boards.rss import AllThreadsFeed, TagThreadsFeed, ThreadPostsFeed
 from boards.views import api, tag_threads, all_threads, \
     settings, all_tags
 from boards.views.authors import AuthorsView
 from boards.views.notifications import NotificationView
 from boards.views.search import BoardSearchView
 from boards.views.static import StaticPageView
 from boards.views.preview import PostPreviewView
 from boards.views.sync import get_post_sync_data
 
 
 js_info_dict = {
     'packages': ('boards',),
 }
 
 urlpatterns = patterns('',
     # /boards/
     url(r'^$', all_threads.AllThreadsView.as_view(), name='index'),
     # /boards/page/
     url(r'^page/(?P<page>\w+)/$', all_threads.AllThreadsView.as_view(),
         name='index'),
 
     # /boards/tag/tag_name/
     url(r'^tag/(?P<tag_name>\w+)/$', tag_threads.TagView.as_view(),
         name='tag'),
     # /boards/tag/tag_id/page/
     url(r'^tag/(?P<tag_name>\w+)/page/(?P<page>\w+)/$',
         tag_threads.TagView.as_view(), name='tag'),
 
     # /boards/thread/
     url(r'^thread/(?P<post_id>\d+)/$', views.thread.normal.NormalThreadView.as_view(),
         name='thread'),
     url(r'^thread/(?P<post_id>\d+)/mode/gallery/$', views.thread.gallery.GalleryThreadView.as_view(),
         name='thread_gallery'),
 
     url(r'^settings/$', settings.SettingsView.as_view(), name='settings'),
     url(r'^tags/(?P<query>\w+)?/?$', all_tags.AllTagsView.as_view(), name='tags'),
     url(r'^authors/$', AuthorsView.as_view(), name='authors'),
 
     url(r'^banned/$', views.banned.BannedView.as_view(), name='banned'),
     url(r'^staticpage/(?P<name>\w+)/$', StaticPageView.as_view(),
         name='staticpage'),
 
     # RSS feeds
     url(r'^rss/$', AllThreadsFeed()),
     url(r'^page/(?P<page>\d+)/rss/$', AllThreadsFeed()),
     url(r'^tag/(?P<tag_name>\w+)/rss/$', TagThreadsFeed()),
     url(r'^tag/(?P<tag_name>\w+)/page/(?P<page>\w+)/rss/$', TagThreadsFeed()),
     url(r'^thread/(?P<post_id>\d+)/rss/$', ThreadPostsFeed()),
 
     # i18n
     url(r'^jsi18n/$', javascript_catalog, js_info_dict,
         name='js_info_dict'),
 
     # API
     url(r'^api/post/(?P<post_id>\d+)/$', api.get_post, name="get_post"),
     url(r'^api/diff_thread$',
         api.api_get_threaddiff, name="get_thread_diff"),
     url(r'^api/threads/(?P<count>\w+)/$', api.api_get_threads,
             name='get_threads'),
     url(r'^api/tags/$', api.api_get_tags, name='get_tags'),
     url(r'^api/thread/(?P<opening_post_id>\w+)/$', api.api_get_thread_posts,
             name='get_thread'),
     url(r'^api/add_post/(?P<opening_post_id>\w+)/$', api.api_add_post,
             name='add_post'),
     url(r'^api/notifications/(?P<username>\w+)/$', api.api_get_notifications,
             name='api_notifications'),
 
+    # Sync protocol API
+    url(r'^api/sync/pull/$', api.sync_pull, name='api_sync_pull'),
+    # TODO 'get' request
+
     # Search
     url(r'^search/$', BoardSearchView.as_view(), name='search'),
 
     # Notifications
     url(r'^notifications/(?P<username>\w+)$', NotificationView.as_view(), name='notifications'),
 
     # Post preview
     url(r'^preview/$', PostPreviewView.as_view(), name='preview'),
 
     url(r'^post_xml/(?P<post_id>\d+)$', get_post_sync_data,
         name='post_sync_data'),
 
 )

boards/views/api.py

@@ -1,231 +1,246 b''
 import json
 import logging
 
 from django.db import transaction
 from django.http import HttpResponse
 from django.shortcuts import get_object_or_404
 from django.core import serializers
 
 from boards.forms import PostForm, PlainErrorList
 from boards.models import Post, Thread, Tag
 from boards.utils import datetime_to_epoch
 from boards.views.thread import ThreadView
 from boards.models.user import Notification
 
 
 __author__ = 'neko259'
 
 PARAMETER_TRUNCATED = 'truncated'
 PARAMETER_TAG = 'tag'
 PARAMETER_OFFSET = 'offset'
 PARAMETER_DIFF_TYPE = 'type'
 PARAMETER_POST = 'post'
 PARAMETER_UPDATED = 'updated'
 PARAMETER_LAST_UPDATE = 'last_update'
 PARAMETER_THREAD = 'thread'
 PARAMETER_UIDS = 'uids'
 
 DIFF_TYPE_HTML = 'html'
 DIFF_TYPE_JSON = 'json'
 
 STATUS_OK = 'ok'
 STATUS_ERROR = 'error'
 
 logger = logging.getLogger(__name__)
 
 
 @transaction.atomic
 def api_get_threaddiff(request):
     """
     Gets posts that were changed or added since time
     """
 
     thread_id = request.GET.get(PARAMETER_THREAD)
     uids_str = request.POST.get(PARAMETER_UIDS).strip()
     uids = uids_str.split(' ')
 
     thread = get_object_or_404(Post, id=thread_id).get_thread()
 
     json_data = {
         PARAMETER_UPDATED: [],
         PARAMETER_LAST_UPDATE: None,  # TODO Maybe this can be removed already?
     }
     posts = Post.objects.filter(threads__in=[thread]).exclude(uid__in=uids)
 
     diff_type = request.GET.get(PARAMETER_DIFF_TYPE, DIFF_TYPE_HTML)
 
     for post in posts:
         json_data[PARAMETER_UPDATED].append(get_post_data(post.id, diff_type, request))
     json_data[PARAMETER_LAST_UPDATE] = str(thread.last_edit_time)
 
     return HttpResponse(content=json.dumps(json_data))
 
 
 def api_add_post(request, opening_post_id):
     """
     Adds a post and return the JSON response for it
     """
 
     opening_post = get_object_or_404(Post, id=opening_post_id)
 
     logger.info('Adding post via api...')
 
     status = STATUS_OK
     errors = []
 
     if request.method == 'POST':
         form = PostForm(request.POST, request.FILES, error_class=PlainErrorList)
         form.session = request.session
 
         if form.need_to_ban:
             # Ban user because he is suspected to be a bot
             # _ban_current_user(request)
             status = STATUS_ERROR
         if form.is_valid():
             post = ThreadView().new_post(request, form, opening_post,
                                          html_response=False)
             if not post:
                 status = STATUS_ERROR
             else:
                 logger.info('Added post #%d via api.' % post.id)
         else:
             status = STATUS_ERROR
             errors = form.as_json_errors()
 
     response = {
         'status': status,
         'errors': errors,
     }
 
     return HttpResponse(content=json.dumps(response))
 
 
 def get_post(request, post_id):
     """
     Gets the html of a post. Used for popups. Post can be truncated if used
     in threads list with 'truncated' get parameter.
     """
 
     post = get_object_or_404(Post, id=post_id)
     truncated = PARAMETER_TRUNCATED in request.GET
 
     return HttpResponse(content=post.get_view(truncated=truncated))
 
 
 def api_get_threads(request, count):
     """
     Gets the JSON thread opening posts list.
     Parameters that can be used for filtering:
     tag, offset (from which thread to get results)
     """
 
     if PARAMETER_TAG in request.GET:
         tag_name = request.GET[PARAMETER_TAG]
         if tag_name is not None:
             tag = get_object_or_404(Tag, name=tag_name)
             threads = tag.get_threads().filter(archived=False)
     else: 
         threads = Thread.objects.filter(archived=False)
 
     if PARAMETER_OFFSET in request.GET:
         offset = request.GET[PARAMETER_OFFSET]
         offset = int(offset) if offset is not None else 0
     else:
         offset = 0
 
     threads = threads.order_by('-bump_time')
     threads = threads[offset:offset + int(count)]
 
     opening_posts = []
     for thread in threads:
         opening_post = thread.get_opening_post()
 
         # TODO Add tags, replies and images count
         post_data = get_post_data(opening_post.id, include_last_update=True)
         post_data['bumpable'] = thread.can_bump()
         post_data['archived'] = thread.archived
 
         opening_posts.append(post_data)
 
     return HttpResponse(content=json.dumps(opening_posts))
 
 
 # TODO Test this
 def api_get_tags(request):
     """
     Gets all tags or user tags.
     """
 
     # TODO Get favorite tags for the given user ID
 
     tags = Tag.objects.get_not_empty_tags()
 
     term = request.GET.get('term')
     if term is not None:
         tags = tags.filter(name__contains=term)
 
     tag_names = [tag.name for tag in tags]
 
     return HttpResponse(content=json.dumps(tag_names))
 
 
 # TODO The result can be cached by the thread last update time
 # TODO Test this
 def api_get_thread_posts(request, opening_post_id):
     """
     Gets the JSON array of thread posts
     """
 
     opening_post = get_object_or_404(Post, id=opening_post_id)
     thread = opening_post.get_thread()
     posts = thread.get_replies()
 
     json_data = {
         'posts': [],
         'last_update': None,
     }
     json_post_list = []
 
     for post in posts:
         json_post_list.append(get_post_data(post.id))
     json_data['last_update'] = datetime_to_epoch(thread.last_edit_time)
     json_data['posts'] = json_post_list
 
     return HttpResponse(content=json.dumps(json_data))
 
 
 def api_get_notifications(request, username):
     last_notification_id_str = request.GET.get('last', None)
     last_id = int(last_notification_id_str) if last_notification_id_str is not None else None
 
     posts = Notification.objects.get_notification_posts(username=username,
                                                         last=last_id)
 
     json_post_list = []
     for post in posts:
         json_post_list.append(get_post_data(post.id))
     return HttpResponse(content=json.dumps(json_post_list))
 
 
 def api_get_post(request, post_id):
     """
     Gets the JSON of a post. This can be
     used as and API for external clients.
     """
 
     post = get_object_or_404(Post, id=post_id)
 
     json = serializers.serialize("json", [post], fields=(
         "pub_time", "_text_rendered", "title", "text", "image",
         "image_width", "image_height", "replies", "tags"
     ))
 
     return HttpResponse(content=json)
 
 
 # TODO Remove this method and use post method directly
 def get_post_data(post_id, format_type=DIFF_TYPE_JSON, request=None,
                   include_last_update=False):
     post = get_object_or_404(Post, id=post_id)
     return post.get_post_data(format_type=format_type, request=request,
                               include_last_update=include_last_update)
+
+
+# TODO Make a separate module for sync API methods
+def sync_pull(request):
+    """
+    Return 'get' request response for all posts.
+    """
+    request_xml = request.get('xml')
+    if request_xml is None:
+        posts = Post.objects.all()
+    else:
+        pass  # TODO Parse the XML and get filters from it
+
+    xml = Post.objects.generate_response_get(posts)
+    return HttpResponse(content=xml)