Added robots.txt

boards/templates/boards/robots.txt

@@ -0,0 +1,8 b''
+User-agent: *
+Disallow: /notifications/
+Disallow: /api/
+Disallow: /random/
+Disallow: /search/
+Disallow: /feed/
+Disallow: /settings/
+Disallow: /admin/
  No newline at end of file

boards/urls.py

@@ -1,5 +1,6 b''
 from django.conf.urls import url
 from django.urls import path
+from django.views.generic import TemplateView
 from django.views.i18n import JavaScriptCatalog
 
 from boards import views
@@ -87,5 +88,9 b' urlpatterns = ['
     path('preview/', PostPreviewView.as_view(), name='preview'),
     path('post_xml/<int:post_id>', get_post_sync_data,
          name='post_sync_data'),
+
+    # Robots
+    path('robots.txt', TemplateView.as_view(template_name='boards/robots.txt',
+                                            content_type='text/plain'))
 ]