add custom robots.txt to block indexing /user and AI bots

ckanext/openafrica/blueprint.py

@@ -1,4 +1,4 @@
-from flask import Blueprint
+from flask import Blueprint, make_response
 from ckan.plugins.toolkit import render
 
 openafrica = Blueprint("openafrica", __name__)
@@ -9,14 +9,21 @@ def render_path():
 
     return render_path
 
+def robots_txt():
+    '''display robots.txt'''
+    resp = make_response(render("robots.txt"))
+    resp.headers["Content-Type"] = "text/plain; charset=utf-8"
+    return resp
+
 rules = [
     ("/about/terms-and-conditions", "toc", static_path("home/about/toc.html")),
     ("/about/accessibility", "accessibility", static_path("home/about/accessibility.html")),
     ("/about/code-of-conduct", "coc", static_path("home/about/coc.html")),
     ("/about/moderation-policy", "moderation", static_path("home/about/moderation.html")),
     ("/about/faq", "faq", static_path("home/about/faq.html")),
     ("/about/privacy", "privacy", static_path("home/about/privacy.html")),
-    ("/about/contact-us", "contact", static_path("home/about/contact.html"))
+    ("/about/contact-us", "contact", static_path("home/about/contact.html")),
+    ("/robots.txt", robots_txt)
 ]
 
 for rule in rules:

ckanext/openafrica/templates/robots.txt

@@ -0,0 +1,78 @@
+# Crawl-delay Specifies the minimum interval (in seconds)
+#for a robot to wait after loading one page, before starting to load another.
+Crawl-delay: 10
+
+# Disallow specifies the paths that are not allowed to be crawled by the robot.
+User-agent: *
+Disallow: /dataset/rate/
+Disallow: /revision/
+Disallow: /dataset/*/history
+Disallow: /api/
+Disallow: /user/
+
+# Amazonbot
+User-agent: Amazonbot
+Disallow: /
+
+# anthropic-ai
+User-agent: anthropic-ai
+Disallow: /
+
+# Applebot
+User-agent: Applebot
+Disallow: /
+
+# Applebot-Extended
+User-agent: Applebot-Extended
+Disallow: /
+
+# Bytespider
+User-agent: Bytespider
+Disallow: /
+
+# CCBot
+User-agent: CCBot
+Disallow: /
+
+# ChatGPT-User
+User-agent: ChatGPT-User
+Disallow: /
+
+# Claude-Web
+User-agent: Claude-Web
+Disallow: /
+
+# ClaudeBot
+User-agent: ClaudeBot
+Disallow: /
+
+# cohere-ai
+User-agent: cohere-ai
+Disallow: /
+
+# Diffbot
+User-agent: Diffbot
+Disallow: /
+
+# FacebookBot
+User-agent: FacebookBot
+Disallow: /
+
+# Google-Extended
+User-agent: Google-Extended
+Disallow: /
+
+# GPTBot
+User-agent: GPTBot
+Disallow: /
+
+# omgili
+User-agent: omgili
+Disallow: /
+
+# PerplexityBot
+User-agent: PerplexityBot
+Disallow: /
+
+
+# Generatedy by RoboShield (https://roboshield.trustlab.africa)