chore: add ai2bot

ai-robots-txt · Aug 29, 2024 · 0f87235 · 0f87235
1 parent 6dc900b
commit 0f87235
Showing 1 changed file with 14 additions and 0 deletions.
diff --git a/robots.json b/robots.json
@@ -1,4 +1,18 @@
 {
+    "AI2Bot": {
+        "description": "Explores 'certain domains' to find web content.",
+        "frequency": "No information. provided.",
+        "function": "Content is used to train open language models.",
+        "operator": "[Ai2](https://allenai.org/crawler)",
+        "respect": "Yes"
+    },
+    "Ai2Bot-Dolma": {
+        "description": "Explores 'certain domains' to find web content.",
+        "frequency": "No information. provided.",
+        "function": "Content is used to train open language models.",
+        "operator": "[Ai2](https://allenai.org/crawler)",
+        "respect": "Yes"
+    },
     "Amazonbot": {
         "description": "Includes references to crawled website when surfacing answers via Alexa; does not clearly outline other uses.",
         "frequency": "No information. provided.",