block two spiders, that are too eager

Codepoints · Jan 17, 2024 · 089f9d2 · 089f9d2
1 parent 984a14e
commit 089f9d2
Showing 1 changed file with 8 additions and 0 deletions.
diff --git a/codepoints.net/robots.txt b/codepoints.net/robots.txt
@@ -3,3 +3,11 @@ Sitemap: https://codepoints.net/sitemap.xml
 User-agent: *
 Disallow: /search
 Disallow: /random
+
+User-agent: search.marginalia.nu
+Disallow: /
+# sorry, but this bot created too much traffic
+
+User-agent: SemrushBot
+Disallow: /
+# same