notes (uk ai)

dbushell · Jan 14, 2025 · 61ca5f8 · 61ca5f8
1 parent bfdb2a1
commit 61ca5f8
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 1 deletion.
diff --git a/public/robots.txt b/public/robots.txt
@@ -39,6 +39,7 @@ User-agent: PanguBot
 User-agent: PerplexityBot
 User-agent: PetalBot
 User-agent: Scrapy
+User-agent: SemrushBot
 User-agent: Sidetrade indexer bot
 User-agent: Timpibot
 User-agent: VelenPublicWebCrawler

diff --git a/src/data b/src/data