From a7bdca44c561197f0193cf76c138154ed94119fd Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 24 Aug 2024 00:49:41 +0000 Subject: [PATCH] chore: generate robots.txt --- public/robots.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/public/robots.txt b/public/robots.txt index fa9ac70..4dac345 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -118,6 +118,12 @@ Disallow: / User-agent: Timpibot Disallow: / +# AI Data Scraper +# https://darkvisitors.com/agents/webzio-extended + +User-agent: Webzio-Extended +Disallow: / + # AI Search Crawler # https://darkvisitors.com/agents/youbot