From 097879ce7ce3545cdb3eefb7c21b77429cf799cd Mon Sep 17 00:00:00 2001 From: mojojojo20 Date: Mon, 28 Oct 2019 16:02:01 +0530 Subject: [PATCH] robots.txt --- robots.txt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/robots.txt b/robots.txt index 6248bbe..7c36297 100644 --- a/robots.txt +++ b/robots.txt @@ -1,3 +1,10 @@ Sitemap: https://karthikmenon.me/sitemap.xml User-agent: * -Disallow: \ No newline at end of file +Disallow: +User-agent: wget +Disallow: https://www.techkshetra.tech/ +Disallow: https://www.techkshetra.tech/about +Disallow: https://www.techkshetra.tech/contact +Disallow: https://www.techkshetra.tech/departments +Disallow: https://www.techkshetra.tech/images +Disallow: https://www.techkshetra.tech/assets \ No newline at end of file