From 291364255c36f254630d8c7b6270a8689c48deb0 Mon Sep 17 00:00:00 2001
From: Navjot Singh <110228+navjotjsingh@users.noreply.github.com>
Date: Sun, 28 Jul 2024 14:45:28 +0530
Subject: [PATCH] Block AI Bots and crawlers

---
 layouts/robots.txt | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/layouts/robots.txt b/layouts/robots.txt
index 5815470c..0176a721 100644
--- a/layouts/robots.txt
+++ b/layouts/robots.txt
@@ -65,4 +65,36 @@ Disallow: /
 User-agent: ZoominfoBot
 Disallow: /
 
+# OpenAI, ChatGPT
+User-agent: GPTBot
+Disallow: /
+
+# Google AI (Bard, etc)
+User-agent: Google-Extended
+Disallow: /
+
+# Block common crawl
+User-agent: CCBot
+Disallow: /
+
+# Facebook
+User-agent: FacebookBot
+Disallow: /
+
+# Cohere.ai
+User-agent: cohere-ai
+Disallow: /
+
+# Perplexity
+User-agent: PerplexityBot
+Disallow: /
+
+# Anthropic
+User-agent: anthropic-ai
+Disallow: /
+
+# Claudebot by Anthropic
+User-agent: ClaudeBot
+Disallow: /
+
 Sitemap: {{ "/sitemap.xml" | absURL }}