From e72d28ac0a884d61b446ee3f6688fccd77a41e51 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Mon, 6 Jul 2026 17:27:27 +0000
Subject: [PATCH 1/2] Initial plan


From f20630a4b183eac03ce5ab9edd388ad92b1828f6 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Mon, 6 Jul 2026 17:39:23 +0000
Subject: [PATCH 2/2] feat(docs): add missing AI crawler bots to robots.txt

Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>
---
 docs/public/robots.txt        | 27 +++++++++++++++++++++++++++
 docs/tests/robots-txt.spec.ts | 27 +++++++++++++++++++++++++++
 2 files changed, 54 insertions(+)

diff --git a/docs/public/robots.txt b/docs/public/robots.txt
index 2244c31f967..d739cee1208 100644
--- a/docs/public/robots.txt
+++ b/docs/public/robots.txt
@@ -16,6 +16,9 @@ Allow: /
 User-agent: ClaudeBot
 Allow: /
 
+User-agent: Claude-SearchBot
+Allow: /
+
 User-agent: PerplexityBot
 Allow: /
 
@@ -28,4 +31,28 @@ Allow: /
 User-agent: Google-CloudVertexBot
 Allow: /
 
+User-agent: Bingbot
+Allow: /
+
+User-agent: cohere-ai
+Allow: /
+
+User-agent: DuckAssistBot
+Allow: /
+
+User-agent: xAI-Bot
+Allow: /
+
+User-agent: Amazonbot
+Allow: /
+
+User-agent: AI2Bot
+Allow: /
+
+User-agent: YouBot
+Allow: /
+
+User-agent: CCBot
+Allow: /
+
 Sitemap: https://github.github.com/gh-aw/sitemap-index.xml
diff --git a/docs/tests/robots-txt.spec.ts b/docs/tests/robots-txt.spec.ts
index 82f5b5d27ad..421dc9d4784 100644
--- a/docs/tests/robots-txt.spec.ts
+++ b/docs/tests/robots-txt.spec.ts
@@ -19,6 +19,9 @@ const EXPECTED_ROBOTS_TXT = [
   'User-agent: ClaudeBot',
   'Allow: /',
   '',
+  'User-agent: Claude-SearchBot',
+  'Allow: /',
+  '',
   'User-agent: PerplexityBot',
   'Allow: /',
   '',
@@ -31,6 +34,30 @@ const EXPECTED_ROBOTS_TXT = [
   'User-agent: Google-CloudVertexBot',
   'Allow: /',
   '',
+  'User-agent: Bingbot',
+  'Allow: /',
+  '',
+  'User-agent: cohere-ai',
+  'Allow: /',
+  '',
+  'User-agent: DuckAssistBot',
+  'Allow: /',
+  '',
+  'User-agent: xAI-Bot',
+  'Allow: /',
+  '',
+  'User-agent: Amazonbot',
+  'Allow: /',
+  '',
+  'User-agent: AI2Bot',
+  'Allow: /',
+  '',
+  'User-agent: YouBot',
+  'Allow: /',
+  '',
+  'User-agent: CCBot',
+  'Allow: /',
+  '',
   'Sitemap: https://github.github.com/gh-aw/sitemap-index.xml',
   '',
 ].join('\n');