40 lines
532 B
Plaintext
40 lines
532 B
Plaintext
# Allow all crawlers to access the entire site.
|
|
User-agent: *
|
|
Allow: /
|
|
|
|
# Generative AI and LLM crawlers
|
|
User-agent: GPTBot
|
|
Allow: /
|
|
|
|
User-agent: ChatGPT-User
|
|
Allow: /
|
|
|
|
User-agent: ClaudeBot
|
|
Allow: /
|
|
|
|
User-agent: Claude-Web
|
|
Allow: /
|
|
|
|
User-agent: PerplexityBot
|
|
Allow: /
|
|
|
|
User-agent: FacebookBot
|
|
Allow: /
|
|
|
|
User-agent: Applebot
|
|
Allow: /
|
|
|
|
User-agent: PerplexityBot
|
|
Allow: /
|
|
|
|
User-agent: Google-Extended
|
|
Allow: /
|
|
|
|
User-agent: GoogleOther
|
|
Allow: /
|
|
|
|
User-agent: DuckAssistBot
|
|
Allow: /
|
|
|
|
Sitemap: https://a2a-protocol.org/latest/sitemap.xml
|