A2A/docs/robots.txt

40 lines
532 B
Plaintext

# Allow all crawlers to access the entire site.
User-agent: *
Allow: /
# Generative AI and LLM crawlers
User-agent: GPTBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: ClaudeBot
Allow: /
User-agent: Claude-Web
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: FacebookBot
Allow: /
User-agent: Applebot
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: Google-Extended
Allow: /
User-agent: GoogleOther
Allow: /
User-agent: DuckAssistBot
Allow: /
Sitemap: https://a2a-protocol.org/latest/sitemap.xml