# robots.txt for allecta.ai
#
# Policy: everything public is crawlable by every search engine and every AI
# crawler. Only the authenticated API and private app surfaces are excluded.
#
# Notes for maintainers:
# - A bot uses the MOST SPECIFIC User-agent group that matches it and ignores
#   the others, so every named group must repeat the full policy.
# - Do not add `Disallow: /*.json$`: it blocks /manifest.json, /knowledge.json
#   and /.well-known/*.json, which are exactly the files AI systems read.
# - Crawl-delay is ignored by Google and merely slows Bing. Leave it out.

User-agent: *
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /history
Disallow: /library
Disallow: /projects
Disallow: /share/
Disallow: /reset-password
Disallow: /run-migration.html
Disallow: /stream-diagnostic.js
Allow: /api/allecta-facts.json
Allow: /api/health

# --- Search engines -------------------------------------------------------
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/
Allow: /api/allecta-facts.json

User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/
Allow: /api/allecta-facts.json

User-agent: DuckDuckBot
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/

User-agent: Applebot
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/

User-agent: YandexBot
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/

User-agent: Baiduspider
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/

User-agent: Slurp
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/

# --- AI search & assistant crawlers ---------------------------------------
# OpenAI: OAI-SearchBot indexes for ChatGPT search, ChatGPT-User fetches on a
# user's behalf, GPTBot is training. All allowed.
User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/
Allow: /api/allecta-facts.json

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/
Allow: /api/allecta-facts.json

User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/
Allow: /api/allecta-facts.json

# Anthropic: ClaudeBot is the current crawler; Claude-Web / anthropic-ai are
# legacy names still seen in logs; Claude-User fetches on a user's behalf.
User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/
Allow: /api/allecta-facts.json

User-agent: Claude-User
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/
Allow: /api/allecta-facts.json

User-agent: Claude-SearchBot
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/
Allow: /api/allecta-facts.json

User-agent: Claude-Web
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/
Allow: /api/allecta-facts.json

User-agent: anthropic-ai
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/
Allow: /api/allecta-facts.json

# Google AI (Gemini, AI Overviews / AI Mode grounding)
User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/
Allow: /api/allecta-facts.json

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/
Allow: /api/allecta-facts.json

User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/
Allow: /api/allecta-facts.json

# Microsoft Copilot (uses Bingbot), Apple Intelligence, Amazon Alexa, Meta AI,
# Mistral, You.com, Cohere, DuckDuckGo AI, Common Crawl, ByteDance
User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/

User-agent: Meta-ExternalFetcher
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/

User-agent: MistralAI-User
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/

User-agent: YouBot
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/

User-agent: cohere-ai
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/

User-agent: DuckAssistBot
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/

User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/

User-agent: Bytespider
Allow: /
Disallow: /api/
Disallow: /v1/
Disallow: /admin
Disallow: /account
Disallow: /partner/
Disallow: /share/

# --- Social previews & safety scanners ------------------------------------
User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: facebookexternalhit
Allow: /

# --- Sitemap ---------------------------------------------------------------
# One canonical host only. Listing both www and apex here told Bing the site
# lives at two addresses.
Sitemap: https://allecta.ai/sitemap.xml

# Machine-readable facts for AI systems:
#   https://allecta.ai/llms.txt
#   https://allecta.ai/ai.txt
#   https://allecta.ai/knowledge.json
#   https://allecta.ai/.well-known/ai-plugin.json
#   https://allecta.ai/.well-known/security.txt