# Enrichr wants to be found and quoted by search engines and AI assistants.
# Every agent below is welcome on the public site. A crawler with its own group
# ignores the `*` group entirely, so each group repeats the same Disallows.
#
# Search: build the index an answer engine cites from.
# User:   fetch a page live because a person asked the assistant to.
# Training: collect for model training (allowed: it is how an assistant learns what Enrichr is).
#
# Google-Extended and Applebot-Extended are robots.txt tokens, not user agents:
# they govern whether Gemini / Apple Intelligence may use pages Googlebot and
# Applebot already crawl.

# --- Search ---
User-agent: OAI-SearchBot
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
User-agent: YouBot
Allow: /
Disallow: /captures/
Disallow: /mcp
Disallow: /api/

# --- User-triggered fetches ---
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
User-agent: MistralAI-User
User-agent: DuckAssistBot
User-agent: Claude-Web
Allow: /
Disallow: /captures/
Disallow: /mcp
Disallow: /api/

# --- Training and grounding ---
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: Amazonbot
User-agent: cohere-ai
User-agent: CCBot
Allow: /
Disallow: /captures/
Disallow: /mcp
Disallow: /api/

# --- Everyone else ---
User-agent: *
Allow: /
# login, account and dashboard carry <meta name="robots" content="noindex">;
# they stay crawlable so Google can see that tag.
Disallow: /captures/
# API and MCP endpoints: machine-only, they answer 401 without a key.
Disallow: /mcp
Disallow: /api/

Sitemap: https://enrichr.dev/sitemap.xml