# ─── How to read this file ────────────────────────────────────────────────────
# Per RFC 9309 a crawler obeys exactly ONE group: the most specific
# `User-agent` match. A group that only says `Allow: /` therefore CANCELS every
# Disallow below — that bug is why the private paths are listed inside each
# group that needs them, not once at the bottom.
#
# Wanted policy:
#   - Search + AI-search crawlers: crawl public pages, never private paths.
#   - AI training crawlers: nothing.
#   - Everyone else: same as search crawlers.

# ─── Search engines and AI search crawlers (traffic comes back) ───────────────
# Named only for transparency: each group repeats the same rules as `*`.
User-agent: Googlebot
User-agent: Bingbot
User-agent: Twitterbot
User-agent: facebookexternalhit
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: DuckAssistBot
User-agent: YouBot
User-agent: MistralAI-User
User-agent: Gemini-Deep-Research
Allow: /
Disallow: /report/
Disallow: /history
Disallow: /admin/
Disallow: /auth
Disallow: /dashboard
Disallow: /notifications
Disallow: /account
Disallow: /compare
Disallow: /results

# ─── AI training crawlers — blocked ───────────────────────────────────────────
User-agent: GPTBot
User-agent: Google-Extended
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: CCBot
User-agent: anthropic-ai
User-agent: cohere-ai
User-agent: Bytespider
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: Applebot-Extended
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: Omgilibot
User-agent: webzio
User-agent: ICC-Crawler
User-agent: DeepSeekBot
User-agent: GrokBot
Disallow: /

# ─── Default — everyone else ──────────────────────────────────────────────────
User-agent: *
Allow: /
Disallow: /report/
Disallow: /history
Disallow: /admin/
Disallow: /auth
Disallow: /dashboard
Disallow: /notifications
Disallow: /account
Disallow: /compare
Disallow: /results


Sitemap: https://indexdecode.com/sitemap.xml

# RSS Feed
# https://indexdecode.com/rss (proxied via edge function)

# ─── Usage intent (Cloudflare Content Signals + IETF AIPREF) ───────────────
# Access rules above say WHO may fetch. These say HOW the content may be used.
# Advisory declarations, not enforcement.
# Content Signals (Cloudflare proposal): https://blog.cloudflare.com/content-signals-policy/
# AIPREF (IETF Internet-Draft): https://datatracker.ietf.org/wg/aipref/documents/
Content-Signal: search=yes, ai-input=yes, ai-train=no
Content-Usage: search=y, train-ai=n

# ─── Licensing (RSL 1.0 — https://rslstandard.org/rsl) ────────────────────
License: https://indexdecode.com/rsl.xml