# Crawl budget (2026-10-07). Every /api/* response and /search-index.json
# carries X-Robots-Tag: noindex, so none of them become search results.
# /api/data/download is disallowed outright: it serves whole CSV series, and
# nothing on this site is free in bulk to a crawler. /api/x402 stays fetchable
# on purpose: agents and x402 catalogs discover the paid API by crawling it
# (with /openapi.json), and a 402 there is the intended answer.
User-agent: *
Allow: /
Disallow: /go/
Disallow: /api/pageview
Disallow: /api/data-export/
Disallow: /api/data/download

# Explicit Googlebot allow. Redundant with the wildcard above but the
# explicit signal helps with crawler-config dashboards that look for a
# named-agent block before falling back to the wildcard.
User-agent: Googlebot
Allow: /
Disallow: /go/
Disallow: /api/pageview
Disallow: /api/data-export/
Disallow: /api/data/download

# Bingbot explicit allow. Bing powers both its own index and (via the
# same crawl) Copilot answers; named here because Bing Webmaster is a
# primary channel for this site.
User-agent: Bingbot
Allow: /
Disallow: /go/
Disallow: /api/pageview
Disallow: /api/data-export/
Disallow: /api/data/download

# ---------------------------------------------------------------------------
# AI crawlers — GEO: we WANT LLMs and AI search to crawl and cite
# thevaultreport.com when users ask about precious-metals prices, COMEX/LME
# inventory, oil/SPR stocks, or the macro context. Two kinds matter, and both
# are allowed:
#
#   1. Training / index crawlers (build the model's/engine's knowledge).
#   2. Real-time "answer" fetchers — the bot that pulls a page AT ANSWER TIME
#      to quote and link it in a live response. These are what actually drive
#      citations in ChatGPT Search, Perplexity, Copilot, Gemini, etc., so they
#      are the highest-value agents to admit.
# ---------------------------------------------------------------------------

# OpenAI — GPTBot (training), OAI-SearchBot (ChatGPT Search index),
# ChatGPT-User (live fetch when a user's ChatGPT browses/cites a page).
User-agent: GPTBot
Allow: /
Disallow: /go/
Disallow: /api/pageview
Disallow: /api/data-export/
Disallow: /api/data/download

User-agent: OAI-SearchBot
Allow: /
Disallow: /go/
Disallow: /api/pageview
Disallow: /api/data-export/
Disallow: /api/data/download

User-agent: ChatGPT-User
Allow: /
Disallow: /go/
Disallow: /api/pageview
Disallow: /api/data-export/
Disallow: /api/data/download

# Anthropic — ClaudeBot (training/index), anthropic-ai (legacy), Claude-Web
# (legacy user-facing fetch), Claude-SearchBot (Claude search index).
User-agent: ClaudeBot
Allow: /
Disallow: /go/
Disallow: /api/pageview
Disallow: /api/data-export/
Disallow: /api/data/download

User-agent: anthropic-ai
Allow: /
Disallow: /go/
Disallow: /api/pageview
Disallow: /api/data-export/
Disallow: /api/data/download

User-agent: Claude-Web
Allow: /
Disallow: /go/
Disallow: /api/pageview
Disallow: /api/data-export/
Disallow: /api/data/download

User-agent: Claude-SearchBot
Allow: /
Disallow: /go/
Disallow: /api/pageview
Disallow: /api/data-export/
Disallow: /api/data/download

# Perplexity — PerplexityBot (index) + Perplexity-User (live answer fetch).
User-agent: PerplexityBot
Allow: /
Disallow: /go/
Disallow: /api/pageview
Disallow: /api/data-export/
Disallow: /api/data/download

User-agent: Perplexity-User
Allow: /
Disallow: /go/
Disallow: /api/pageview
Disallow: /api/data-export/
Disallow: /api/data/download

# Google AI — Google-Extended (Gemini/Vertex training gate) +
# Google-CloudVertexBot (Vertex grounding fetch on behalf of customers).
User-agent: Google-Extended
Allow: /
Disallow: /go/
Disallow: /api/pageview
Disallow: /api/data-export/
Disallow: /api/data/download

User-agent: Google-CloudVertexBot
Allow: /
Disallow: /go/
Disallow: /api/pageview
Disallow: /api/data-export/
Disallow: /api/data/download

# Apple Intelligence / Siri knowledge.
User-agent: Applebot
Allow: /
Disallow: /go/
Disallow: /api/pageview
Disallow: /api/data-export/
Disallow: /api/data/download

User-agent: Applebot-Extended
Allow: /
Disallow: /go/
Disallow: /api/pageview
Disallow: /api/data-export/
Disallow: /api/data/download

# Common Crawl — feeds a long tail of downstream LLM training sets.
User-agent: CCBot
Allow: /
Disallow: /go/
Disallow: /api/pageview
Disallow: /api/data-export/
Disallow: /api/data/download

# Meta AI.
User-agent: meta-externalagent
Allow: /
Disallow: /go/
Disallow: /api/pageview
Disallow: /api/data-export/
Disallow: /api/data/download

# DuckDuckGo AI (DuckAssist) + Amazon (Alexa/Rufus).
User-agent: DuckAssistBot
Allow: /
Disallow: /go/
Disallow: /api/pageview
Disallow: /api/data-export/
Disallow: /api/data/download

User-agent: Amazonbot
Allow: /
Disallow: /go/
Disallow: /api/pageview
Disallow: /api/data-export/
Disallow: /api/data/download

# Primary sitemap
Sitemap: https://thevaultreport.com/sitemap.xml