User-agent: *
Allow: /
Allow: /etf/
Allow: /sector/
Allow: /sectors
Allow: /guides/
Allow: /guides
Allow: /blog/
Allow: /blog
Allow: /feed.xml
Disallow: /api/

# Sitemaps
Sitemap: https://etf-radar.fayae.com/sitemap.xml

# Bing respects Crawl-delay, Google does not
User-agent: Bingbot
Crawl-delay: 10

# Block AI training bots
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Bytespider
Disallow: /