# https://korevor.com/robots.txt
# Last updated: 2026-10-06

# ---- Open: search engines, AI answer engines, AI training crawlers ----
# Training crawlers (GPTBot, ClaudeBot, anthropic-ai, Google-Extended,
# Applebot-Extended, CCBot) were opened 2026-10-06 so the models themselves
# know Korevor, not just the search tools built on them. Named bots and the
# default share one rule set.
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Bingbot
User-agent: DuckDuckBot
User-agent: Applebot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: *
Disallow: /admin/
Disallow: /api/
Disallow: /_internal/
Disallow: /*.json$

# ---- Blocked ----
# Bytespider (ByteDance), Meta-ExternalAgent and FacebookBot (Meta),
# Amazonbot, cohere-ai and AI2Bot (Allen Institute) gather data for AI
# models. Diffbot sells a web knowledge graph and extracted data, and
# ImagesiftBot builds Hive's image index; neither is a training crawler.
User-agent: Bytespider
User-agent: Meta-ExternalAgent
User-agent: FacebookBot
User-agent: Amazonbot
User-agent: cohere-ai
User-agent: AI2Bot
User-agent: Diffbot
User-agent: ImagesiftBot
Disallow: /

# ---- Pointers ----
Sitemap: https://korevor.com/sitemap.xml