# =============================================================================
# robots.txt for meditailor.app
# MediTailor - AI-Powered Personalized Meditation
# Last updated: 2026-04-05
# =============================================================================

# --- Default: Allow all well-behaved crawlers ---
User-agent: *
Allow: /
Disallow: /api/
Disallow: /_astro/

# --- AI crawlers: citation/search allowed, training scrapers blocked ---
# We allow AI search and citation crawlers for GEO (Generative Engine Optimization).
# We block only AI training scrapers that harvest content without attribution.

# ALLOWED — Google Gemini and AI Overviews (GEO citation)
User-agent: Google-Extended
Allow: /

# ALLOWED — Perplexity AI citation crawler (GEO)
User-agent: PerplexityBot
Allow: /

# ALLOWED — OpenAI search and citation crawlers (ChatGPT real-time browsing)
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# ALLOWED — Anthropic citation crawler (Claude real-time browsing)
User-agent: ClaudeBot
Allow: /

# BLOCKED — Training data scrapers (harvest content without attribution)
User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Cohere-ai
Disallow: /

# --- Sitemap location ---
Sitemap: https://meditailor.app/sitemap-index.xml