# =======================================
# Robots.txt — Allow trusted crawlers
# =======================================

# Google (Search + Gemini)
User-agent: Googlebot
Allow: /

User-agent: Google-Extended
Allow: /

# Microsoft (Bing + Copilot + Edge)
User-agent: Bingbot
Allow: /

User-agent: BingPreview
Allow: /

# ChatGPT (OpenAI)
User-agent: GPTBot
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# CCBot (Common Crawl, often used by AI systems)
User-agent: CCBot
Allow: /


# =======================================
# Optional: Sensitive paths (add as needed)
# =======================================
# Disallow: /admin/
# Disallow: /api/
# Disallow: /staging/
# Disallow: /internal/