# Robots.txt for Archivo Salvador Allende
# https://salvador-allende.cl
# Optimized for AI crawlers and search engines - 2026

# Sitemap location
Sitemap: https://salvador-allende.cl/sitemap_index.xml

# =====================================================
# AI/LLM CRAWLERS - FULL ACCESS
# =====================================================

# OpenAI - ChatGPT, GPT-4, GPT-5
User-agent: GPTBot
Allow: /
Crawl-delay: 0

User-agent: ChatGPT-User
Allow: /

# Google AI - Gemini, Bard, AI Overviews
User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

# Anthropic - Claude
User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: ClaudeBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Microsoft - Bing Copilot, Sydney
User-agent: Bingbot
Allow: /
Crawl-delay: 0

User-agent: bingbot
Allow: /

# Common Crawl (training data for many LLMs)
User-agent: CCBot
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /

# Meta AI - Llama
User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# AI21 Labs
User-agent: AI2Bot
Allow: /

# Hugging Face
User-agent: HuggingFaceBot
Allow: /

# You.com AI
User-agent: YouBot
Allow: /

# Brave Search AI
User-agent: BraveBot
Allow: /

# Neeva AI
User-agent: NeevaBot
Allow: /

# Phind AI (code/developer AI)
User-agent: PhindBot
Allow: /

# Kagi AI Search
User-agent: KagiBot
Allow: /

# Exa AI
User-agent: ExaBot
Allow: /

# Apple Intelligence / Siri
User-agent: Applebot
Allow: /
Crawl-delay: 0

User-agent: Applebot-Extended
Allow: /

# Amazon Alexa
User-agent: Amazonbot
Allow: /

# ByteDance / TikTok AI
User-agent: Bytespider
Allow: /

# Baidu AI
User-agent: Baiduspider
Allow: /

# Yandex AI
User-agent: YandexBot
Allow: /
Crawl-delay: 1

# =====================================================
# SEARCH ENGINE CRAWLERS
# =====================================================

# Google (fastest crawl)
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Storebot-Google
Allow: /

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# Mojeek
User-agent: MojeekBot
Allow: /

# Qwant
User-agent: Qwantify
Allow: /

# Ecosia
User-agent: Ecosia
Allow: /

# =====================================================
# SOCIAL MEDIA CRAWLERS
# =====================================================

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Pinterest
Allow: /

User-agent: Slackbot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: Discordbot
Allow: /

User-agent: Redditbot
Allow: /

# =====================================================
# ARCHIVE & RESEARCH CRAWLERS
# =====================================================

User-agent: ia_archiver
Allow: /

User-agent: archive.org_bot
Allow: /

# =====================================================
# GENERAL RULES
# =====================================================

# Allow all other crawlers
User-agent: *
Allow: /
Crawl-delay: 1

# Disallow system paths only
Disallow: /api/
Disallow: /_next/
Disallow: /admin/
Disallow: /.well-known/acme-challenge/

# Host directive
Host: https://salvador-allende.cl

# LLMs.txt for AI-specific instructions
# See: https://salvador-allende.cl/llms.txt