# TerraFunded — robots.txt # Last updated: 2026-11-08 # Note: robots.txt rules match by prefix, so "Disallow: /auth" would also # block /authors/. The "Allow: /authors/" lines (longer match wins) keep the # founder profile crawlable. # Spec: https://www.robotstxt.org/ # Content-Signal (https://contentsignals.org/): TerraFunded allows search, # AI answers (ai-input) and AI training (ai-train) on its public content. # ────────────────────────────────────────────── # Default — allow all well-behaved crawlers # ────────────────────────────────────────────── User-agent: * Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / Disallow: /admin/ Disallow: /auth Allow: /authors/ Disallow: /reservation-success # ────────────────────────────────────────────── # AI / LLM crawlers — explicit allowlist # We want to be cited by these systems. # ────────────────────────────────────────────── # OpenAI — ChatGPT training, retrieval, and search User-agent: GPTBot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / Disallow: /admin/ Disallow: /auth Allow: /authors/ User-agent: ChatGPT-User Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / Disallow: /admin/ Disallow: /auth Allow: /authors/ User-agent: OAI-SearchBot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / Disallow: /admin/ Disallow: /auth Allow: /authors/ # Anthropic — Claude User-agent: ClaudeBot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / Disallow: /admin/ Disallow: /auth Allow: /authors/ User-agent: anthropic-ai Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / Disallow: /admin/ Disallow: /auth Allow: /authors/ User-agent: Claude-Web Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / Disallow: /admin/ Disallow: /auth Allow: /authors/ # Perplexity User-agent: PerplexityBot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / Disallow: /admin/ Disallow: /auth Allow: /authors/ User-agent: Perplexity-User Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / Disallow: /admin/ Disallow: /auth Allow: /authors/ # Google — Search and AI Overviews User-agent: Googlebot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / Disallow: /admin/ Disallow: /auth Allow: /authors/ User-agent: Google-Extended Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / Disallow: /admin/ Disallow: /auth Allow: /authors/ User-agent: GoogleOther Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / Disallow: /admin/ Disallow: /auth Allow: /authors/ # Microsoft — Bing and Copilot User-agent: Bingbot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / Disallow: /admin/ Disallow: /auth Allow: /authors/ User-agent: msnbot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / Disallow: /admin/ Disallow: /auth Allow: /authors/ # Apple Intelligence User-agent: Applebot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / Disallow: /admin/ Disallow: /auth Allow: /authors/ User-agent: Applebot-Extended Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / Disallow: /admin/ Disallow: /auth Allow: /authors/ # Meta AI User-agent: Meta-ExternalAgent Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / Disallow: /admin/ Disallow: /auth Allow: /authors/ # Common Crawl — training corpus for many open LLMs User-agent: CCBot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / Disallow: /admin/ Disallow: /auth Allow: /authors/ # Amazon User-agent: Amazonbot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / Disallow: /admin/ Disallow: /auth Allow: /authors/ # Mistral User-agent: MistralAI-User Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / Disallow: /admin/ Disallow: /auth Allow: /authors/ # DuckDuckGo User-agent: DuckAssistBot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / Disallow: /admin/ Disallow: /auth Allow: /authors/ # ────────────────────────────────────────────── # Sitemap — served via clean URL rewrite to the Supabase Edge Function # ────────────────────────────────────────────── Sitemap: https://terrafunded.com/sitemap.xml # Spanish llms files # https://terrafunded.com/es/llms.txt # https://terrafunded.com/es/llms-full.txt