# aioproductos.com — robots.txt
#
# Policy: allow all classic search bots AND all AI crawlers. ProductOS
# benefits from being cited in AI answers — opt-in by default.
#
# If we ever want to disallow a specific AI bot (e.g. some bot starts
# misbehaving or there's a strategic reason), add a User-agent: <bot>
# block above the wildcard Disallow.
#
# RFC 9309 §2.2.1 — READ THIS BEFORE ADDING A RULE. A crawler picks the ONE
# most specific matching group and ignores every other group, including
# `User-agent: *`. So a Disallow that only lives in the wildcard group is
# invisible to every bot named below. Any path we actually want excluded must
# be repeated in EVERY named group. That is why `Disallow: /_md/` appears
# ~25 times in this file rather than once — it is not duplication by accident.
# (Until 2026-08-07 it appeared exactly once, in a second `User-agent: *` group
# at the bottom, which meant it bound Googlebot and Bingbot and no AI crawler
# at all — the precise opposite of its stated intent. check-seo.mjs now fails
# the build if any named group is missing it.)

User-agent: *
Allow: /
# /_md/ holds the SOURCE of the Markdown twins. The readable URL is /<path>.md
# (e.g. /pricing.md), which every page advertises via rel="alternate" and which
# llms.txt indexes. Both serve identical content; only the /<path>.md route is
# counted, so pointing crawlers at the internal copy would make reads invisible.
Disallow: /_md/
# NOTE: our AI-usage stance (search=yes, ai-input=yes, ai-train=yes — we WANT to
# be cited and trained on) is expressed by the explicit AI-crawler Allow blocks
# below. The Cloudflare `Content-Signal:` robots directive was removed 2026-07-07:
# it's not part of RFC 9309, so strict validators (Semrush) report it as an
# "invalid robots.txt format" error, and real adoption is near-zero. If we want
# it back, deliver it as a `Content-Signal` HTTP response header via Vercel
# instead of a robots.txt line, so no crawler's parser trips on it.

# Explicit allow for the major AI crawlers — some respect a tighter
# default than `*` so naming them removes ambiguity.

# ── OpenAI ──────────────────────────────────────────────────────────────────
User-agent: GPTBot
Allow: /
Disallow: /_md/

User-agent: OAI-SearchBot
Allow: /
Disallow: /_md/

User-agent: ChatGPT-User
Allow: /
Disallow: /_md/

# ── Anthropic ───────────────────────────────────────────────────────────────
# We are listed in Anthropic's own connector directory, so Claude-originated
# traffic is the highest-intent agent traffic we get. Claude-User (user-initiated
# fetch) and Claude-SearchBot (search indexing) are the CURRENT agents; until
# 2026-08-07 this file named only the retired Claude-Web and omitted both.
User-agent: ClaudeBot
Allow: /
Disallow: /_md/

User-agent: Claude-User
Allow: /
Disallow: /_md/

User-agent: Claude-SearchBot
Allow: /
Disallow: /_md/

User-agent: anthropic-ai
Allow: /
Disallow: /_md/

# Retired name, kept because an old crawler build costs nothing to honour.
User-agent: Claude-Web
Allow: /
Disallow: /_md/

# ── Perplexity ──────────────────────────────────────────────────────────────
User-agent: PerplexityBot
Allow: /
Disallow: /_md/

User-agent: Perplexity-User
Allow: /
Disallow: /_md/

# ── Google / Apple ──────────────────────────────────────────────────────────
User-agent: Google-Extended
Allow: /
Disallow: /_md/

User-agent: Applebot
Allow: /
Disallow: /_md/

User-agent: Applebot-Extended
Allow: /
Disallow: /_md/

# ── Everyone else ───────────────────────────────────────────────────────────
User-agent: CCBot
Allow: /
Disallow: /_md/

User-agent: Bytespider
Allow: /
Disallow: /_md/

User-agent: Amazonbot
Allow: /
Disallow: /_md/

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /_md/

User-agent: cohere-ai
Allow: /
Disallow: /_md/

User-agent: Diffbot
Allow: /
Disallow: /_md/

User-agent: FacebookBot
Allow: /
Disallow: /_md/

User-agent: DuckAssistBot
Allow: /
Disallow: /_md/

User-agent: MistralAI-User
Allow: /
Disallow: /_md/

User-agent: Ai2Bot
Allow: /
Disallow: /_md/

User-agent: YouBot
Allow: /
Disallow: /_md/

User-agent: Timpibot
Allow: /
Disallow: /_md/

# Sitemap — generated at build time by @astrojs/sitemap.
Sitemap: https://aioproductos.com/sitemap-index.xml