# simpler.asia: everything here is meant to be read, by people and by machines.
# Simpler — AI-driven business forecasting + operations platform for SEA MSMEs
# https://simpler.asia
#
# Policy (ported from the audited pre-prod posture, trimmed): only known bad actors get
# explicit blocks; all other AI crawlers — training, search, retrieval — are ALLOWED per
# the Content-Signal in the default block (ai-train=yes). Being in AI training corpora =
# being in future AI answers. The pre-prod file enumerated ~21 internal app pages by
# name (a directory listing for attackers and a rename-drift list); this file disallows
# only the API prefixes — authed app shells render nothing useful to a crawler anyway.

# ── Abusive crawlers — blocked ──
User-agent: Bytespider
Disallow: /

User-agent: AnchorBrowser
Disallow: /

# Every named group below repeats the default group's lines: a bot obeys only the most
# specific group that names it (RFC 9309), so a group without them would drop the
# Content-Signal and /api/ lines for that bot.

# ── AI search / retrieval bots — allowed (shared group) ──
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Amazonbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /api/

# ── Training-opt-in tokens — explicit allows ──
User-agent: GPTBot
User-agent: ClaudeBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /api/

User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: YandexAdditionalBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /api/

# Perplexity-User officially ignores robots.txt (user-initiated fetch); listed for
# explicitness only.
User-agent: Perplexity-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /api/

# ── Traditional search engines — allowed ──
User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
User-agent: YandexBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /api/

# ── Default — all other bots ──
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /api/

# ── AI discovery files ──
# LLM summary: /llms.txt
# AI agent instructions: /agents.json
# The home page as markdown: /index.md
# REST description: /openapi.json
# MCP endpoint: /mcp
# The Simpler family: /family.json

Sitemap: https://simpler.asia/sitemap.xml