# sudohulk.com robots.txt
#
# Policy: everything public is open to everything, including AI training. The point
# of this site is reach: if an assistant can read it, quote it and cite the canonical
# URL, that is the whole goal.
#
# Content-Signal says the same thing in the form machines parse: what a crawler may do
# with a page once it has fetched it. search = index it and link to it; ai-input = use
# it to answer a question as it is asked; ai-train = train a model on it. All three are
# yes. The line only counts inside a User-agent group, so every group carries it.
#
# Markdown twin of every page with something to read: its path plus .md, and
# /index.md for the home page.
#   Machine index:  https://sudohulk.com/llms.txt
#   Full corpus:    https://sudohulk.com/llms-full.txt
#   JSON index:     https://sudohulk.com/content.json
#   How to use it:  https://sudohulk.com/for-ai
#   Licence:        CC BY 4.0, cite https://sudohulk.com/<path>

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /api/agent/
Disallow: /admin
Disallow: /api/
Disallow: /search?

# The groups below are deliberately identical to the one above. They exist so the
# policy is explicit per vendor, and so that turning one bot off later is a one-line
# edit. Under RFC 9309 a named group REPLACES the "*" group for that agent, so each
# group must repeat every line of it.
#
#   Training:        GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, CCBot,
#                    Amazonbot, meta-externalagent, MistralAI-Training
#   Search indexes:  Googlebot, bingbot, OAI-SearchBot, Claude-SearchBot,
#                    PerplexityBot, Applebot, Amzn-SearchBot, DuckDuckBot,
#                    DuckAssistBot, MistralAI-Index, meta-webindexer
#   User-triggered:  ChatGPT-User, Claude-User, Perplexity-User, Amzn-User,
#                    meta-externalfetcher, MistralAI-User
#                    (Perplexity-User and meta-externalfetcher document that they
#                     may ignore robots.txt. They are welcome here anyway.)

User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: Googlebot
User-agent: Google-Extended
User-agent: bingbot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: Amazonbot
User-agent: Amzn-SearchBot
User-agent: Amzn-User
User-agent: meta-externalagent
User-agent: meta-webindexer
User-agent: meta-externalfetcher
User-agent: MistralAI-Training
User-agent: MistralAI-Index
User-agent: MistralAI-User
User-agent: DuckDuckBot
User-agent: DuckAssistBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /api/agent/
Disallow: /admin
Disallow: /api/
Disallow: /search?

Sitemap: https://sudohulk.com/sitemap-index.xml