# Yechiel's Fishery Farm — open to search, AI discovery, and AI training.
#
# ClaudeBot, Claude-User, Claude-SearchBot, GPTBot, OAI-SearchBot,
# ChatGPT-User, PerplexityBot, Perplexity-User, Google-Extended,
# Applebot-Extended and meta-externalagent are all permitted by the
# wildcard group below. There are deliberately no named groups, so
# that nothing can override the wildcard.
#
# As a condition of accessing this website, you agree to abide by the
# following content signals:
#   (a) content-signal = yes  -> you may collect content for that use
#   (b) content-signal = no   -> you may not collect content for that use
#   (c) signal absent         -> neither granted nor restricted
# Signals:
#   search    build a search index and show results
#   ai-input  use content as live input for AI answers (RAG / grounding)
#   ai-train  train or fine-tune AI models

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /api/
# ...but NOT the Markdown mirror of the homepage. The root layout advertises it
# as <link rel="alternate" type="text/markdown"> specifically so AI agents can
# fetch a clean copy, and the Content-Signal above grants ai-input/ai-train.
# Without this carve-out, Disallow: /api/ barred every robots-respecting crawler
# from the one resource we are deliberately handing them. A more specific Allow
# wins over a broader Disallow, and it must stay inside this same group.
Allow: /api/homepage-md
# /api/chat is already inside Disallow: /api/, but it is pinned explicitly so
# that a future narrowing of the /api/ rule cannot silently invite crawlers to
# hammer a paid-LLM endpoint. Same group, no blank line — see header comment.
Disallow: /api/chat
Disallow: /free-chapter-files/

Sitemap: https://www.yechielsfisheryfarm.com/sitemap.xml