# yfarmx.com robots policy
# Search engines and AI assistants are welcome on every content page.
# Site guide for machines: https://yfarmx.com/llms.txt
#
# Licensing (RSL 1.0): the journalism is free to index, ground answers on
# and train from, with attribution. Original photography and artwork under
# /media/ and /img/ need a licence to train on. The tracker datasets are
# CC BY 4.0. Full terms: https://yfarmx.com/terms/#data-reuse
License: https://yfarmx.com/license.xml
#
# Content Signals (contentsignals.org): yes to search, ai-input and ai-train.
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /desk/
Disallow: /api/
Disallow: /offline/
Disallow: /sw.js
Disallow: /manifest.webmanifest
Disallow: /app/
Disallow: /pagefind/
Allow: /

# ---- AI crawlers & assistants: explicitly welcome ----
# OpenAI
User-agent: GPTBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /desk/
Disallow: /api/
Disallow: /offline/
Disallow: /sw.js
Disallow: /manifest.webmanifest
Disallow: /app/
Disallow: /pagefind/
Allow: /

User-agent: OAI-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /desk/
Disallow: /api/
Disallow: /offline/
Disallow: /sw.js
Disallow: /manifest.webmanifest
Disallow: /app/
Disallow: /pagefind/
Allow: /

User-agent: ChatGPT-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /desk/
Disallow: /api/
Disallow: /offline/
Disallow: /sw.js
Disallow: /manifest.webmanifest
Disallow: /app/
Disallow: /pagefind/
Allow: /

# Anthropic
User-agent: ClaudeBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /desk/
Disallow: /api/
Disallow: /offline/
Disallow: /sw.js
Disallow: /manifest.webmanifest
Disallow: /app/
Disallow: /pagefind/
Allow: /

User-agent: Claude-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /desk/
Disallow: /api/
Disallow: /offline/
Disallow: /sw.js
Disallow: /manifest.webmanifest
Disallow: /app/
Disallow: /pagefind/
Allow: /

User-agent: Claude-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /desk/
Disallow: /api/
Disallow: /offline/
Disallow: /sw.js
Disallow: /manifest.webmanifest
Disallow: /app/
Disallow: /pagefind/
Allow: /

# Perplexity
User-agent: PerplexityBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /desk/
Disallow: /api/
Disallow: /offline/
Disallow: /sw.js
Disallow: /manifest.webmanifest
Disallow: /app/
Disallow: /pagefind/
Allow: /

User-agent: Perplexity-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /desk/
Disallow: /api/
Disallow: /offline/
Disallow: /sw.js
Disallow: /manifest.webmanifest
Disallow: /app/
Disallow: /pagefind/
Allow: /

# Google AI and Apple Intelligence permission tokens
User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

# Meta AI crawler
User-agent: meta-externalagent
Disallow: /

# Common Crawl (feeds many open models)
User-agent: CCBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /desk/
Disallow: /api/
Disallow: /offline/
Disallow: /sw.js
Disallow: /manifest.webmanifest
Disallow: /app/
Disallow: /pagefind/
Allow: /

# Backlink database crawlers
User-agent: DataForSeoBot
Disallow: /

User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: Barkrowler
User-agent: SERankingBacklinksBot
Disallow: /

Sitemap: https://yfarmx.com/sitemap-index.xml
Sitemap: https://yfarmx.com/news-sitemap.xml