# =====================================================
# robots.txt template: AI training preference control
# Updated: November 2026
# Source: https://www.transfon.com/blog/robots-txt-ai-training-directive
# IETF draft: draft-kelley-aipref-training-preference-signal-00
# =====================================================

# Standard search engine access (allow all by default)
User-Agent: *
Disallow:

# =====================================================
# Per-company AI training blocks
# Keep these until each company confirms it reads the
# __usage_ai-training token (see timeline below)
#
# Anthropic (ClaudeBot): honours __usage_ai-training from 30 Oct 2026
# Microsoft: honours from 1 Feb 2027
# Google (Google-Extended): honours from early 2027
# OpenAI (GPTBot): coming months, no fixed date
# Meta: expressed support, no date committed
# =====================================================

User-Agent: GPTBot
Disallow: /

User-Agent: ClaudeBot
Disallow: /

User-Agent: Google-Extended
Disallow: /

User-Agent: CCBot
Disallow: /

User-Agent: meta-externalagent
Disallow: /

User-Agent: Bytespider
Disallow: /

# FacebookBot is Meta's AI model training crawler.
# It is distinct from facebookexternalhit (link previews) — do not block facebookexternalhit.
User-Agent: FacebookBot
Disallow: /

# =====================================================
# Cross-vendor AI training preference (IETF draft)
# Blocks any crawler that honours this standard from
# using your content to train generative AI models.
# Does NOT block search indexing or AI grounding.
# Does NOT affect content already collected.
# =====================================================

User-Agent: __usage_ai-training
Disallow: /

# Uncomment and adjust to allow specific paths for AI training use.
# Example: a press kit or public API documentation.
# Allow: /press-kit/
# Allow: /docs/public-api/