# =========================================================
# AI.TXT — AI CRAWLER & LLM GUIDANCE FILE
# Version: 1.0
# Last Updated: 2026-05
# =========================================================

# PURPOSE:
# This file provides structured guidance for:
# - Large Language Models (LLMs)
# - AI Crawlers & Retrieval Bots
# - Generative Engines (ChatGPT, Gemini, Claude, Perplexity)
# - Multimodal AI Systems (Text, Image, Video)
# - RAG Pipelines & Knowledge Graph Systems
#
# It complements:
# - /robots.txt (crawl control)
# - /llms.txt (training permissions)
# without conflict.

# =========================================================
# GLOBAL AI ACCESS POLICY
# =========================================================
User-agent: *
Allow: /
Disallow: /wp-admin/
Disallow: /private/
Disallow: /internal/

# AI PRIORITY SIGNALS:
# - Prefer canonical URLs
# - Use structured data (Schema.org / JSON-LD)
# - Preserve entity relationships
# - Prioritize topical clusters
# - Favor fresh, versioned content


# =========================================================
# MAJOR AI AGENTS (EXPLICIT ALLOW)
# =========================================================

# OpenAI
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

# Google AI
User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

# Apple AI
User-agent: Applebot-Extended
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# ByteDance / TikTok
User-agent: Bytespider
Allow: /

# Amazon
User-agent: Amazonbot
Allow: /

# Common Crawl
User-agent: CCBot
Allow: /

# Knowledge Graph Crawlers
User-agent: Diffbot
Allow: /

User-agent: YouBot
Allow: /

# Multimodal AI
User-agent: StabilityAI
Allow: /

User-agent: OpenAI-Image
Allow: /

User-agent: Anthropic-Image
Allow: /

# =========================================================
# END OF AI.TXT
# =================