# robots.txt - 胡杨导航 (12.xj.cn)
# 新ICP备2025020011号-5
# 新公网安备65292202240012号

# === 通用爬虫规则 ===
User-agent: *
Allow: /
Disallow: /api/
Disallow: /assets/

# AI & LLM 协议声明
# AI-Crawler-Policy: open
# LLM-Training: allowed

# Sitemap
Sitemap: https://12.xj.cn/sitemap-index.xml
Sitemap: https://12.xj.cn/sitemap1.xml

# AI 爬虫友好文件
# ai.txt: https://12.xj.cn/ai.txt
# llms.txt: https://12.xj.cn/llms.txt

# === 搜索引擎爬虫 ===
User-agent: Baiduspider
Allow: /
Crawl-delay: 1

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Sogou web spider
Allow: /

User-agent: 360Spider
Allow: /

User-agent: YandexBot
Allow: /

# === LLMs / AI 爬虫协议 ===

# OpenAI (ChatGPT, GPTBot)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-Search
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: claude-ai
Allow: /

# Google AI (Bard/Gemini)
User-agent: Google-Extended
Allow: /

User-agent: Googlebot-News
Allow: /

# ByteDance (豆包/抖音)
User-agent: Bytespider
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Apple AI
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Common Crawl
User-agent: CCBot
Allow: /

# AI Search Engines
User-agent: YouBot
Allow: /

User-agent: Timpibot
Allow: /

User-agent: AI2Bot
Allow: /

# Diffbot
User-agent: Diffbot
Allow: /

# Awario
User-agent: AwarioBot
Allow: /

# Amazon
User-agent: Amazonbot
Allow: /

# Mozilla AI
User-agent: Mozillabot
Allow: /

# === AI 爬虫协议声明 ===
# 本网站对所有 AI/LLM 爬虫完全开放。
# 允许抓取公开内容用于训练、检索和生成式AI应用。
# 详见 ai.txt 和 llms.txt 获取完整内容说明。