# =========================
# 主流搜索引擎允许抓取 + AI 禁止规则
# =========================

# Google
User-agent: Googlebot
Disallow: /
User-agent: Googlebot-Image
Disallow: /
User-agent: Googlebot-Video
Disallow: /
User-agent: Googlebot-News
Disallow: /
User-agent: Googlebot-Render
Disallow: /

# Bing
User-agent: Bingbot
Allow: /
User-agent: Bingbot-Render
Allow: /

# 今日头条
User-agent: Bytespider
Allow: /

# 百度
User-agent: Baiduspider
Allow: /
User-agent: Baiduspider-HTML5
Allow: /
User-agent: Baiduspider-Mobile
Allow: /
User-agent: Baiduspider-Render
Allow: /

# 搜狗
User-agent: Sogou Spider
Allow: /
User-agent: Sogou Spider-Render
Allow: /

# 360
User-agent: 360Spider
Allow: /
User-agent: 360Spider-Render
Allow: /

# 神马
User-agent: YisouSpider
Allow: /

# Yandex
User-agent: YandexBot
Allow: /

# Petal
User-agent: PetalBot
Allow: /

# AI 禁止
User-agent: GPTBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /
User-agent: OAI-SearchBot
Disallow: /
User-agent: OAI-ImageBot
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: Claude-SearchBot
Disallow: /
User-agent: Claude-User
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: Google-CloudVertexBot
Disallow: /
User-agent: Meta-ExternalAgent
Disallow: /
User-agent: Meta-ExternalFetcher
Disallow: /
User-agent: Amazonbot
Disallow: /
User-agent: Applebot-Extended
Disallow: /

# 恶意采集器禁止
User-agent: AhrefsBot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: SEOkicks-Robot
Disallow: /
User-agent: spbot
Disallow: /
User-agent: megaindex.com
Disallow: /

# 默认允许所有普通蜘蛛
User-agent: *
Allow: /