# Block AI crawlers that ignore robots.txt best practices
# See: https://www.reddit.com/r/webdev/comments/1qpqapj/metas_crawler_made_11_million_requests_to_my_site/

User-agent: *
Allow: /

Sitemap: https://pagegains.com/sitemap.xml

# AI/ML Crawlers - many ignore robots.txt but blocking helps with some
User-agent: meta-externalagent
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: BingBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Twitterbot
Allow: /