# almanacstats.com — crawling policy
User-agent: *
Allow: /
Disallow: /cookie-banner/
# служебные отчёты сервера (текст без meta robots): в поиске им не место
Disallow: /ops/

# --- AI / LLM crawlers & AI-search/answer engines: explicitly allowed ---
# Bulk AI crawlers read the English pages only: the /ru/ /es/ /pt-br/ /tr/
# copies carry the same data, and crawling all five burned the Workers quota.
User-agent: OAI-SearchBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: ClaudeBot
Allow: /
Disallow: /ru/
Disallow: /es/
Disallow: /pt-br/
Disallow: /tr/
User-agent: anthropic-ai
Allow: /
User-agent: PerplexityBot
Allow: /
Disallow: /ru/
Disallow: /es/
Disallow: /pt-br/
Disallow: /tr/
User-agent: Perplexity-User
Allow: /
User-agent: Google-Extended
Allow: /
User-agent: Applebot
Allow: /
Disallow: /ops/
User-agent: Applebot-Extended
Allow: /
User-agent: Bingbot
Allow: /
Disallow: /ops/
User-agent: Amazonbot
Allow: /
Disallow: /ru/
Disallow: /es/
Disallow: /pt-br/
Disallow: /tr/

# --- Crawlers that send no visitors: not welcome ---
# Model training and SEO tools. Also blocked at the edge by a Cloudflare WAF rule
# on the user agent: on 30.09.2026 meta-externalagent (96k) and AhrefsBot (38k)
# made 134k of 165k requests and ran the Workers free quota out. GPTBot (OpenAI
# training) followed on 02.10 with 42k in six hours; ChatGPT search and answers
# come through OAI-SearchBot and ChatGPT-User, which stay allowed above.
User-agent: GPTBot
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: meta-externalagent
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: DataForSeoBot
Disallow: /
User-agent: Barkrowler
Disallow: /
User-agent: SeekportBot
Disallow: /
User-agent: serpstatbot
Disallow: /

Sitemap: https://almanacstats.com/seo/sitemap-data.xml
Sitemap: https://almanacstats.com/sitemap.xml