44 lines
764 B
Plaintext
44 lines
764 B
Plaintext
# robots.txt for anotherreflections.ru
|
|
# Static site (Astro), no admin area, no parametric URLs.
|
|
|
|
User-agent: *
|
|
Allow: /
|
|
|
|
User-agent: Yandex
|
|
Allow: /
|
|
Host: https://anotherreflections.ru
|
|
|
|
# AI/LLM crawlers — индексировать разрешено,
|
|
# для уточнения политики обучения см. ai.txt
|
|
User-agent: GPTBot
|
|
Allow: /
|
|
|
|
User-agent: ChatGPT-User
|
|
Allow: /
|
|
|
|
User-agent: OAI-SearchBot
|
|
Allow: /
|
|
|
|
User-agent: ClaudeBot
|
|
Allow: /
|
|
|
|
User-agent: Applebot-Extended
|
|
Allow: /
|
|
|
|
User-agent: Google-Extended
|
|
Allow: /
|
|
|
|
User-agent: CCBot
|
|
Allow: /
|
|
|
|
User-agent: PerplexityBot
|
|
Allow: /
|
|
|
|
User-agent: anthropic-ai
|
|
Allow: /
|
|
|
|
Sitemap: https://anotherreflections.ru/sitemap-index.xml
|
|
Sitemap: https://anotherreflections.ru/sitemap.txt
|
|
|
|
Crawl-delay: 2
|