# Doitong robots.txt — last updated 2026-04-28
#
# We're a creator-economy AI generation platform. We *want* search engines and
# AI bots to discover our public works, profiles, and explore pages.
#
# What we DON'T want indexed:
#   - /search?q=…  → infinite parameter space, spider trap
#   - /create/*    → admin SPA (sign-in required), not public
#   - /api/* /graphql → JSON endpoints, not user-facing
#   - /auth/*      → OAuth callbacks
#
# Specific clean URLs we DO want indexed are exposed via sitemap.

User-agent: *
Allow: /
Disallow: /create/
Disallow: /search
Disallow: /api/
Disallow: /graphql
Disallow: /auth/
Disallow: /webhook/
Disallow: /admin/

# Explicit invitations to AI crawlers — we want our works + creators
# discoverable through ChatGPT, Claude, Perplexity, Google AI Overviews.

User-agent: GPTBot
Allow: /
Disallow: /create/
Disallow: /api/

User-agent: ChatGPT-User
Allow: /
Disallow: /create/
Disallow: /api/

User-agent: ClaudeBot
Allow: /
Disallow: /create/
Disallow: /api/

User-agent: anthropic-ai
Allow: /
Disallow: /create/
Disallow: /api/

User-agent: PerplexityBot
Allow: /
Disallow: /create/
Disallow: /api/

User-agent: Perplexity-User
Allow: /
Disallow: /create/
Disallow: /api/

User-agent: Google-Extended
Allow: /
Disallow: /create/
Disallow: /api/

User-agent: Bytespider
Allow: /
Disallow: /create/
Disallow: /api/

User-agent: Amazonbot
Allow: /
Disallow: /create/
Disallow: /api/

# Master sitemap-index — points to landing pages, creator profiles,
# work pages (chunked), and faceted hubs (models/styles/tags).

# Legacy entries (kept for backwards-compatibility with crawlers that
# already cached these — sitemap-index.xml lists them as children).

Sitemap: https://doitong.ru/sitemap.xml