# robots.txt for tensorblue.com

User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /create-blog
Disallow: /create-blog/
Disallow: /create-project
Disallow: /create-project/
Disallow: /create-newsletter
Disallow: /create-newsletter/
Disallow: /newsletter
Disallow: /newsletter/
Disallow: /onboarding
Disallow: /onboarding/
Disallow: /agent-control-room
Disallow: /agent-control-room/
Disallow: /thank-you
Disallow: /categories/
Disallow: /tag/
Disallow: /*?utm_*
Disallow: /*?gclid=
Disallow: /*?fbclid=
Disallow: /*?ref=

# Block scraper / SEO-spying crawlers that waste crawl budget without value.
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# Keep AI training and answer engines fully crawlable for citation visibility.
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

Host: https://tensorblue.com
Sitemap: https://tensorblue.com/sitemap.xml