# Allow all crawlers
User-agent: *
Allow: /

# Sitemap location
Sitemap: https://virtuans.ai/sitemap-index.xml
Sitemap: https://virtuans.ai/sitemap.xml
Sitemap: https://virtuans.ai/blog-sitemap.xml

# Prevent crawling of admin areas and sensitive paths
Disallow: /studio/
Disallow: /admin/
Disallow: /api/
Disallow: /_next/
Disallow: /404
Disallow: /500

# Prevent crawling of 404 pages
Disallow: /blog-pages/blog-v1
Disallow: /template-pages/start-here
Disallow: /home-pages/home-v3
Disallow: /utility-page/forgot-password
Disallow: /help-center
Disallow: /integrations
Disallow: /contact-pages/contact-v3
Disallow: /blog-pages/blog-v3
Disallow: /utility-page/landing-page
Disallow: /team
Disallow: /about
Disallow: /careers
Disallow: /template-pages/licenses
Disallow: /template-pages/style-guide
Disallow: /contact-pages/contact-v2
Disallow: /home-pages/home-v2
Disallow: /blog-pages/blog-v2
Disallow: /utility-page/request-a-demo
Disallow: /utility-page/sign-in
Disallow: /utility-page/email-confirmation
Disallow: /terms-conditions
Disallow: /features
Disallow: /template-pages/changelog
Disallow: /utility-page/sign-up
Disallow: /utility-page/coming-soon
Disallow: /demo

# Allow important crawlers with specific directives
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Allow AI crawlers
User-agent: GPTBot
Allow: /
Crawl-delay: 2

User-agent: ChatGPT-User
Allow: /
Crawl-delay: 2

User-agent: Claude-Web
Allow: /
Crawl-delay: 2

# Default crawl delay for other bots
User-agent: *
Crawl-delay: 5