User-agent: *
# Allow all by default, then disallow specific paths
Allow: /

# Explicitly allow indexable content paths (fixes "Blocked by robots.txt" issues)
Allow: /ideas/
Allow: /locations/
Allow: /blog/

# Allow crawlable API routes BEFORE disallowing /api/ (most specific rule wins)
Allow: /api/og
Allow: /api/og/
Allow: /api/ideas/count-today

# Disallow admin, protected, and non-indexable areas
Disallow: /admin/
Disallow: /protected/
Disallow: /api/

# /profile and /myideas are not listed here on purpose: pages use noindex, but blocking
# them prevented Googlebot from refetching and clearing "Indexed, though blocked by robots.txt".

# /roast/results?ideaId=… must be crawlable so Google can follow 301 → /ideas/[slug].
# Do not disallow /roast/results (was causing "Blocked by robots.txt").

# Allow AI crawlers for enhanced content visibility
User-agent: GPTBot
Allow: /
User-agent: Google-Extended
Allow: /
User-agent: Claude-Web
Allow: /
User-agent: ChatGPT-User
Allow: /

# Sitemap index (lists all sitemaps including 16k+ idea pages)
Sitemap: https://www.dontbuildthis.com/sitemap-index.xml