# Production robots.txt for www.ibuildwith.ai # Allow all search engines to crawl the site User-agent: * Allow: / # Sitemap location Sitemap: https://www.ibuildwith.ai/sitemap.xml # Specific crawler optimizations User-agent: Googlebot Allow: / User-agent: Bingbot Allow: / # Block access to admin/private areas (if any exist in the future) Disallow: /admin/ Disallow: /private/ Disallow: /.git/ # Only meaningful after a Stripe payment; also noindex and out of the sitemap Disallow: /thank-you/ # Allow access to all content types Allow: /blog/ Allow: /about-marcelo/ Allow: /contact/ Allow: /newsletter/