User-agent: *
Allow: /
Allow: /sitemap.xml

Sitemap: https://42117.com/sitemap.xml

# Allow crawling of all content
User-agent: Googlebot
Allow: /
Allow: /sitemap.xml

User-agent: Bingbot
Allow: /
Allow: /sitemap.xml

User-agent: Slurp
Allow: /
Allow: /sitemap.xml

# Prevent crawling of certain directories
Disallow: /vendors/
Disallow: /.vscode/
Disallow: /node_modules/
Disallow: /log/
Disallow: /lint-report/

# Optimize crawling
Crawl-delay: 10