# Allow all robots to access all content User-agent: * Allow: / Disallow: /cdn-cgi # Specific crawl-delay for major search engines (optional - remove if not needed) # This helps prevent server overload while ensuring good crawling User-agent: Googlebot Crawl-delay: 1 User-agent: Bingbot Crawl-delay: 1 # Sitemap location (update this URL when you create your sitemap) Sitemap: https://www.toolref.com/sitemap.xml # Additional sitemaps if you have category-specific ones # Sitemap: https://www.toolref.com/tools-sitemap.xml # Sitemap: https://www.toolref.com/calculators-sitemap.xml