User-agent: SemrushBot/6~bl Disallow: / User-agent: DotBot/1.1 Disallow: / User-agent: MJ12bot/v1.4.8 Disallow: / User-agent: BLEXBot/1.0 Disallow: / User-agent: AhrefsBot/7.0 Disallow: / User-agent: kinjabot Allow: /blog/ # exclude help system from robots User-agent: * Disallow: /cgi-bin/ Disallow: /_private/ Disallow: /_vti_bin/ Disallow: /_vti_log/ Disallow: /_vti_pvt/ Disallow: /_vti_txt/ # but allow htdig to index our doc-tree User-agent: susedig Disallow: # disallow stress test user-agent: stress-agent Disallow: / # disallow stress test user-agent: OmniExplorer_Bot Disallow: / User-agent: * Disallow: /auditions/# BEGIN XML-SITEMAP-PLUGIN Sitemap: http://coulmont.com/sitemap.xml.gz # END XML-SITEMAP-PLUGIN