User-agent: googlebot Disallow: User-Agent:MSIECrawler Disallow: / User-agent: Offline Explorer/1.9 Disallow: / User-agent: Wget/1.4.0 Disallow: / User-agent: Wget/* Disallow: / # Tell Yahoo-Inktomi to limit requests in seconds User-agent: Slurp Crawl-delay: 300 # Tell search engines not to index these folders User-Agent: * Disallow: /admin Disallow: /modules/directory/admin Disallow: /bannerman # Polybot User-agent: polybot Disallow: /