# Disallow requests from TurnitinBot # $Revision: 34874 $ User-agent: TurnitinBot Disallow: / # Disallow requests from Zealbot User-agent: Zealbot Disallow: / # Disallow some requests from Slurp (Inktomi) so they dont see cross-links #User-agent: Slurp #Crawl-delay: 5 #Disallow: /about/ #Disallow: /examples/ User-agent: msnbot Crawl-delay: 5 Disallow: /inc Disallow: /csutility Disallow: /backend Disallow: /clients Disallow: /requestinfo # Allow requests from other robots, except to the disallowed dirs and files User-agent: * Crawl-delay: 5 Disallow: /inc Disallow: /csutility Disallow: /backend Disallow: /clients Disallow: /requestinfo Disallow: /ccsearch Disallow: /ccsearch/