User-Agent: * Disallow: /index.php # This spider is particularly aggressive and therefore banned. User-Agent: baiduspider Disallow: / # Doesn't read robots.txt. User-Agent: sohu agent Disallow: /