User-Agent: Openfind data gatherer
Disallow: /

User-Agent: ia_archiver
Disallow: /

User-agent: http://www.almaden.ibm.com/cs/crawler 
Disallow: /

User-Agent: fast
Disallow: /

User-Agent: nabot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: baiduspider
Disallow: /

User-agent: fastres.net
Disallow: /

User-agent: seventwentyfour.com
Disallow: /

User-agent: TurnitinBot
Disallow: /hide/