User-Agent: Openfind data gatherer
Disallow: /

User-Agent: ia_archiver
Disallow: /

User-agent: http://www.almaden.ibm.com/cs/crawler 
Disallow: /

User-Agent: fast
Disallow: /

User-Agent: nabot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: baiduspider
Disallow: /

User-agent: fastres.net
Disallow: /

User-agent: seventwentyfour.com
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: dotbot 
Disallow: / 

User-agent: Teoma
Disallow: /
Allow: /public

User-Agent: Yetibot
Disallow: /