User-agent: Mediapartners-Google* 
# disallow so google ads don't do repeat joins, etc.
Disallow: /cgi-bin/
Disallow: /critters/cgi-bin/
Disallow: /c/cgi-bin/
Allow: /cgi-bin/getpw.cgi
Allow: /cgi-bin/getpw2.cgi
Allow: /cgi-bin/prefill.cgi
User-agent: *
Disallow: /bios/
Disallow: /cgi-bin/
Disallow: /critters/cgi-bin/
Disallow: /blackholes/webdbget.cgi
Disallow: /store/ppverify.cgi
Disallow: cleaner.cgi
Disallow: /cleaner.cgi
Disallow: /contraster.cgi
Disallow: contraster.cgi
Disallow: urler.cgi
Disallow: /urler.cgi
Disallow: getstuff.cgi
Disallow: /getstuff.cgi
Disallow: /predpoll/email.cgi

User-agent: Baiduspider
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: discobot
Disallow: / 

User-agent: AhrefsBot
Disallow: / 

User-agent: sistrix
Disallow: / 

User-agent: ahrefs
Disallow: /