# /robots.txt file for http://myrddin.org/ # mail anton@myrddin.org for constructive criticism Disallow: /email-addresses/ # Checking the following will tell me who has the dodgy spiders # grep \/email-addresses access.log | awk '{print $12}' | uniq User-agent: * Disallow: /cgi-bin/ Disallow: /cgi-sys/ Disallow: /dev/ Disallow: /errors/ Disallow: /images/ Disallow: /includes/ Disallow: /wp*