# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism

User-agent: *
Disallow: /_admin_images
Disallow: /_images
Disallow: /cgi-bin
Disallow: /cgiemail
Disallow: /easynews
Disallow: /jscode
Disallow: /styles
Disallow: /thumbnails