# $Id: robots.txt,v 1.3 2001/05/15 00:17:27 krusch Exp $
#
# This is a file retrieved by webwalkers a.k.a. spiders that 
# conform to a defacto standard.
# See <URL:http://www.robotstxt.org/wc/exclusion.html#robotstxt>
#
# The webmaster for this site is <webmaster@www.newspeakdictionary.com>
#
# Format is:
#       User-agent: <name of spider>
#       Disallow: <nothing> | <path>
# Flag  Date      By      Reason
# $L1   20010710  CMS     finally understood what the file was for!
# -----------------------------------------------------------------------------
User-agent: *
Crawl-delay: 20 
Request-rate: 1/20
Disallow: /aw_stats
Disallow: /_MODS
Disallow: /posting
Disallow: /search
Disallow: /login
Disallow: /vote
#
#User-agent: Googlebot-Image
#Disallow: /
User-agent: Yahoo-MMCrawler
Disallow: /
User-agent: msnbot-MM
Disallow: /