# exclude help system from robots
User-agent: *
Disallow: /manual/
Disallow: /doc/
Disallow: /gif/
Disallow: /typo3temp/
Disallow: /typo3conf/

# but allow htdig to index our doc-tree
User-agent: susedig
Disallow:
# disallow stress test
user-agent: stress-agent
Disallow: /

#slow down msn bot crawling too fast, crawl-delay value in seconds
User-Agent: msnbot
Crawl-Delay: 2

#slow down yahoo bot crawling too fast, crawl-delay value in seconds
User-Agent: Slurp
Crawl-Delay: 2

#disallow archive.org mirroring spider
User-agent: ia_archiver
Disallow: / 

#disallow dugg mirroring spider
User-agent: duggmirror
Disallow: /