# exclude help system from robots
USER-AGENT: *
DISALLOW: /manual/ 
DISALLOW: /doc/ 
DISALLOW: /gif/
DISALLOW: /ftp/
DISALLOW: /dave_ortlepp/
DISALLOW: /Y26/
# but allow htdig to index our doc-tree
uSER-AGENT: susedig
DISALLOW:
# disallow stress test
USER-AGENT: stress-agent
DISALLOW: /