User-agent: Googlebot
Allow: *
User-agent: Googlebot-Image
Allow: * 
User-agent: Slurp
Allow: * 
User-agent: yahoo-mmcrawler
Allow: *
User-agent: msnbot
Allow: *
User-agent: *
Disallow: /
Crawl-delay: 15 
# Block all search engines from the cgi-bin directory to prevent
# system from getting overloaded with gbrowse requests
Disallow: /cgi-bin/
# Block all requests to Pathway tools - if it gets too many crawler requests, it will cause problems
Disallow: /pathway/
#Block access to sequence feature pages - they can be slow sometimes and there
#are a lot of them
Disallow: *sequenceFeatureDetailsReport*$