# robots.txt for http://montereylawngarden.com/
# Revised 2006-06-15 17:45pm pst
#
# Exclude images from Google
# http://www.google.com/remove.html#images
# Yahoo Slurp 
# http://help.yahoo.com/help/us/ysearch/slurp/slurp-02.html
# btbot http://www.btbot.com/btbot.html
# Shim-Crawler 
# http://ow.bbclone.de/2005/10/23/unveiled-details-about-the-shim-crawler/
# 2005-11-09 10am Shim-Crawler seems OK, but I watch
#
# UPDATED 2006-06-15 PDT
User-agent: BaiDuSpider
User-agent: ComputingSite Robi
User-agent: ComputingSite Robi/1.0
User-agent: Internet Shinchakubin
User-agent: MSIECrawler
User-agent: MSRBot
User-agent: psbot
User-agent: SpiderMan
User-agent: WebReaper
Disallow: /

User-agent: The Python Robot
Disallow: /

User-agent: Googlebot-Image
Disallow: /

User-agent: SurveyBot
Disallow: /

User-agent: WebZIP
Disallow: /

# ALLOWED WITH EXCLUSIONS
User-Agent: msnbot
Crawl-Delay: 10

User-Agent: Slurp
Crawl-Delay: 10

User-agent: Googlebot
Disallow: /*.cgi$
Disallow: /*.css$
Disallow: /*.js$
Disallow: /*.pl$
Disallow: /cgi
Disallow: /images
Disallow: /js
Disallow: /styles
##

User-agent: Alexa (IA Archiver)
User-agent: AskJeeves
User-agent: BecomeBot 
User-agent: BigCliqueBOT
User-agent: BigCliqueBOT/1.03-dev (bigclicbot; http://www.bigclique.com; bot@bigclique.com)
User-agent: boitho.com-dc
User-agent: BruinBot
User-agent: BruinBot Crawler
User-agent: BSpider
User-agent: btbot
User-agent: Digimarc MarcSpider
User-agent: Esther
User-agent: Fast-Webcrawler
User-agent: Girafabot
User-agent: ia_archiver
User-agent: ia_archiver-web.archive.org
User-agent: InfoSeek
User-agent: InfoSeek Robot 1.0
User-agent: Ingrid
User-agent: Jeeves
User-agent: Mediapartners-Google/2.1
User-agent: Mediapartners
User-Agent: msnbot
User-agent: Pita (webmaster@pita.stanford.edu)
User-agent: Pita
User-agent: Shim-Crawler
User-agent: Slurp
User-agent: Sproose
User-agent: StackRambler
User-agent: Swooglebot
User-agent: Teoma
User-agent: UCSD Crawl
User-agent: Yahoo-MMCrawler
User-agent: WISENutbot
User-agent: ZealBot

# exclude some areas
Disallow: /cgi
Disallow: /images
Disallow: /js
Disallow: /secret
Disallow: /styles
#
## END