# # /robots.txt file for http://webcrawler.com/
# # mail webmaster@webcrawler.com for constructive criticism
# 
# User-agent: webcrawler
# Disallow:
# 
# User-agent: lycra
# Disallow: /
# 
# User-agent: *
# Disallow: /tmp
# Disallow: /logs
# 
# The first two lines, starting with '#', specify a comment
# 
# The first paragraph specifies that the robot called 'webcrawler' has nothing disallowed: it may go anywhere.
# 
# The second paragraph indicates that the robot called 'lycra' has all relative URLs starting with '/' disallowed. Because all relative URL's on a server start with '/', this means the entire site is closed off.
# 
# The third paragraph indicates that all other robots should not visit URLs starting with /tmp or /log. Note the '*' is a special token, meaning "any other User-agent"; you cannot use wildcard patterns or regular expressions in either User-agent or Disallow lines.
# 
# Two common errors:
# 
#     * Wildcards are _not_ supported: instead of 'Disallow: /tmp/*' just say 'Disallow: /tmp'.
#     * You shouldn't put more than one path on a Disallow line (this may change in a future version of the spec) 
# 

User-agent: *

Disallow: /belizeCLOSED
# Disallow: /bergen
Disallow: /bin
Disallow: /Bonaire
Disallow: /carmel
Disallow: /cgi
# Disallow: /cossedesigns
Disallow: /_db_backups
Disallow: /dennis
Disallow: /dycusreunion
# Disallow: /fish-eggs
# Disallow: /hike
Disallow: /images
Disallow: /jared
Disallow: /llamas
# Disallow: /marine
Disallow: /marine-old
Disallow: /marine-test
Disallow: /micro
Disallow: /mini-mill
Disallow: /misc
Disallow: /mona
Disallow: /old
Disallow: /photos
Disallow: /phpBB30
Disallow: /php_uploads
Disallow: /piwik
Disallow: /powershopping
Disallow: /rick
Disallow: /rosie
Disallow: /ssfm
Disallow: /stats
Disallow: /test
Disallow: /tmp
Disallow: /_private
Disallow: /_vti_bin
Disallow: /_vti_cnf
Disallow: /_vti_log
Disallow: /_vti_pvt
Disallow: /_vti_script
Disallow: /_vti_txt
Disallow: /.hcc.thumbs