# This is to allow spiders to search the whole site
# except the directories after the first forward slash.
# Specific files are disallowed by omitting all slashes.
User-agent: *
Disallow: /cgi-bin/