#this robots.txt file will index everything except the /cgi-bin/ directory
#if there are other directories to DISALLOW add them as required  e.g. disallow /images/
#there MUST be a blank line or comment line after the last disallow directive
user-agent: *
disallow: /cgi-bin/