#/robots.txt file for exclusion
#syntax
#User-agent: excite
#Disallow: / * would disallow all
#Disallow: /cgi-bin * would disallow all files in the CGI-BIN file