#
# list robot agents to disallow
#
# User-agent: *
# Disallow: /
#
# User-agent: Googlebot
# Disallow: /
#
# list specific files robots are not allowed to index
#
# Disallow: /directory/foo.html
# Disallow: /folder2/whatever.html
#