Robots.txt for cdc.gov

        # Ignore FrontPage files
User-agent: *
Disallow: /_borders
Disallow: /_derived
Disallow: /_fpclass
Disallow: /_overlay
Disallow: /_private
Disallow: /_themes
Disallow: /_vti_bin
Disallow: /_vti_cnf
Disallow: /_vti_log
Disallow: /_vti_map
Disallow: /_vti_pvt
Disallow: /_vti_txt

# Do not index the following URLs
Disallow: /travel/
Disallow: /flu/espanol/
Disallow: /migration/
Disallow: /Features/SpinaBifidaProgram/
Disallow: /concussion/HeadsUp/training/
Disallow: /niosh/archive/
Disallow: /TemplatePackage/examples
Disallow: /TemplatePackage/gadgets
Disallow: /TemplatePackage/modules
Disallow: /TemplatePackage/subtopic
Disallow: /TemplatePackage/Templates
Disallow: /TemplatePackage/3.0/examples
Disallow: /TemplatePackage/3.0/Templates
Disallow: /TemplatePackage/4.0/docs
Disallow: /TemplatePackage/4.0/gallery-internal
Disallow: /templatepackage/examples
Disallow: /templatepackage/gadgets
Disallow: /templatepackage/modules
Disallow: /templatepackage/subtopic
Disallow: /templatepackage/Templates
Disallow: /templatepackage/templates
Disallow: /templatepackage/3.0/examples
Disallow: /templatepackage/3.0/Templates
Disallow: /templatepackage/3.0/templates
Disallow: /templatepackage/4.0/docs
Disallow: /templatepackage/4.0/gallery-internal

# Rover is a bad dog
User-agent: Roverbot
Disallow: /

# EmailSiphon is a hunter/gatherer which extracts email addresses for spam-mailers to use
User-agent: EmailSiphon
Disallow: /

# Exclude MindSpider since it appears to be ill-behaved
User-agent: MindSpider
Disallow: /

# Sitemap link per CR14586
Sitemap: https://www.cdc.gov/wcms-auto-sitemap-index.xml