Robots.txt for food.com

textile

# Directives for all crawlers
User-agent: *
Disallow: /recipeprint.do*
Disallow: /rssapi.do*
# Disallow: /about/tour/
# Disallow: /admin/
# Disallow: /alt/
# Disallow: /mail/
# Disallow: /members/photos.php
# Disallow: /recipe/photo.php
Disallow: /bb/
# Disallow: /void/
# Disallow: /services/
Disallow: /null/
# Exclude page construction elements (commented out some due to new google guidelines)
# Disallow: /pushdownAd.do
# Disallow: /global/header.jsp
# Disallow: /global/pickle-header.jsp
# Disallow: /footer.jsp
# Disallow: /headerbreadcrumbs.do
# Disallow: /getrzuserstats.do
# Disallow: /seasonal-header.do
# Exclude authenticated pages
Disallow: /members/subm/
Disallow: /mail/compose*
Disallow: /cp_asset_stats.do
Disallow: /dpn.do*
Disallow: /mobile/api/
# Exclude search results pages
Disallow: /ingredient-finder/
Disallow: /cookbook-finder/
Disallow: /menu-finder/
Disallow: /recipe-finder/
Disallow: /photo-finder/
Disallow: /member-finder/
Disallow: /search/
Disallow: /books/
Disallow: /community/
# Block invalid Omniture injected URLs within javascript
# Disallow: /*_1
# Disallow: /*_2
# Disallow: /*_3
# Disallow: /*_4
# Disallow: /*_5
# Disallow: /*_6
# Disallow: /*_7
# Disallow: /*_8
# Block .php & .zsp files
Disallow: *.zsp*
Disallow: *.php*
Disallow: /~asset*

# Directives for Mediapartners-Google
User-agent: Mediapartners-Google
Disallow:

# Directives for GoogleBot News
User-agent: Googlebot-News
Allow: /news
Allow: /article
Disallow: /

# Directives for 008
User-agent: 008
Disallow: /

# Chat GPT
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: PerplexityBot
Disallow: /

# Sitemap
Sitemap: https://www.food.com/sitemap.xml
Sitemap: https://www.food.com/news/news-sitemap.xml