Free tools

Robots.txt Examples

Robots.txt file content for nature.com.

Robot.txt file for: nature.com

      User-agent: *
Sitemap: https://www.nature.com/sitemap.xml
Sitemap: https://www.nature.com/subjects/sitemap.xml
Sitemap: https://www.nature.com/naturecareers/sitemapindex.xml
Sitemap: https://www.nature.com/latest-news/sitemap.xml
Disallow: /search
Disallow: */1000$
Disallow: /*/*/*/pf/
Disallow: /*.otmi$
Disallow: /*/*/*/*/otmi/
Disallow: /*/*/*/*/fp/
Disallow: /protocolexchange/labgroups/
Disallow: /naturecareers/jobs/search
Disallow: /webcasts/*
Disallow: /*draft=*
Disallow: /*foxtrotcallback=*
Disallow: /*origin=*
Disallow: /my-account
Disallow: /*proof=t%2Btarget%3D\*
Disallow: /*proof=*
Disallow: /*/figures
Disallow: /*/tables
Disallow: /*/schemes
Disallow: /*/structures
Disallow: /*/metrics
Disallow: /platform/contextual
Disallow: /articles/*.ris
Disallow: /nature-index/article-api/
Disallow: /nature-index/articles/saudi-arabia
Disallow: /nature-index/articles/saudi-arabia-altmetric
Disallow: /nature-index/city-maps/
Disallow: /nature-index/country-outputs-api/
Disallow: /nature-index/country-suggestion
Disallow: /nature-index/country-suggestion/news-archive
Disallow: /nature-index/global-city-map/data
Disallow: /nature-index/institution-outputs-api/
Disallow: /nature-index/institution-suggestion
Disallow: /nature-index/institution-suggestion/news-archive
Disallow: /naturecareers/session-img/
Disallow: /naturecareers/jobsrss/
Disallow: /nature-index/annual-tables/export/
Disallow: /nature-index/country-outputs/export/
Disallow: /nature-index/country-territory-research-output/export
Disallow: /nature-index/institution-outputs/export/
Disallow: /nature-index/institution-research-output/export
Disallow: /nature-index/news/archive/search/by-country
Disallow: /nature-index/news/archive/search/by-institution
Disallow: /nature-index/annual-tables/2021/academic-normalized
Disallow: */platform/disqus*
Disallow: */cas-redirect/*
User-Agent: Amazonbot
Disallow: /
User-Agent: anthropic-ai
Disallow: /
User-Agent: Applebot-Extended
Disallow: /
User-Agent: Bytespider
Disallow: /
User-Agent: CCBot
Disallow: /
User-Agent: ClaudeBot
Disallow: /
User-Agent: FacebookBot
Disallow: /
User-Agent: GPTBot
Disallow: /
User-Agent: Google-Extended
Disallow: /
User-Agent: GoogleOther
Disallow: /
User-Agent: PerplexityBot
Disallow: /
User-Agent: YouBot
Disallow: /
User-Agent: meta-externalagent
Disallow: /
User-Agent: ImagesiftBot
Disallow: /
User-Agent: cohere-ai
Disallow: /
User-Agent: Diffbot
Disallow: /
User-Agent: FriendlyCrawler
Disallow: /
User-Agent: img2dataset
Disallow: /
User-Agent: omgili
Disallow: /
User-Agent: omgilibot
Disallow: /
User-Agent: Claude-Web
Disallow: /
User-Agent: ChatGPT-User
Disallow: /