Free tools

Robots.txt Examples

Robots.txt file content for trustpilot.com.

Robot.txt file for: trustpilot.com

      User-agent: Googlebot
Disallow: /error
Disallow: /evaluate/
Disallow: /evaluate-link
Disallow: /evaluate-unique-link
Disallow: /pingdom
Disallow: /reviews/
Disallow: /review/*/transparency
Disallow: /*?*editmode=
Disallow: /api/*
Disallow: /*?*languages=
Disallow: /*?*stars=
Disallow: /*?*sort=
Disallow: /*?*verified=
Disallow: /*?*aspects=
Disallow: /*?*replies=
Disallow: /*?*date=
Disallow: /*?*trustscore=
Disallow: /*?*claimed=
Disallow: /*?*country=
Disallow: /*?*location=
Disallow: /*?*subcategories=
Disallow: /search
Disallow: /sessions
Disallow: /submitted/review
Disallow: /users/
Allow: /users/connect
Disallow: /users/connect?redirect
Disallow: /users/connect?signup
Disallow: /users/activate
Disallow: /users/embed-login-to-edit
Disallow: /users/create
Disallow: /unsubscribe/invitation

User-agent: Mediapartners-Google
Disallow: /error
Disallow: /evaluate/
Disallow: /evaluate-link
Disallow: /evaluate-unique-link
Disallow: /pingdom
Disallow: /reviews/
Disallow: /review/*/transparency
Disallow: /*?*editmode=
Disallow: /api/*
Disallow: /*?*languages=
Disallow: /*?*stars=
Disallow: /*?*sort=
Disallow: /*?*verified=
Disallow: /*?*aspects=
Disallow: /*?*replies=
Disallow: /*?*date=
Disallow: /*?*trustscore=
Disallow: /*?*claimed=
Disallow: /*?*country=
Disallow: /*?*location=
Disallow: /*?*subcategories=
Disallow: /search
Disallow: /sessions
Disallow: /submitted/review
Disallow: /users/
Allow: /users/connect
Disallow: /users/connect?redirect
Disallow: /users/connect?signup
Disallow: /users/activate
Disallow: /users/embed-login-to-edit
Disallow: /users/create
Disallow: /unsubscribe/invitation

User-agent: Googlebot-Mobile
Disallow: /error
Disallow: /evaluate/
Disallow: /evaluate-link
Disallow: /evaluate-unique-link
Disallow: /pingdom
Disallow: /reviews/
Disallow: /review/*/transparency
Disallow: /*?*editmode=
Disallow: /api/*
Disallow: /*?*languages=
Disallow: /*?*stars=
Disallow: /*?*sort=
Disallow: /*?*verified=
Disallow: /*?*aspects=
Disallow: /*?*replies=
Disallow: /*?*date=
Disallow: /*?*trustscore=
Disallow: /*?*claimed=
Disallow: /*?*country=
Disallow: /*?*location=
Disallow: /*?*subcategories=
Disallow: /search
Disallow: /sessions
Disallow: /submitted/review
Disallow: /users/
Allow: /users/connect
Disallow: /users/connect?redirect
Disallow: /users/connect?signup
Disallow: /users/activate
Disallow: /users/embed-login-to-edit
Disallow: /users/create
Disallow: /unsubscribe/invitation

User-agent: AdsBot-Google
Disallow: /error
Disallow: /evaluate/
Disallow: /evaluate-link
Disallow: /evaluate-unique-link
Disallow: /pingdom
Disallow: /reviews/
Disallow: /review/*/transparency
Disallow: /*?*editmode=
Disallow: /api/*
Disallow: /*?*languages=
Disallow: /*?*stars=
Disallow: /*?*sort=
Disallow: /*?*verified=
Disallow: /*?*aspects=
Disallow: /*?*replies=
Disallow: /*?*date=
Disallow: /*?*trustscore=
Disallow: /*?*claimed=
Disallow: /*?*country=
Disallow: /*?*location=
Disallow: /*?*subcategories=
Disallow: /search
Disallow: /sessions
Disallow: /submitted/review
Disallow: /users/
Allow: /users/connect
Disallow: /users/connect?redirect
Disallow: /users/connect?signup
Disallow: /users/activate
Disallow: /users/embed-login-to-edit
Disallow: /users/create
Disallow: /unsubscribe/invitation

User-agent: AdsBot-Google-Mobile
Disallow: /error
Disallow: /evaluate/
Disallow: /evaluate-link
Disallow: /evaluate-unique-link
Disallow: /pingdom
Disallow: /reviews/
Disallow: /review/*/transparency
Disallow: /*?*editmode=
Disallow: /api/*
Disallow: /*?*languages=
Disallow: /*?*stars=
Disallow: /*?*sort=
Disallow: /*?*verified=
Disallow: /*?*aspects=
Disallow: /*?*replies=
Disallow: /*?*date=
Disallow: /*?*trustscore=
Disallow: /*?*claimed=
Disallow: /*?*country=
Disallow: /*?*location=
Disallow: /*?*subcategories=
Disallow: /search
Disallow: /sessions
Disallow: /submitted/review
Disallow: /users/
Allow: /users/connect
Disallow: /users/connect?redirect
Disallow: /users/connect?signup
Disallow: /users/activate
Disallow: /users/embed-login-to-edit
Disallow: /users/create
Disallow: /unsubscribe/invitation

User-agent: bingbot
Disallow: /error
Disallow: /evaluate/
Disallow: /evaluate-link
Disallow: /evaluate-unique-link
Disallow: /pingdom
Disallow: /reviews/
Disallow: /review/*/transparency
Disallow: /*?*editmode=
Disallow: /api/*
Disallow: /*?*languages=
Disallow: /*?*stars=
Disallow: /*?*sort=
Disallow: /*?*verified=
Disallow: /*?*aspects=
Disallow: /*?*replies=
Disallow: /*?*date=
Disallow: /*?*trustscore=
Disallow: /*?*claimed=
Disallow: /*?*country=
Disallow: /*?*location=
Disallow: /*?*subcategories=
Disallow: /search
Disallow: /sessions
Disallow: /submitted/review
Disallow: /users/
Allow: /users/connect
Disallow: /users/connect?redirect
Disallow: /users/connect?signup
Disallow: /users/activate
Disallow: /users/embed-login-to-edit
Disallow: /users/create
Disallow: /unsubscribe/invitation

User-agent: msnbot
Disallow: /error
Disallow: /evaluate/
Disallow: /evaluate-link
Disallow: /evaluate-unique-link
Disallow: /pingdom
Disallow: /reviews/
Disallow: /review/*/transparency
Disallow: /*?*editmode=
Disallow: /api/*
Disallow: /*?*languages=
Disallow: /*?*stars=
Disallow: /*?*sort=
Disallow: /*?*verified=
Disallow: /*?*aspects=
Disallow: /*?*replies=
Disallow: /*?*date=
Disallow: /*?*trustscore=
Disallow: /*?*claimed=
Disallow: /*?*country=
Disallow: /*?*location=
Disallow: /*?*subcategories=
Disallow: /search
Disallow: /sessions
Disallow: /submitted/review
Disallow: /users/
Allow: /users/connect
Disallow: /users/connect?redirect
Disallow: /users/connect?signup
Disallow: /users/activate
Disallow: /users/embed-login-to-edit
Disallow: /users/create
Disallow: /unsubscribe/invitation

User-agent: ia_archiver
Disallow: /error
Disallow: /evaluate/
Disallow: /evaluate-link
Disallow: /evaluate-unique-link
Disallow: /pingdom
Disallow: /reviews/
Disallow: /review/*/transparency
Disallow: /*?*editmode=
Disallow: /api/*
Disallow: /*?*languages=
Disallow: /*?*stars=
Disallow: /*?*sort=
Disallow: /*?*verified=
Disallow: /*?*aspects=
Disallow: /*?*replies=
Disallow: /*?*date=
Disallow: /*?*trustscore=
Disallow: /*?*claimed=
Disallow: /*?*country=
Disallow: /*?*location=
Disallow: /*?*subcategories=
Disallow: /search
Disallow: /sessions
Disallow: /submitted/review
Disallow: /users/
Allow: /users/connect
Disallow: /users/connect?redirect
Disallow: /users/connect?signup
Disallow: /users/activate
Disallow: /users/embed-login-to-edit
Disallow: /users/create
Disallow: /unsubscribe/invitation

User-agent: Alexabot
Disallow: /error
Disallow: /evaluate/
Disallow: /evaluate-link
Disallow: /evaluate-unique-link
Disallow: /pingdom
Disallow: /reviews/
Disallow: /review/*/transparency
Disallow: /*?*editmode=
Disallow: /api/*
Disallow: /*?*languages=
Disallow: /*?*stars=
Disallow: /*?*sort=
Disallow: /*?*verified=
Disallow: /*?*aspects=
Disallow: /*?*replies=
Disallow: /*?*date=
Disallow: /*?*trustscore=
Disallow: /*?*claimed=
Disallow: /*?*country=
Disallow: /*?*location=
Disallow: /*?*subcategories=
Disallow: /search
Disallow: /sessions
Disallow: /submitted/review
Disallow: /users/
Allow: /users/connect
Disallow: /users/connect?redirect
Disallow: /users/connect?signup
Disallow: /users/activate
Disallow: /users/embed-login-to-edit
Disallow: /users/create
Disallow: /unsubscribe/invitation

User-agent: ScoutJet
Disallow: /error
Disallow: /evaluate/
Disallow: /evaluate-link
Disallow: /evaluate-unique-link
Disallow: /pingdom
Disallow: /reviews/
Disallow: /review/*/transparency
Disallow: /*?*editmode=
Disallow: /api/*
Disallow: /*?*languages=
Disallow: /*?*stars=
Disallow: /*?*sort=
Disallow: /*?*verified=
Disallow: /*?*aspects=
Disallow: /*?*replies=
Disallow: /*?*date=
Disallow: /*?*trustscore=
Disallow: /*?*claimed=
Disallow: /*?*country=
Disallow: /*?*location=
Disallow: /*?*subcategories=
Disallow: /search
Disallow: /sessions
Disallow: /submitted/review
Disallow: /users/
Allow: /users/connect
Disallow: /users/connect?redirect
Disallow: /users/connect?signup
Disallow: /users/activate
Disallow: /users/embed-login-to-edit
Disallow: /users/create
Disallow: /unsubscribe/invitation

User-agent: Slurp
Disallow: /error
Disallow: /evaluate/
Disallow: /evaluate-link
Disallow: /evaluate-unique-link
Disallow: /pingdom
Disallow: /reviews/
Disallow: /review/*/transparency
Disallow: /*?*editmode=
Disallow: /api/*
Disallow: /*?*languages=
Disallow: /*?*stars=
Disallow: /*?*sort=
Disallow: /*?*verified=
Disallow: /*?*aspects=
Disallow: /*?*replies=
Disallow: /*?*date=
Disallow: /*?*trustscore=
Disallow: /*?*claimed=
Disallow: /*?*country=
Disallow: /*?*location=
Disallow: /*?*subcategories=
Disallow: /search
Disallow: /sessions
Disallow: /submitted/review
Disallow: /users/
Allow: /users/connect
Disallow: /users/connect?redirect
Disallow: /users/connect?signup
Disallow: /users/activate
Disallow: /users/embed-login-to-edit
Disallow: /users/create
Disallow: /unsubscribe/invitation

User-agent: Twitterbot
Disallow: /error
Disallow: /evaluate/
Disallow: /evaluate-link
Disallow: /evaluate-unique-link
Disallow: /pingdom
Disallow: /reviews/
Disallow: /review/*/transparency
Disallow: /*?*editmode=
Disallow: /api/*
Disallow: /*?*languages=
Disallow: /*?*stars=
Disallow: /*?*sort=
Disallow: /*?*verified=
Disallow: /*?*aspects=
Disallow: /*?*replies=
Disallow: /*?*date=
Disallow: /*?*trustscore=
Disallow: /*?*claimed=
Disallow: /*?*country=
Disallow: /*?*location=
Disallow: /*?*subcategories=
Disallow: /search
Disallow: /sessions
Disallow: /submitted/review
Disallow: /users/
Allow: /users/connect
Disallow: /users/connect?redirect
Disallow: /users/connect?signup
Disallow: /users/activate
Disallow: /users/embed-login-to-edit
Disallow: /users/create
Disallow: /unsubscribe/invitation

User-agent: Baiduspider
Disallow: /error
Disallow: /evaluate/
Disallow: /evaluate-link
Disallow: /evaluate-unique-link
Disallow: /pingdom
Disallow: /reviews/
Disallow: /review/*/transparency
Disallow: /*?*editmode=
Disallow: /api/*
Disallow: /*?*languages=
Disallow: /*?*stars=
Disallow: /*?*sort=
Disallow: /*?*verified=
Disallow: /*?*aspects=
Disallow: /*?*replies=
Disallow: /*?*date=
Disallow: /*?*trustscore=
Disallow: /*?*claimed=
Disallow: /*?*country=
Disallow: /*?*location=
Disallow: /*?*subcategories=
Disallow: /search
Disallow: /sessions
Disallow: /submitted/review
Disallow: /users/
Allow: /users/connect
Disallow: /users/connect?redirect
Disallow: /users/connect?signup
Disallow: /users/activate
Disallow: /users/embed-login-to-edit
Disallow: /users/create
Disallow: /unsubscribe/invitation

User-agent: FacebookBot
Disallow: /error
Disallow: /evaluate/
Disallow: /evaluate-link
Disallow: /evaluate-unique-link
Disallow: /pingdom
Allow: /reviews/
Disallow: /review/*/transparency
Disallow: /*?*editmode=
Disallow: /api/*
Disallow: /*?*languages=
Disallow: /*?*stars=
Disallow: /*?*sort=
Disallow: /*?*verified=
Disallow: /*?*aspects=
Disallow: /*?*replies=
Disallow: /*?*date=
Disallow: /*?*trustscore=
Disallow: /*?*claimed=
Disallow: /*?*country=
Disallow: /*?*location=
Disallow: /*?*subcategories=
Disallow: /search
Disallow: /sessions
Disallow: /submitted/review
Disallow: /users/
Allow: /users/connect
Disallow: /users/connect?redirect
Disallow: /users/connect?signup
Disallow: /users/activate
Disallow: /users/embed-login-to-edit
Disallow: /users/create
Disallow: /unsubscribe/invitation

User-agent: facebookexternalhit
Disallow: /error
Disallow: /evaluate/
Disallow: /evaluate-link
Disallow: /evaluate-unique-link
Disallow: /pingdom
Allow: /reviews/
Disallow: /review/*/transparency
Disallow: /*?*editmode=
Disallow: /api/*
Disallow: /*?*languages=
Disallow: /*?*stars=
Disallow: /*?*sort=
Disallow: /*?*verified=
Disallow: /*?*aspects=
Disallow: /*?*replies=
Disallow: /*?*date=
Disallow: /*?*trustscore=
Disallow: /*?*claimed=
Disallow: /*?*country=
Disallow: /*?*location=
Disallow: /*?*subcategories=
Disallow: /search
Disallow: /sessions
Disallow: /submitted/review
Disallow: /users/
Allow: /users/connect
Disallow: /users/connect?redirect
Disallow: /users/connect?signup
Disallow: /users/activate
Disallow: /users/embed-login-to-edit
Disallow: /users/create
Disallow: /unsubscribe/invitation

User-agent: DuckDuckBot
Disallow: /error
Disallow: /evaluate/
Disallow: /evaluate-link
Disallow: /evaluate-unique-link
Disallow: /pingdom
Disallow: /reviews/
Disallow: /review/*/transparency
Disallow: /*?*editmode=
Disallow: /api/*
Disallow: /*?*languages=
Disallow: /*?*stars=
Disallow: /*?*sort=
Disallow: /*?*verified=
Disallow: /*?*aspects=
Disallow: /*?*replies=
Disallow: /*?*date=
Disallow: /*?*trustscore=
Disallow: /*?*claimed=
Disallow: /*?*country=
Disallow: /*?*location=
Disallow: /*?*subcategories=
Disallow: /search
Disallow: /sessions
Disallow: /submitted/review
Disallow: /users/
Allow: /users/connect
Disallow: /users/connect?redirect
Disallow: /users/connect?signup
Disallow: /users/activate
Disallow: /users/embed-login-to-edit
Disallow: /users/create
Disallow: /unsubscribe/invitation

User-agent: sentibot
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: AwarioRssBot
Disallow: /

User-agent: AwarioSmartBot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Kangaroo Bot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: Webzio-Extended
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: *
Disallow: /

Sitemap: https://sitemaps.trustpilot.com/index_en-us.xml
Sitemap: https://trustpilot.com/trust/sitemaps/domain_en-us.xml
Sitemap: https://trustpilot.com/blog/sitemaps/domain_en-us.xml