Free tools

Robots.txt Examples

Robots.txt file content for www.nhs.uk.

Robot.txt file for: www.nhs.uk

      User-agent: Ultraseek
Disallow: /
User-Agent: AhrefsBot 
Crawl-Delay: 5 	
User-agent: Algolia Crawler
Disallow: /scorecard/
Disallow: /Services/Dentists/Overview/DentistPatientAcceptanceVerification.aspx*
Disallow: /sign-up-to-be-contacted-for-research
Disallow: /contact-us/get-help-with-signing-up-for-contact-for-coronavirus-vaccine-studies/
User-agent: *
Disallow: /service-search/other-services/Psychological%20therapies%20(IAPT)/LocationSearch/10008
Disallow: /service-search/other-services/Psychological%20therapies%20%28IAPT%29/LocationSearch/10008
Disallow: /service-search/other-services/Psychologicaltherapies(IAPT)/LocationSearch/10008
Disallow: /contact-us/get-help-with-signing-up-for-contact-for-coronavirus-vaccine-studies/
Disallow: /contact-us/get-help-with-asking-for-a-coronavirus-test/
Disallow: /ask-for-a-coronavirus-test
Disallow: /ask-for-a-coronavirus-test/
Disallow: /book-a-coronavirus-vaccination
Disallow: /book-a-coronavirus-vaccination/
Disallow: /externalservices/surveyfeedapi/api/testfeed
Disallow: /Review/List/
Disallow: /Review/LeaveReview/
Disallow: /Conditions/
Disallow: /conditions/pregnancy-and-baby/pages/your-birth-plan.aspx
Disallow: /conditions/coronavirus-covid-19/testing-and-tracing/get-an-antigen-test-to-check-if-you-have-coronavirus/
Disallow: /campaignshowcase/
Disallow: /scorecard/ 
Disallow: /Service-Search/other-services/Cognitive-behavioural-therapy-(CBT)/LocationSearch/1908
Disallow: /Service-Search/other-services/NHS%20psychological%20therapies%20services%20(IAPT),%20including%20cognitive%20behavioural%20therapy%20(CBT)/LocationSearch/10008
Disallow: /service-search/other-services/Psychological-therapy-(NHS-IAPT)/LocationSearch/396
Disallow: /service-search/find-a-psychological-therapies-service/what-happens-when-you-refer-yourself
Disallow: /Service-Search/find-a-psychological-therapies-service/what-happens-when-you-refer-yourself
Disallow: /service-search/find-a-psychological-therapies-service/find-your-gp
Disallow: /Service-Search/find-a-psychological-therapies-service/find-your-gp
Disallow: /service-search/find-a-psychological-therapies-service/results
Disallow: /Service-Search/find-a-psychological-therapies-service/results
Disallow: /Services/Dentists/Overview/DentistPatientAcceptanceVerification.aspx*
Disallow: /tools/pages/toolslibrary.aspx?tag=*
Disallow: /sign-up-to-be-contacted-for-research
Disallow: /service-search/other-services/*/*/Export/*
Disallow: /every-mind-matters/mental-wellbeing-tips/your-mind-plan-quiz/application
User-agent: Googlebot
Disallow: /service-search/dist/images/loading.gif
Sitemap: https://www.nhs.uk/sitemap.xml