Robots.txt for dailymail.co.uk

        # Robots.txt for https://www.dailymail.co.uk/ updated 18/11/2024
#
#
# All Robots
User-agent: *
#
# Begin Standard Rules
Disallow: /*readcommentshtml*
Disallow: /*login?redirectPath=
Disallow: /*logout?redirectPath=
Disallow: /*nextThread.html
Disallow: /*previousThread.html
Disallow: /*questionId*
Disallow: /*selectedImage*
Disallow: /*threadIndex=*
Disallow: /*topGallery*
Disallow: /ce/item.cms*
Disallow: /guide/*
Disallow: /registration/*
Disallow: /best-buys/clickout/*
Disallow: /best-buys/m/clickout/*
Disallow: /weather.html?latitude=*
Disallow: /weather.html?old=*
Disallow: /home/search.html*
Disallow: /best-buys/mercury/out*
Disallow: /api/infinite-list.html

#
# Disallow Money for Google News
User-agent: Googlebot-News
Disallow: /tmoney/*
#
# Allow Adsense
User-agent: Mediapartners-Google
Disallow:
#
User-agent: CrystalSemanticsBot
Disallow: /
#
User-agent: CCBot
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: Amazonbot
User-agent: anthropic-ai
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: Claude-Web
User-agent: ClaudeBot
User-agent: cohere-ai
User-agent: Diffbot
User-agent: FacebookBot
User-agent: FriendlyCrawler
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: GPTBot
User-agent: iaskspider/2.0
User-agent: ICC-Crawler
User-agent: ImagesiftBot
User-agent: img2dataset
User-agent: ISSCyberRiskCrawler
User-agent: Kangaroo Bot
User-agent: Meltwater
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: OAI-SearchBot
User-agent: omgili
User-agent: omgilibot
User-agent: PerplexityBot
User-agent: PetalBot
User-agent: Scrapy
User-agent: Seekr
User-agent: Sidetrade indexer bot
User-agent: Timpibot
User-agent: VelenPublicWebCrawler
User-agent: Webzio-Extended
User-agent: YouBot
Disallow: /
#
# Sitemap Files
Sitemap: https://www.dailymail.co.uk/google-news-sitemap.xml
Sitemap: https://www.dailymail.co.uk/sitemap-channels.xml
Sitemap: https://www.dailymail.co.uk/sitemap-columnist.xml
Sitemap: https://www.dailymail.co.uk/sitemap-topics.xml
Sitemap: https://www.dailymail.co.uk/sitemap-videos-year~2021.xml
Sitemap: https://www.dailymail.co.uk/sitemap-facts-year~2021.xml
Sitemap: https://www.dailymail.co.uk/sitemap-articles-year~2021.xml
Sitemap: https://www.dailymail.co.uk/sitemap-videos-year~2022.xml
Sitemap: https://www.dailymail.co.uk/sitemap-articles-year~2022.xml
Sitemap: https://www.dailymail.co.uk/sitemap-facts-year~2022.xml
Sitemap: https://www.dailymail.co.uk/best-buys/sitemap.xml
Sitemap: https://www.dailymail.co.uk/sitemap-videos-year~2023.xml
Sitemap: https://www.dailymail.co.uk/sitemap-articles-year~2023.xml
Sitemap: https://www.dailymail.co.uk/sitemap-facts-year~2023.xml
Sitemap: https://www.dailymail.co.uk/sitemap-videos-year~2024.xml
Sitemap: https://www.dailymail.co.uk/sitemap-articles-year~2024.xml
Sitemap: https://www.dailymail.co.uk/sitemap-authors.xml