November 26, 2024

Robots.txt for the-sun.com

        User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

Disallow: /search/
Disallow: /*?s=*
Disallow: *&s=*
Disallow: /?p=*
Disallow: /amp-tealium/*
Disallow: /simwidgets/
Disallow: /app/
Disallow: /sso/login/
Disallow: /wp-login.php
Disallow: /video/
#Archive Sitemaps
Sitemap: https://www.the-sun.com/nav-sitemap.xml
Sitemap: https://www.the-sun.com/news-sitemap.xml
Sitemap: https://www.the-sun.com/author-sitemap.xml
Sitemap: https://www.the-sun.com/video-sitemap.xml


Disallow: /amp-tealium/
Disallow: /archives/


Disallow: /betting/
Disallow: /social-casino/


User-agent: CCBot
Disallow: /


User-agent: anthropic-ai
Disallow: /


User-agent: cohere-ai
Disallow: /


User-agent: ia_archiver
Disallow: /


User-agent: omgili
Disallow: /


User-agent: omgilibot
Disallow: /


User-agent: MJ12bot
Disallow: /


User-agent: PiplBot
Disallow: /


User-agent: Google-Extended
Disallow: /


User-agent: Meltwater
Disallow: /


User-agent: Claude-Web
Disallow: /


User-agent: Claudebot
Disallow: /


User-agent: Perplexity-ai
Disallow: /


User-agent: Seekr
Disallow: /


User-agent: anthropic-aiBytespider
Disallow: /


User-agent: Bytespider
Disallow: /


User-agent: magpie-crawler
Disallow: /


User-agent: NewsNow
Disallow: /


User-agent: news-please
Disallow: /


User-agent: FacebookBot
Disallow: /


User-agent: Applebot
Disallow: /


User-agent: Applebot-Extended
Disallow: /


User-agent: PerplexityBot
Disallow: /


User-agent: Meta-ExternalAgent
Disallow: /

# Sitemap archive
Sitemap: https://www.the-sun.com/sitemap.xml