Robots.txt for nypost.com

        Sitemap: https://nypost.com/news-sitemap.xml

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /account/
Allow: /account/subscribe
User-agent: *
Disallow: /tag/credible/
Disallow: /personal-finance/
Disallow: /banking/
Disallow: /credit-cards/
Disallow: /loans/
Disallow: /personal-loans/
Disallow: /refinance-student-loans/
Disallow: /student-loans/
Disallow: /mortgages/
Disallow: /home-equity/
Disallow: /mortgage-rates/
Disallow: /mortgage-refinance/
Disallow: /5850/

User-agent: *
Disallow: /search/
Disallow: /horoscope/
Disallow: /*?x=
Disallow: /*?action=
Disallow: /*&action=
Disallow: /*?adid=
Disallow: /*&adid=
Disallow: /*?bitrate=
Disallow: /*&bitrate=
Disallow: /*?channel=
Disallow: /*&channel=
Disallow: /*?clipid=
Disallow: /*&clipid=
Disallow: /*?contentCollection=
Disallow: /*&contentCollection=
Disallow: /*?contentId=
Disallow: /*&contentId=
Disallow: /*?ei=
Disallow: /*&ei=
Disallow: /*?format=
Disallow: /*&format=
Disallow: /*?height=
Disallow: /*&height=
Disallow: /*?iframe=
Disallow: /*&iframe=
Disallow: /*?mediaId=
Disallow: /*&mediaId=
Disallow: /*?module=
Disallow: /*&module=
Disallow: /*?monthnum=
Disallow: /*&monthnum=
Disallow: /*?pgtype=
Disallow: /*&pgtype=
Disallow: /*?preview=
Disallow: /*&preview=
Disallow: /*?priority=
Disallow: /*&priority=
Disallow: /*?ref=
Disallow: /*&ref=
Disallow: /*?referrer=
Disallow: /*&referrer=
Disallow: /*?sa=
Disallow: /*&sa=
Disallow: /*?section=
Disallow: /*§ion=
Disallow: /*?share=
Disallow: /*&share=
Disallow: /*?TB_iframe=
Disallow: /*&TB_iframe=
Disallow: /*?TBL_Prod100Page=
Disallow: /*&TBL_Prod100Page=
Disallow: /*?theme_preview=
Disallow: /*&theme_preview=
Disallow: /*?usg=
Disallow: /*&usg=
Disallow: /*?ved=
Disallow: /*&ved=
Disallow: /*?version=
Disallow: /*&version=
Disallow: /*?VID=
Disallow: /*&VID=
Disallow: /*?vxBitrate=
Disallow: /*&vxBitrate=
Disallow: /*?vxChannel=
Disallow: /*&vxChannel=
Disallow: /*?vxClipId=
Disallow: /*&vxClipId=
Disallow: /*?vxSiteId=
Disallow: /*&vxSiteId=
Disallow: /*?width=
Disallow: /*&width=
Disallow: /*?year=
Disallow: /*&year=
Disallow: /*.xml?uuid=
Disallow: /*?post_type=feedback&p=
Disallow: /*?wptouch_preview_theme=
Disallow: /*>
Disallow: /?attachment_id=
Disallow: /affiliate/
Disallow: /ai_log/
Disallow: /*?*pg99.asia*
Disallow: /*&*pg99.asia*
Disallow: /*?*gg155.cn*
Disallow: /*&*gg155.cn*
Disallow: /*?*To66.Asia*
Disallow: /*&*To66.Asia*

User-agent: *
Disallow: /new-york-post-instagram-feed/
Disallow: /new-york-post-tiktok-feed/
Disallow: /alexa-new-york-post-instagram-feed/
Disallow: /alexa-new-york-post-tiktok-feed/

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: NewsNow
Disallow: /

User-agent: Perplexity-ai
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: PiplBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: news-please
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: meta-externalfetcher
Disallow: /

# Sitemap archive
Sitemap: https://nypost.com/sitemap-502.xml
Sitemap: https://nypost.com/sitemap-1865.xml
Sitemap: https://nypost.com/sitemap-1999.xml
Sitemap: https://nypost.com/sitemap-2000.xml
Sitemap: https://nypost.com/sitemap-2001.xml
Sitemap: https://nypost.com/sitemap-2002.xml
Sitemap: https://nypost.com/sitemap-2003.xml
Sitemap: https://nypost.com/sitemap-2004.xml
Sitemap: https://nypost.com/sitemap-2005.xml
Sitemap: https://nypost.com/sitemap-2006.xml
Sitemap: https://nypost.com/sitemap-2007.xml
Sitemap: https://nypost.com/sitemap-2008.xml
Sitemap: https://nypost.com/sitemap-2009.xml
Sitemap: https://nypost.com/sitemap-2010.xml
Sitemap: https://nypost.com/sitemap-2011.xml
Sitemap: https://nypost.com/sitemap-2012.xml
Sitemap: https://nypost.com/sitemap-2013.xml
Sitemap: https://nypost.com/sitemap-2014.xml
Sitemap: https://nypost.com/sitemap-2015.xml
Sitemap: https://nypost.com/sitemap-2016.xml
Sitemap: https://nypost.com/sitemap-2017.xml
Sitemap: https://nypost.com/sitemap-2018.xml
Sitemap: https://nypost.com/sitemap-2019.xml
Sitemap: https://nypost.com/sitemap-2020.xml
Sitemap: https://nypost.com/sitemap-2021.xml
Sitemap: https://nypost.com/sitemap-2022.xml
Sitemap: https://nypost.com/sitemap-2023.xml
Sitemap: https://nypost.com/sitemap-2024.xml

# Additional sitemaps
Sitemap: https://nypost.com/sitemap-nypost-section.xml
Sitemap: https://nypost.com/sitemap-nypost-post_tag.xml
Sitemap: https://nypost.com/sitemap-nypost-authors.xml
Sitemap: https://nypost.com/sitemap-nypost-pages.xml
Sitemap: https://nypost.com/sitemap-nypost-sportstat.xml