Free tools
Robots.txt Examples
Robots.txt file content for news18.com.
Robot.txt file for: news18.com
User-agent: * Allow: / Sitemap: https://www.news18.com/commonfeeds/v1/eng/sitemap/google-news.xml Sitemap: https://www.news18.com/commonfeeds/v1/eng/sitemap/webstories-sitemap.xml Sitemap: https://www.news18.com/commonfeeds/v1/eng/sitemap/google-news/today.xml Sitemap: https://www.news18.com/commonfeeds/v1/eng/sitemap/google-news/yesterday.xml Sitemap: https://www.news18.com/commonfeeds/v1/eng/sitemap-video.xml Disallow: /cricketnext/series/asia-cup-2023.html Disallow: /cricketnext/amp/series/asia-cup-2023.html Disallow: /photogallery/6*.html Disallow: /assembly-elections-2018/*/*/ Disallow: /delhi-assembly-elections-2020/* Disallow: /bihar-assembly-elections-2020/*/ Disallow: /jw_player/video/*/ Disallow: /assembly-elections/uttar-pradesh/*/ Disallow: /assembly-elections-2021/*/*/ Disallow: /amp/assembly-elections-2021/*/ Disallow: /election-result-live/*/ Disallow: /photogallery/page-* Disallow: /cricketnext/series/* Disallow: /cricketnext/amp/series/* Disallow: /cricketnext/photogallery/series/* Disallow: /cricketnext/news/series/* Disallow: /cricketnext/profile/* Disallow: /cricketnext/amp/profile/* Disallow: /amp/cricketnext/* Disallow: /cricketnext/newstopics/* Disallow: /blogs/* Disallow: /indiwo/* Disallow: /preview/* Disallow: /series/* Disallow: /newstopics/* Disallow: /amp/newstopics/* Disallow: /astrology/horoscope/todays-*-horoscope-in-english* Disallow: /astrology/todays-panchang-in-english-* Disallow: /ipl-predict-win-contest/ Disallow: /win-ipl-tickets-contest-2017/ Disallow: /index-test.php Disallow: /nodeapi/ Disallow: /article_without_ad.php* Disallow: /cat_without_ad.php* Disallow: /topic_listing_without_ads.php* Disallow: /cricketnext/wc2019/* Disallow: /minis_ajax.html Disallow: /minis_ajax_mobile.html Disallow: /load_homepage_astrology_horoscope_ajax.html Disallow: /astrology/widget/article-pages-mobile-ajax.php Disallow: /mission-paani/home-page-mission-paani-pledge/ Disallow: /microsite/p-and-g/ajax/video-listing.php Disallow: /bihar-assembly-elections-2020/home-page-ajax/* Disallow: /ipl-2021-chronicles-ajax.html Disallow: /footer_ajax.html Disallow: /cricketnext/revamp/resources/ajax/* Disallow: /section_and_ivideo_widget_ajax.html Disallow: /us-presidential-elections-2016/* Disallow: /gossip/getAjaxgossip/* Disallow: /load_homepage_podcast_ajax.html Disallow: /photogallery_widget_ajax.html Disallow: /microsite/itsnotokay/ajax/video-consumption.php Disallow: /byline/*/videos/* Disallow: /byline/*/news/* Disallow: /byline/*/photos/* Disallow: /amp/amp/* Disallow: /search.php* Disallow: /cricketnext/search/* Disallow: /news18_revamp/* Disallow: /topics/*.html$ Disallow: /topics/*/news/$ Disallow: /topics/*/videos/$ Disallow: /topics/*/photos/$ Disallow: /cricketnext/videos/series/*/page-*/news18.com/*/page-*/ Disallow: /cricketnext/videos/series/*/page-*/news18.com/*.html$ Disallow: /amp/*/page-*/$ Disallow: /*/page-*/$ Disallow: /cricketnext/cricket-live-scorecard/* Disallow: /amp/topics/*/page-* Disallow: /topics/*/page-* Disallow: *.html\\\"$ Disallow: */page-*/news18.com/* Disallow: */photogallery/*/news18.com/* Disallow: /amp/lok-sabha-elections-2019/* Disallow: /news/cricketnext/* Disallow: /amp/topics/*/news/$ Disallow: /amp/topics/*/videos/$ Disallow: /amp/topics/*/photos/$ Disallow: /topics/*%25*/$ Disallow: /topics/*%25*/news/$ Disallow: /topics/*%25*/videos/$ Disallow: /topics/*%25*/photos/$ Disallow: /amp/topics/*%25*/$ Disallow: /amp/topics/*%25*/news/$ Disallow: /amp/topics/*%25*/videos/$ Disallow: /amp/topics/*%25*/photos/$ Disallow: /lok-sabha-elections-2019/page-*/$ Disallow: /amp/assembly-elections-2022*/$ Disallow: /assembly-elections-2022*/$ Disallow: /amp/assembly-elections-march-2022*/$ Disallow: /assembly-elections-march-2022*/$ Disallow: /election_framework_next/* Disallow: /cricketnext/amp/cricket-live-scorecard/* Disallow: /cricketnext/series/icc-cricket-world-cup-2023.html Disallow: /cricketnext/amp/series/icc-cricket-world-cup-2023.html Disallow: /cricketnext/series/icc-cricket-world-cup-2023-warm-up.html Disallow: /cricketnext/amp/series/icc-cricket-world-cup-2023-warm-up.html Disallow: /studio18/* Disallow: /amp/studio18/* Disallow: /tech-reviews/* Disallow: /auto-reviews/* Disallow: /amp/tech-reviews/* Disallow: /amp/auto-reviews/* Disallow: /astrology/horoscope/todays* Disallow: /amp/astrology/horoscope/todays* Disallow: /business/cryptocurrency/trade-bitcoin/* Disallow: /sustainable-is-attainable-news18-and-tata-power-initiative/* Disallow: /givewingstoyoursavings/* Disallow: /netrasuraksha/* Disallow: /missionswachhtapaani/* Disallow: /itsnotok/* Disallow: /swabhimanbharat/* Disallow: /periodofpride/* Disallow: /electralhydrationforhealth/* Disallow: /younggenius/* Disallow: /fit-india-hit-india/* Disallow: /town-hall/* Disallow: /solvefortomorrow/* Disallow: /rising-india-summit/* Disallow: /sustainable-is-attainable-fest/* Disallow: /sustainable-is-attainable-news18-and-tata-power-initiative-2021/* Disallow: /reel-awards-2023/* Disallow: /makeadent/* Disallow: /rising-india-she-shakti/* Disallow: /lok-sabha-elections-2019/* Disallow: /monitoring/?o=4506733701693440&p=4506733712244736 Disallow: /amp/astrology/horoscope/*/ Disallow: /astrology/horoscope/*/ Disallow: /cricket/profile/*/batting-*.html Disallow: /cricket/profile/*/bowling-*.html Disallow: /cricket/IPL%202024/ Disallow: /amp/delhi-assembly-elections-2020/ Disallow: /delhi-assembly-elections-2020/ Disallow: /pro-kabaddi-2019/ Disallow: /amp/topics/*/latest-photogallery/ Disallow: /topics/*/latest-photogallery/ Disallow: /amp/assembly-elections/ Disallow: /assembly-elections/ Disallow: /short-news/*a Disallow: /podcast/*.html User-agent: CCBot Disallow: / User-agent: GPTBot Disallow: / User-Agent: omgili Disallow: / User-Agent: omgilibot Disallow: / User-agent: MAZBot Disallow: / User-agent: ChatGPT-User Disallow: / User-agent: Baiduspider Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: Google-Extended Disallow: / User-agent: Bytespider Disallow: / User-agent: ClaudeBot Disallow: / User-agent: ImagesiftBot Disallow: / User-agent: Diffbot Disallow: / User-agent: Claude-Web Disallow: / User-agent: cohere-ai Disallow: / User-agent: FriendlyCrawler Disallow: / User-agent: img2dataset Disallow: / User-agent: Scrapy Disallow: / User-agent: Timpibot Disallow: / User-agent: VelenPublicWebCrawler Disallow: /