Free tools

Robots.txt Examples

Robots.txt file content for euronews.com.

Robot.txt file for: euronews.com

      # www.robotstxt.org/
# www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449
User-agent: *
Disallow: /api/
Disallow: /2016/09/22/thailand-s-king-bhumibol-the-peoples-monarch-has-died/
Disallow: /2016/09/22/thailand-s-king-bhumibol-the-peoples-monarch-has-died
Disallow: /media/Player6_0.swf*
Disallow: /js/main-*.js
Disallow: /js/streamsense-*.min.js
Disallow: /modules
Disallow: /plugin
Disallow: /src
Disallow: /handler
Disallow: /utils
Disallow: /lib
Disallow: /helper
Disallow: /update_
Disallow: /instances
Disallow: /dom
Disallow: /class
Disallow: /client
Disallow: /_partners/
Disallow: /embed/
Disallow: /login
Disallow: /register

# weather
Disallow: /meteo/*p=
Disallow: /weather/*p=
Disallow: /wetter/*p=
Disallow: /tempo/*p=
Disallow: /tiempo/*p=
Disallow: /meteorologia/*p=
Disallow: /hava-durumu/*p=
Disallow: /idojaras/*p=

# ai bots
User-agent: amazonbot
Disallow: /
User-agent: anthropic-ai
Disallow: /
User-agent: applebot
Disallow: /
User-agent: bingsapphire
Disallow: /
User-agent: copilotsapphire
Disallow: /
User-agent: bytespider
Disallow: /
User-agent: ccbot
Disallow: /
User-agent: chatglm-spider
Disallow: /
User-agent: chatgpt-user
Disallow: /
User-agent: claudebot
Disallow: /
User-agent: diffbot
Disallow: /
User-agent: facebookbot
Disallow: /
User-agent: friendlycrawler
Disallow: /
User-agent: gptbot
Disallow: /
User-agent: ia_archiver
Disallow: /
User-agent: img2dataset
Disallow: /
User-agent: imagesiftbot
Disallow: /
User-agent: magpie-crawler
Disallow: /
User-agent: meta-externalagent
Disallow: /
User-agent: omgili
Disallow: /
User-agent: openai
Disallow: /
User-agent: spawning-ai
Disallow: /
User-agent: the knowledge ai
Disallow: /
User-agent: timpibot
Disallow: /
User-agent: velenpublicwebcrawler
Disallow: /
User-agent: webzio
Disallow: /
User-agent: youbot
Disallow: /

Sitemap: https://www.euronews.com/sitemaps/en/sitemap.xml
Sitemap: https://www.euronews.com/sitemaps/en/articles.xml
Sitemap: https://www.euronews.com/sitemaps/en/videos.xml
Sitemap: https://www.euronews.com/sitemaps/en/images.xml
Sitemap: https://www.euronews.com/sitemaps/en/newswires.xml