Robot.txt file for: welt.de

      # The Facebook Crawler
User-agent: Facebot
Allow: /

User-agent: sogou spider
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: Flamingo_SearchEngine 
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /


User-agent: *
Disallow: /channels-extern/
Disallow: /sportdaten/
Disallow: /testgpr/
Disallow: /boerse/data/
Disallow: /partner/
Disallow: /reisetipps/
Disallow: /z/
Disallow: /appl/
Disallow: /woa/
Disallow: /am-sonntag/vorproduktion/
Disallow: /audiofiles/
Disallow: /out-of-home/
Disallow: /immobilien/expose
Disallow: /suche
Disallow: /onward/
Disallow: /api/
Disallow: /*?config
Disallow: /*?config=newsmli_bloomberg2
Disallow: /*.xmli
Disallow: /*?service=ajax
Disallow: /*?service=Ajax
Disallow: /*?ajax
Disallow: /*?ajax&wid
Disallow: /*?config=print
Disallow: /*?config=articleidfromurl
Disallow: /*?config=endscreen
Disallow: /*?config=iframewelt
Disallow: /*?config=langeslestueck
Disallow: /*?config=latest_videos
Disallow: /*?config=menu_home
Disallow: /*?config=mostviewed_videos
Disallow: /*?config=recommended_videos
Disallow: /*?config=regioarticlemarginal
Disallow: /*?config=zoom
Disallow: /*?config=zoomopener
Disallow: /*?noredirect=true&config=standalone
Disallow: /*?config=standalone
Disallow: /*?wtmc=XING
Disallow: /*?config=articleidfromurl
Disallow: /*?print=yes
Disallow: /*?tabPane
Disallow: /video/embeded/
Disallow: /img/*-wWIDTH*.jpg


Sitemap: https://www.welt.de/sitemaps/newssitemap/newssitemap.xml
Sitemap: https://www.welt.de/sitemaps/sitemap/sitemap.xml
Sitemap: https://www.welt.de/sitemaps/videositemap/videositemap.xml