Free tools

Robots.txt Examples

Robots.txt file content for cbc.ca.

Robot.txt file for: cbc.ca

      # robots, scram
User-agent: *
Disallow:   /*index.html$
Disallow:   /sports/includes/
Disallow:   /sports/hockey/includes/
Disallow:   /sports/baseball/includes/
Disallow:   /sports/basketball/includes/
Disallow:   /sports/cfl/includes/
Disallow:   /sports/nfl/includes/
Disallow:   /sports/whatson/
Disallow:   /sports/cfl/whatson/
Disallow:   /sports/cfl/cfl-on-cbc/includes/
Disallow:   /sports/hockey/whatson/
Disallow:   /sports/hockey/fantasypool/
Disallow:   /sports/hockey/hnic/includes/
Disallow:   /sports/hockey/hnic/bios/
Disallow:   /cp/
Disallow:   /m/rich/
Disallow:   /search/
Disallow:   /photogallery/*.xml$
Disallow:   /cgi-bin/
Disallow:   /money/stockChart.jsp
Disallow:   /money/stockQuoteChart.jsp
Disallow:   /money/stockQuote.jsp
Disallow:   /index.new.html
Disallow:   /podcasting/includes/tsoas.xml
Disallow:   /podcasting/includes/theheart.xml
Disallow:   /podcasting/includes/bloodlines.xml
Disallow:   /podcasting/includes/resistance.xml
Disallow:   /podcasting/includes/levillage.xml
Disallow:   /podcasting/includes/histoirecrochie.xml
Disallow:   /podcasting/includes/brainwash.xml
Disallow:   /podcasting/includes/hollywoodexiles.xml
Disallow:   /podcasting/includes/avril.xml
Disallow:   /podcasting/includes/goldscam.xml

User-agent: Bingbot
Allow: /doncherry/

User-agent: Googlebot-News
Disallow: /radio/thisisthat/
Disallow: /comedy/
Disallow: /radio/becausenews/
Disallow: /radio/podcasts/comedy/
Disallow: /radio/thedebaters/
Disallow: /radio/irrelevantshow/
Disallow: /radio/laughoutloud/
Disallow: /kimsconvenience/
Disallow: /22minutes/
Disallow: /workinmoms/
Disallow: /schittscreek/
Disallow: /mrd/
Disallow: /baroness/
Disallow: /fourinthemorning/
Disallow: /mercerreport/

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: Google-Extended
Disallow: / 

User-agent: CCBot
Disallow: /

User-agent: Anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /