Robots.txt for pbs.org

        User-agent: *
Crawl-delay: 1
Disallow: /wgbh/pages/frontline/digitalnation/%3F
Disallow: /wgbh/pages/frontline/digitalnation/%3F*
Disallow: /wgbh/frontline/*elqat*
Disallow: /wgbh/pages/frontline/*elqat*
Disallow: /wgbh/pages/frontline/bushswar*
Disallow: /independentlens/getinvolved/cinema/
Disallow: /cgi-bin/*
Disallow: /cgi-registry/*
Disallow: /survey/
Disallow: /whatson/stations/
Disallow: /whatson/schedulex/
Disallow: /whatson/pbsyou/
Disallow: /whatson/dbs/
Disallow: /remotecontrol/bestpractices/prototype/
Disallow: /remotecontrol/bestpractices/jl-prototype/
Disallow: /pressroom/
Disallow: /als/shakespeare_pilot/
#Disallow: /.
Disallow: /newshour/bb/military/military.html/
Disallow: /producers/webmanual/
Disallow: /devcon2007/
Disallow: /smcot/
Disallow: /pbsholidaycard/
Disallow: /gpra/
Disallow: /tlqa/
Disallow: /remote.control/
Disallow: /teachers/connect/
Disallow: /engage/sites/all/files/blog_images/monkeys.gif
Disallow: /teachers/edcar/
Disallow: /teachers/dll_research/
Disallow: /peerconnection/feeds/
Disallow: /teachers/innovators/toolkit/
Disallow: /mt4/
Disallow: /klru/mt4/
Disallow: /newshour/mt4/
Disallow: /mediashift/mt4/
Disallow: /kcet/mt4/
Disallow: /newshour/mt3/
Disallow: /now/mt3/
Disallow: /itvs/mt3/
Disallow: /pbs/mt3/
Disallow: /pbs/mt3o4/
Disallow: /pbs/mt4/
Disallow: /pbs/teachers/mt3/
Disallow: /pbs/kids/mt3/
Disallow: /pbs/kids/mt4/
Disallow: /pbs/src/mt4/
Disallow: /pbs/parents/mt4/
Disallow: /pbsi/mt3/
Disallow: /moyers/mt3/
Disallow: /moyers/mt4/
Disallow: /nbr/mt3/
Disallow: /nbr/mt4/
Disallow: /cringely/mt3/
Disallow: /cringely/mt4/
Disallow: /wgbh/mt4/
Disallow: /wnet/expose/mt4/
Disallow: /wnet/mt4/
Disallow: /opb/mt4/
Disallow: /search/search_results.html?q=*
Disallow: /wgbh/nova/search/
Disallow: /wgbh/nova/search/results/*
Disallow: /wgbh/nova/beta/search/results/*
Disallow: /nbr/info/search.html?*
Disallow: /independentlens/video/index.php?*
Disallow: /independentlens/video/?filter=*
Disallow: /wnet/religionandethics/*/index.html?*
Disallow: /wnet/religionandethics/search_results.html*
Disallow: /teachers/search/resources/?q=* 
Disallow: /teacherline/catalog/browse/ 
Disallow: /thewar/search_results.php?* 
Disallow: */emailthispage.html?* 
Disallow: /pov_mt/mt4/ 
Disallow: /newshour/indepth_coverage/middle_east/iraq/honorroll/search.php?*
Disallow: /includes/ 
Disallow: */emailthispage.html 
Disallow: /about/faq/station-finder/?state-text=* 
Disallow: /about/faq/station-finder/?zip-text=* 
Disallow: /wgbh/nova/search-results.html 
Disallow: /wgbh/americanexperience/about/search/search_results.html 
Disallow: /moyers/journal/about/search_google.html
Disallow: /foundation/contact/
Disallow: *?replytocom
Disallow: /social_media/
Disallow: /independentlens/film-lists/watch-online/
Disallow: /show/koth-show/
Disallow: /my-list-videos/
Disallow: /my-list-shows/
Disallow: /my-list-viewing-history/
Disallow: /parents/activity-finder/
Disallow: /parents/search/
Sitemap: https://www.pbs.org/sitemap.xml
Sitemap: http://www.pbs.org/mercy-street/sitemap.xml
Sitemap: https://www.pbs.org/newshour/sitemaps/sitemap.xml
Sitemap: https://www.pbs.org/weta/washingtonweek/sitemap.xml

User-Agent: Baiduspider 
Disallow: /independentlens/getinvolved/cinema/
Disallow: /independentlens/films/watch-online/*

User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /