Robots.txt for abc.net.au

        # robots.txt for https://www.abc.net.au/ -- ABC Online
User-agent: *
Disallow: /classic/contact/concerts.htm
Disallow: /classic/contact/default.htm
Disallow: /classic/contact/eventsdiary.htm
Disallow: /classic/contact/formerror.htm
Disallow: /classic/contact/formthanks.htm
Disallow: /classic/contact/general.htm
Disallow: /classic/contact/limelight.htm
Disallow: /classic/contact/mailinglist.htm
Disallow: /classic/contact/music.htm
Disallow: /classic/contact/presenter.htm
Disallow: /classic/contact/website.htm
Disallow: /classic/contact/word.htm
Disallow: /xmlcontent/
Disallow: /classicfm/

#OPSSD-340 2015/5/5
Disallow: /iview/



#INNG-46: 2014-12-30
Disallow: /site-archive/

# Added for corporate communications, as they have migrated to a new site
Disallow: /corp/
Disallow: /contact/

# Added for Homepage Beta, prevent indexing during public beta
Disallow: /homepage/2013/
Disallow: /beta/

# Added for WCMS Tennent testing, not a public 
Disallow: /abc4000/

Disallow: /res/

User-agent: Googlebot
Crawl-delay: 5

User-agent: Googlebot-Image
Crawl-delay: 5

User-agent: MSNBot
Crawl-delay: 5

User-agent: Slurp
Crawl-delay: 5

########################################

User-agent: FlipboardProxy
Crawl-delay: 2
Disallow: /news/image/

User-agent: iSec_Bot
Disallow: /

User-agent: TurnitinBot
Disallow: / 

User-agent: ICC-Crawler
Disallow: /

User-agent: trendkite-akashic-crawler
Disallow: /

User-agent: TinEye-bot
Disallow: /

User-agent: R6_CommentReader
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Nutch
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

# sitemaps
Sitemap: https://www.abc.net.au/sitemaps/sitemap-index.xml.gz