Konigle

November 26, 2024

Robots.txt for rateyourmusic.com

        # Sonemic, Inc. prohibits any kind of automated means, (e.g. crawling, scraping, etc) of access to the service without express permission. Please see 
# https://rateyourmusic.com/tos

User-agent: Mediapartners-Google*
Disallow:

User-agent: Googlebot-Image
Disallow: /account
Disallow: /redir/
Disallow: /admin/
Disallow: /artist/profile_ac
Disallow: /compare
Disallow: /lists/new_item
Disallow: /favorited
Disallow: /find_similar_users
Disallow: /releases/ac
Disallow: /user_albums_export/
Disallow: /contributed/
Disallow: /stats/
Disallow: /messages/
Disallow: /recommend/
Disallow: /recs/
Disallow: /rdescriptor/set
Disallow: /rgenre/set
Disallow: /contact
Disallow: /search
Disallow: /artist/profile_details
Disallow: /images/upload
Disallow: /images/all
Disallow: /rymzilla

User-agent: googlebot
Disallow: /account
Disallow: /redir/
Disallow: /admin/
Disallow: /artist/profile_ac
Disallow: /compare
Disallow: /lists/new_item
Disallow: /favorited
Disallow: /find_similar_users
Disallow: /releases/ac
Disallow: /user_albums_export/
Disallow: /contributed/
Disallow: /stats/
Disallow: /messages/
Disallow: /recommend/
Disallow: /recs/
Disallow: /rdescriptor/set
Disallow: /rgenre/set
Disallow: /contact
Disallow: /search
Disallow: /artist/profile_details
Disallow: /images/upload
Disallow: /images/all
Disallow: /rymzilla

User-agent: Twitterbot
Disallow: /account
Disallow: /redir/
Disallow: /admin/
Disallow: /artist/profile_ac
Disallow: /compare
Disallow: /lists/new_item
Disallow: /favorited
Disallow: /find_similar_users
Disallow: /releases/ac
Disallow: /user_albums_export/
Disallow: /contributed/
Disallow: /stats/
Disallow: /messages/
Disallow: /recommend/
Disallow: /recs/
Disallow: /rdescriptor/set
Disallow: /rgenre/set
Disallow: /contact
Disallow: /search
Disallow: /artist/profile_details
Disallow: /images/upload
Disallow: /images/all
Disallow: /rymzilla

User-agent: msnbot
Crawl-delay: 5
Disallow: /account
Disallow: /redir/
Disallow: /admin/
Disallow: /artist/profile_ac
Disallow: /compare
Disallow: /lists/new_item
Disallow: /favorited
Disallow: /find_similar_users
Disallow: /releases/ac
Disallow: /user_albums_export/
Disallow: /contributed/
Disallow: /stats/
Disallow: /messages/
Disallow: /recommend/
Disallow: /recs/
Disallow: /rdescriptor/set
Disallow: /rgenre/set
Disallow: /contact
Disallow: /search
Disallow: /artist/profile_details
Disallow: /images/upload
Disallow: /images/all
Disallow: /rymzilla

User-agent: Slurp
Crawl-delay: 12
Disallow: /account
Disallow: /redir/
Disallow: /admin/
Disallow: /artist/profile_ac
Disallow: /compare
Disallow: /lists/new_item
Disallow: /favorited
Disallow: /find_similar_users
Disallow: /releases/ac
Disallow: /user_albums_export/
Disallow: /contributed/
Disallow: /stats/
Disallow: /messages/
Disallow: /recommend/
Disallow: /recs/
Disallow: /rdescriptor/set
Disallow: /rgenre/set
Disallow: /contact
Disallow: /search
Disallow: /artist/profile_details
Disallow: /images/upload
Disallow: /images/all
Disallow: /rymzilla

User-agent: Amazonbot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: AwarioRssBot
User-agent: AwarioSmartBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: FirecrawlAgent
Disallow: /

User-agent: FriendlyCrawler
Disallow: /

User-agent: Google-CloudVertexBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: Meta-ExternalAgent
User-agent: meta-externalagent
Disallow: /

User-agent: NewsNow
Disallow: /

User-agent: news-please
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: peer39_crawler
User-agent: peer39_crawler/1.0
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Quora-Bot
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: TurnitinBot
Disallow: /


User-agent: *
Disallow: /

Keep Reading