Free tools

Robots.txt Examples

Robots.txt file content for last.fm.

Robot.txt file for: last.fm

      User-Agent: *


Disallow: /music/+noredirect/
Disallow: /de/music/+noredirect/
Disallow: /es/music/+noredirect/
Disallow: /fr/music/+noredirect/
Disallow: /it/music/+noredirect/
Disallow: /ja/music/+noredirect/
Disallow: /pl/music/+noredirect/
Disallow: /pt/music/+noredirect/
Disallow: /ru/music/+noredirect/
Disallow: /sv/music/+noredirect/
Disallow: /tr/music/+noredirect/
Disallow: /zh/music/+noredirect/

Disallow: /user/*/library/music/
Disallow: /de/user/*/library/music/
Disallow: /es/user/*/library/music/
Disallow: /fr/user/*/library/music/
Disallow: /it/user/*/library/music/
Disallow: /ja/user/*/library/music/
Disallow: /pl/user/*/library/music/
Disallow: /pt/user/*/library/music/
Disallow: /ru/user/*/library/music/
Disallow: /sv/user/*/library/music/
Disallow: /tr/user/*/library/music/
Disallow: /zh/user/*/library/music/

# Stop traversal of user libraries
Disallow: /user/*/library*
Disallow: /de/user/*/library*
Disallow: /es/user/*/library*
Disallow: /fr/user/*/library*
Disallow: /it/user/*/library*
Disallow: /ja/user/*/library*
Disallow: /pl/user/*/library*
Disallow: /pt/user/*/library*
Disallow: /ru/user/*/library*
Disallow: /sv/user/*/library*
Disallow: /tr/user/*/library*
Disallow: /zh/user/*/library*

Disallow: /*/+wiki/diff

Disallow: /search
Disallow: /de/search
Disallow: /es/search
Disallow: /fr/search
Disallow: /it/search
Disallow: /ja/search
Disallow: /pl/search
Disallow: /pt/search
Disallow: /ru/search
Disallow: /sv/search
Disallow: /tr/search
Disallow: /zh/search

# Old pages
Disallow: /forum*
Disallow: /de/forum*
Disallow: /es/forum*
Disallow: /fr/forum*
Disallow: /it/forum*
Disallow: /ja/forum*
Disallow: /pl/forum*
Disallow: /pt/forum*
Disallow: /ru/forum*
Disallow: /sv/forum*
Disallow: /tr/forum*
Disallow: /zh/forum*

Disallow: /label*
Disallow: /de/label*
Disallow: /es/label*
Disallow: /fr/label*
Disallow: /it/label*
Disallow: /ja/label*
Disallow: /pl/label*
Disallow: /pt/label*
Disallow: /ru/label*
Disallow: /sv/label*
Disallow: /tr/label*
Disallow: /zh/label*

Disallow: /group*
Disallow: /de/group*
Disallow: /es/group*
Disallow: /fr/group*
Disallow: /it/group*
Disallow: /ja/group*
Disallow: /pl/group*
Disallow: /pt/group*
Disallow: /ru/group*
Disallow: /sv/group*
Disallow: /tr/group*
Disallow: /zh/group*

Disallow: /user/*/journal*
Disallow: /de/user/*/journal*
Disallow: /es/user/*/journal*
Disallow: /fr/user/*/journal*
Disallow: /it/user/*/journal*
Disallow: /ja/user/*/journal*
Disallow: /pl/user/*/journal*
Disallow: /pt/user/*/journal*
Disallow: /ru/user/*/journal*
Disallow: /sv/user/*/journal*
Disallow: /tr/user/*/journal*
Disallow: /zh/user/*/journal*

# Shouts
Disallow: /event/*/shoutbox?
Disallow: /de/event/*/shoutbox?
Disallow: /es/event/*/shoutbox?
Disallow: /fr/event/*/shoutbox?
Disallow: /it/event/*/shoutbox?
Disallow: /ja/event/*/shoutbox?
Disallow: /pl/event/*/shoutbox?
Disallow: /pt/event/*/shoutbox?
Disallow: /ru/event/*/shoutbox?
Disallow: /sv/event/*/shoutbox?
Disallow: /tr/event/*/shoutbox?
Disallow: /zh/event/*/shoutbox?

Disallow: /event/*/shoutbox/*
Disallow: /de/event/*/shoutbox/*
Disallow: /es/event/*/shoutbox/*
Disallow: /fr/event/*/shoutbox/*
Disallow: /it/event/*/shoutbox/*
Disallow: /ja/event/*/shoutbox/*
Disallow: /pl/event/*/shoutbox/*
Disallow: /pt/event/*/shoutbox/*
Disallow: /ru/event/*/shoutbox/*
Disallow: /sv/event/*/shoutbox/*
Disallow: /tr/event/*/shoutbox/*
Disallow: /zh/event/*/shoutbox/*

Disallow: /festival/*/shoutbox?
Disallow: /de/festival/*/shoutbox?
Disallow: /es/festival/*/shoutbox?
Disallow: /fr/festival/*/shoutbox?
Disallow: /it/festival/*/shoutbox?
Disallow: /ja/festival/*/shoutbox?
Disallow: /pl/festival/*/shoutbox?
Disallow: /pt/festival/*/shoutbox?
Disallow: /ru/festival/*/shoutbox?
Disallow: /sv/festival/*/shoutbox?
Disallow: /tr/festival/*/shoutbox?
Disallow: /zh/festival/*/shoutbox?

Disallow: /festival/*/shoutbox/*
Disallow: /de/festival/*/shoutbox/*
Disallow: /es/festival/*/shoutbox/*
Disallow: /fr/festival/*/shoutbox/*
Disallow: /it/festival/*/shoutbox/*
Disallow: /ja/festival/*/shoutbox/*
Disallow: /pl/festival/*/shoutbox/*
Disallow: /pt/festival/*/shoutbox/*
Disallow: /ru/festival/*/shoutbox/*
Disallow: /sv/festival/*/shoutbox/*
Disallow: /tr/festival/*/shoutbox/*
Disallow: /zh/festival/*/shoutbox/*

Disallow: /tag/*/shoutbox?
Disallow: /de/tag/*/shoutbox?
Disallow: /es/tag/*/shoutbox?
Disallow: /fr/tag/*/shoutbox?
Disallow: /it/tag/*/shoutbox?
Disallow: /ja/tag/*/shoutbox?
Disallow: /pl/tag/*/shoutbox?
Disallow: /pt/tag/*/shoutbox?
Disallow: /ru/tag/*/shoutbox?
Disallow: /sv/tag/*/shoutbox?
Disallow: /tr/tag/*/shoutbox?
Disallow: /zh/tag/*/shoutbox?

Disallow: /tag/*/shoutbox/*
Disallow: /de/tag/*/shoutbox/*
Disallow: /es/tag/*/shoutbox/*
Disallow: /fr/tag/*/shoutbox/*
Disallow: /it/tag/*/shoutbox/*
Disallow: /ja/tag/*/shoutbox/*
Disallow: /pl/tag/*/shoutbox/*
Disallow: /pt/tag/*/shoutbox/*
Disallow: /ru/tag/*/shoutbox/*
Disallow: /sv/tag/*/shoutbox/*
Disallow: /tr/tag/*/shoutbox/*
Disallow: /zh/tag/*/shoutbox/*

Disallow: /user/*/shoutbox?
Disallow: /de/user/*/shoutbox?
Disallow: /es/user/*/shoutbox?
Disallow: /fr/user/*/shoutbox?
Disallow: /it/user/*/shoutbox?
Disallow: /ja/user/*/shoutbox?
Disallow: /pl/user/*/shoutbox?
Disallow: /pt/user/*/shoutbox?
Disallow: /ru/user/*/shoutbox?
Disallow: /sv/user/*/shoutbox?
Disallow: /tr/user/*/shoutbox?
Disallow: /zh/user/*/shoutbox?

Disallow: /user/*/shoutbox/*
Disallow: /de/user/*/shoutbox/*
Disallow: /es/user/*/shoutbox/*
Disallow: /fr/user/*/shoutbox/*
Disallow: /it/user/*/shoutbox/*
Disallow: /ja/user/*/shoutbox/*
Disallow: /pl/user/*/shoutbox/*
Disallow: /pt/user/*/shoutbox/*
Disallow: /ru/user/*/shoutbox/*
Disallow: /sv/user/*/shoutbox/*
Disallow: /tr/user/*/shoutbox/*
Disallow: /zh/user/*/shoutbox/*

Disallow: /music/*/+shoutbox?
Disallow: /de/music/*/+shoutbox?
Disallow: /es/music/*/+shoutbox?
Disallow: /fr/music/*/+shoutbox?
Disallow: /it/music/*/+shoutbox?
Disallow: /ja/music/*/+shoutbox?
Disallow: /pl/music/*/+shoutbox?
Disallow: /pt/music/*/+shoutbox?
Disallow: /ru/music/*/+shoutbox?
Disallow: /sv/music/*/+shoutbox?
Disallow: /tr/music/*/+shoutbox?
Disallow: /zh/music/*/+shoutbox?

Disallow: /music/*/+shoutbox/*
Disallow: /de/music/*/+shoutbox/*
Disallow: /es/music/*/+shoutbox/*
Disallow: /fr/music/*/+shoutbox/*
Disallow: /it/music/*/+shoutbox/*
Disallow: /ja/music/*/+shoutbox/*
Disallow: /pl/music/*/+shoutbox/*
Disallow: /pt/music/*/+shoutbox/*
Disallow: /ru/music/*/+shoutbox/*
Disallow: /sv/music/*/+shoutbox/*
Disallow: /tr/music/*/+shoutbox/*
Disallow: /zh/music/*/+shoutbox/*

# N.B: these are not covered by the above /music/ rule
# (shoutbox vs +shoutbox)
Disallow: /music/*/+images/*/shoutbox?
Disallow: /de/music/*/+images/*/shoutbox?
Disallow: /es/music/*/+images/*/shoutbox?
Disallow: /fr/music/*/+images/*/shoutbox?
Disallow: /it/music/*/+images/*/shoutbox?
Disallow: /ja/music/*/+images/*/shoutbox?
Disallow: /pl/music/*/+images/*/shoutbox?
Disallow: /pt/music/*/+images/*/shoutbox?
Disallow: /ru/music/*/+images/*/shoutbox?
Disallow: /sv/music/*/+images/*/shoutbox?
Disallow: /tr/music/*/+images/*/shoutbox?
Disallow: /zh/music/*/+images/*/shoutbox?

Disallow: /music/*/+images/*/shoutbox/*
Disallow: /de/music/*/+images/*/shoutbox/*
Disallow: /es/music/*/+images/*/shoutbox/*
Disallow: /fr/music/*/+images/*/shoutbox/*
Disallow: /it/music/*/+images/*/shoutbox/*
Disallow: /ja/music/*/+images/*/shoutbox/*
Disallow: /pl/music/*/+images/*/shoutbox/*
Disallow: /pt/music/*/+images/*/shoutbox/*
Disallow: /ru/music/*/+images/*/shoutbox/*
Disallow: /sv/music/*/+images/*/shoutbox/*
Disallow: /tr/music/*/+images/*/shoutbox/*
Disallow: /zh/music/*/+images/*/shoutbox/*

Disallow: /music/*/+wiki/shoutbox?
Disallow: /de/music/*/+wiki/shoutbox?
Disallow: /es/music/*/+wiki/shoutbox?
Disallow: /fr/music/*/+wiki/shoutbox?
Disallow: /it/music/*/+wiki/shoutbox?
Disallow: /ja/music/*/+wiki/shoutbox?
Disallow: /pl/music/*/+wiki/shoutbox?
Disallow: /pt/music/*/+wiki/shoutbox?
Disallow: /ru/music/*/+wiki/shoutbox?
Disallow: /sv/music/*/+wiki/shoutbox?
Disallow: /tr/music/*/+wiki/shoutbox?
Disallow: /zh/music/*/+wiki/shoutbox?

Disallow: /music/*/+wiki/shoutbox/*
Disallow: /de/music/*/+wiki/shoutbox/*
Disallow: /es/music/*/+wiki/shoutbox/*
Disallow: /fr/music/*/+wiki/shoutbox/*
Disallow: /it/music/*/+wiki/shoutbox/*
Disallow: /ja/music/*/+wiki/shoutbox/*
Disallow: /pl/music/*/+wiki/shoutbox/*
Disallow: /pt/music/*/+wiki/shoutbox/*
Disallow: /ru/music/*/+wiki/shoutbox/*
Disallow: /sv/music/*/+wiki/shoutbox/*
Disallow: /tr/music/*/+wiki/shoutbox/*
Disallow: /zh/music/*/+wiki/shoutbox/*

Disallow: /music/*/+images/*
Disallow: /de/music/*/+images/*
Disallow: /es/music/*/+images/*
Disallow: /fr/music/*/+images/*
Disallow: /it/music/*/+images/*
Disallow: /ja/music/*/+images/*
Disallow: /pl/music/*/+images/*
Disallow: /pt/music/*/+images/*
Disallow: /ru/music/*/+images/*
Disallow: /sv/music/*/+images/*
Disallow: /tr/music/*/+images/*
Disallow: /zh/music/*/+images/*

# AJAX content
Disallow: /search/autocomplete
Disallow: /de/search/autocomplete
Disallow: /es/search/autocomplete
Disallow: /fr/search/autocomplete
Disallow: /it/search/autocomplete
Disallow: /ja/search/autocomplete
Disallow: /pl/search/autocomplete
Disallow: /pt/search/autocomplete
Disallow: /ru/search/autocomplete
Disallow: /sv/search/autocomplete
Disallow: /tr/search/autocomplete
Disallow: /zh/search/autocomplete

Disallow: /player
Disallow: /de/player
Disallow: /es/player
Disallow: /fr/player
Disallow: /it/player
Disallow: /ja/player
Disallow: /pl/player
Disallow: /pt/player
Disallow: /ru/player
Disallow: /sv/player
Disallow: /tr/player
Disallow: /zh/player

Disallow: /features/partial/page-bottom
Disallow: /de/features/partial/page-bottom
Disallow: /es/features/partial/page-bottom
Disallow: /fr/features/partial/page-bottom
Disallow: /it/features/partial/page-bottom
Disallow: /ja/features/partial/page-bottom
Disallow: /pl/features/partial/page-bottom
Disallow: /pt/features/partial/page-bottom
Disallow: /ru/features/partial/page-bottom
Disallow: /sv/features/partial/page-bottom
Disallow: /tr/features/partial/page-bottom
Disallow: /zh/features/partial/page-bottom

Disallow: /features/partial/footer
Disallow: /de/features/partial/footer
Disallow: /es/features/partial/footer
Disallow: /fr/features/partial/footer
Disallow: /it/features/partial/footer
Disallow: /ja/features/partial/footer
Disallow: /pl/features/partial/footer
Disallow: /pt/features/partial/footer
Disallow: /ru/features/partial/footer
Disallow: /sv/features/partial/footer
Disallow: /tr/features/partial/footer
Disallow: /zh/features/partial/footer


Disallow: /harming/humans
Disallow: /ignoring/human/orders
Disallow: /harm/to/self

Allow: /

Sitemap: https://www.last.fm/sitemap-index-secure.xml

User-Agent: Bingbot
Crawl-Delay: 1