Free tools

Robots.txt Examples

Robots.txt file content for kakaku.com.

Robot.txt file for: kakaku.com

      User-Agent: * 
Allow: /auth/profile/
Disallow: /ksearch/
Disallow: /auth/
Disallow: /prdsearch/pricehistory_data/
Disallow: /kakakuad/
Disallow: /history/
Disallow: /jump/
Disallow: /ad/
Disallow: /ria/
Disallow: /item/*/localshops/
Disallow: /localshops/shop/*/print/
Disallow: /localshops/shop/*/shopitem/
Disallow: /kuruma/hoken/
Disallow: /kuchikomi/review/history/
Disallow: /shopreg/
Disallow: /localshopreg/
Disallow: /mapsearch/Default.aspx
Disallow: /okurimono/script/
Disallow: /script/kakakuad/
Disallow: /modules/
Disallow: /pt/
Disallow: /ref/rd_kused.asp
Disallow: /card/compare/compare.asp*
Disallow: /kuchikomi/review/results/
Disallow: /smartphonerd/
Disallow: /altitem/
Disallow: /error/
Disallow: /se.asp
Disallow: /prdnews/newssearch/
Disallow: /err/
Disallow: /*/jump/

User-agent: Mediapartners-Google
Allow: /kuchikomi/review/history/

User-agent: bingbot
Allow: /auth/profile/
Disallow: /ksearch/
Disallow: /auth/
Disallow: /prdsearch/pricehistory_data/
Disallow: /kakakuad/
Disallow: /history/
Disallow: /jump/
Disallow: /ad/
Disallow: /ria/
Disallow: /item/*/localshops/
Disallow: /localshops/shop/*/print/
Disallow: /localshops/shop/*/shopitem/
Disallow: /kuruma/hoken/
Disallow: /kuchikomi/review/history/
Disallow: /shopreg/
Disallow: /localshopreg/
Disallow: /mapsearch/Default.aspx
Disallow: /okurimono/script/
Disallow: /script/kakakuad/
Disallow: /modules/
Disallow: /pt/
Disallow: /ref/rd_kused.asp
Disallow: /card/compare/compare.asp*
Disallow: /kuchikomi/review/results/
Disallow: /smartphonerd/
Disallow: /altitem/
Disallow: /error/
Disallow: /se.asp
Disallow: /prdnews/newssearch/
Disallow: /err/
Disallow: /used/*?*
Disallow: /*/jump/

User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /