Free tools

Robots.txt Examples

Robots.txt file content for bol.com.

Robot.txt file for: bol.com

      # De robots.txt van ons allemaal
# Served by team 20seo

User-agent: *
Allow: /
# Sitemap
Sitemap: https://www.bol.com/sitemap/nl-nl/
Sitemap: https://www.bol.com/sitemap/nl-be/
Sitemap: https://www.bol.com/sitemap/fr-be/

Disallow: /atg/commerce/order/purchase/
Disallow: /common/error/*
Disallow: /nl/fr/
Disallow: /be/fr/p/-/*
Disallow: /*/account/
Disallow: /*/ajax/jsLogging.html
Disallow: /*/albert-heijn-premium
Disallow: /*/b/*/*+*/
Disallow: /*/b/*/*/*/
Disallow: /*/bltg/*
Disallow: /*/contactformulier*
Disallow: /*/c/*/*/
Allow: /*/c/*/*+8299/$
Allow: /*/c/*/*+3132/$
Allow: /*/c/*/*+3133/$
Allow: /*/b/*/*+8299/$
Allow: /*/b/*/*+3132/$
Allow: /*/b/*/*+3133/$
Disallow: /*/ebook/preview.html*
Disallow: /*/order/
Disallow: /*/lijstjes/*
Disallow: /*/prijsoverzicht/
Disallow: /*/productaanbevelingen/
Disallow: /*/producten-vergelijken/
Disallow: /*/ra/
Disallow: /*/rnwy/account/
Disallow: /*/rnwy/ajax/*
Disallow: /*/rnwy/artikelen/
Disallow: /*/rnwy/product_compare/
Disallow: /*/rnwy/product/reviews/
Disallow: /*/rnwy/slots/ajax/*
Disallow: /*/rnwy/studybooks/
Disallow: /*/s/
Disallow: /*/schrijf-een-review/
Disallow: /*/sdl/
Allow: /*/sdl/shop-de-look
Disallow: /*/select*?*
Disallow: /*/select-cadeau-sparen/?*
Disallow: /*/serie/*/
Disallow: /*/u/
Disallow: /*/w/*/
Disallow: /*/vaak-samen-gekocht/
Disallow: /*/*+*+*+*
Disallow: /*?page=*&promo
Disallow: /*offerType=refurbished*
Disallow: /*/url$
Disallow: /*/@type$

User-Agent: Baiduspider
Disallow: /
User-Agent: omgili
Disallow: /
User-Agent: Slurp
Crawl-delay: 20
User-agent: rogerbot
Disallow: /
User-agent: YandexBot
Disallow: /

#                                    ....::^^.:.                       ....                         
#                             .      .^^~!77^.^^                     .:^::::^:.                      
#                           .7YY?~.  :77~^^::^7~                    ..:!7^..::^^                     
#                          ~YPPGBBG!:^?JJ7!~!7!.                    ..:~7!~^:...                     
#                        :JPPGGBB##P: .!JJ?77^        ..::::..      .^~~~!7?^   .:^!^                
#                       ^YPPGGBB#5~     .^~~^     :~7JJYYYYYYYJ7:    .!JJ??7:.~JPGGGP!               
#                      :Y5PGGGBB?.              .7JJJ?7JYYYY555PP?.   .:^:. .JBBBGGPPG7              
#                     .J55GGGB#Y               .7J?7^:::~!!7?JY55G?          .JBGGGGPPG7             
#                     !5YPGGGBB~               ^?~^:.:?J^~YJ~~!77?J:           ?BGGGGPPG!            
#                     J5YGGGBBG:               :::::.^##~7&B!~!!77?!            YGPGGGPPP^           
#                    .YP5GBBBBB^               ::::..:~~~~!!~~!!7777:           !GPPGGPPG5.          
#                    .YP5GBBBB#J              .::::J7^^^~!!!?5Y!777?~           ^GP5GGGGGG~          
#                     JG5GBBBBBB~             .::::^7JY555555J7777777.          ^GP5GGGGGG7          
#                     !GPPBBBBBBG:            :::::. :~!777!!!777??7?^          7G55GGGGGG!          
#                     :PG5GBBBBBBP~          :^::::.:^~!77777777?????7.        .5GYPGGGGGG~          
#                      7G5YGGGGGGBG?:  .:~!JY~^^^^::^~!77?7?????????7?^        JGP5GGGGGG5.          
#                      .5GYYGGGGGBB#G77YPGGG!^~~?5YJJJJYYYYYYJ??777?JYPY!:   .?GG5PGGGGGG?           
#                       ^PPY5BBBBBBGGPPPPPPP7^:.^B@@@@@@@@@#P7^^^!J5GGGGGPY7~5GGPPGGPPPGG^           
#                        !GG5P#BGPPPGGPPP5PPJ.    5@@@@@#5!:  .^JPPPPPPPPPGBBBGPPGGPPPGGJ            
#                         JBBPP5555555PP555PP?. .?B@@@@@G:   :?5PPPPGGGGGGGGGGGGGGGGGGGG^            
#                         ^GP55555555PP555555P??#@@@@@@@@B~^?5PPPPPPGGPPPPPPPPGGGGGGGGGY.            
#                        ^J5555555555PPPP555555B@@@@@@@@@@G5PPPPPPPPPGPPPPPPPPPPGGGGGGG!             
#                      .7555555555555YYYY5PP5555B@@@@@@@#P5P5PPPPPGGGPPPPPPPPPPPPPGGGGP^             
#                     :J55555555555YYJ??JY5PPPP55B@@@@@BP5PPPGGGGGPPPPPPPPPPPPPPPPPGGGP.             
#                    ^Y55555555555YJ????JJY5PPGPPP#@@@BPPPGGGGPPPPPPPPPPPPPPPPPPPPPGGGG?             
#                   ^Y55555555555YJ??77?JJY555PPPPG@@#PPPGGPPPPPPPPPPPPPPPPPPPPPPPPGGGGG7            
#                  :Y55555555555YYJ?777?JYY55555PPP#@BPPPPPPPPPPPPPPPPPPPPPPPPPPPPGGGGGGP^           
#                 .J555555555555YJ?777??JYY55555555B@GPPPPPPPPPPPPPPPPPPPPPPPPPPPGGGGGGGGJ           
#                 ~5555555555555YJ?777?JYY5555PPPP5G&PPPPPPPPPPPPPPPPPPPPPPPPPPGGGGGGGGGGP^          
#                .J555555555555YY?777??JY5555PPPPPPG&GPPPPPPPPPPPPPPPPPPPPPGGGGGGGGGGGGGGG7          
#                ^5555555555555YJ?777?JYY555PPPPPPPG&GPPPPPPGGGGGGGGGGGGGGGGGGGGGGGGGGGGPGJ          
#                ~P55555555PP555Y??7??JY55PPPPPPPPPP&BPGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGY          
#                !PP5555PPPPPPP5YJ???JYY55PPPPPPPPPP#BGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGY          
#                !PPPPPPPPPPPP55YJ?JYJY55PPPPPPPPPPP##GGGGGGGGGGGGGGGGGBBBBBBBGGGGGGGGGGPG?          
#                ~PPPPPPPPGGGPPP5YJJYYY55PPPPPPPPGPP##GGGGGGGGGGGGGGGBGGGGGBBBGGGGGGGGGGGG!          
#                :PPPPPPPPPPPPPP5YJJJYY5PPPPGGGGGGGGB#GGGGGGGGGGGGGGBBBBBBBBGGGGGGGGGGGGGP:          
#                 JGPPPPPPPPPPPP5YJJJY5PPPGGGGGGGGGGB#BGBBBBBBBBBBBBBBBBBBBBBBBBGGGGGGGGG7           
#                 ^GGPPPPPPPPPPPP5YJJY5PPGGGGGGGGGGGB&#BBBBBBBBBBBBBBBBBBBBGGGGGGGGGGGGG5.           
#                  JGGGGGGGGGGGPPP5YYY5PPGGGGGGBGBB#####BBBBBBBBBBBBBBBBBBBGGGGGGGGGGGGG^            
#                  :5GGGGGGGGGGGGPP5YY5PPGGGGGBBBB#########BBBBBBBBBBBBBBBBGGGGGGGGGGGG~             
#                   ~GGGGGGGGGGGGGPP5Y5PGGGBBBB###############BBBBBBBBBBBGGGGGGGGGGGGG~              
#                    !GBBBBBBBBBBBBBBGGBBB#######################BBBBBBBGGGGGGGGGGGBP^               
#                     .!G#BBBBBBBBBBBBBBB############################BBBBBBBBBBBBBBY:                
#                       :Y##BBBBBBBBBBB##########################BBBBBBBBBBBBBB#GY7.                 
#                         ~5#####################################BBBBBBBBBBBBBP7.                    
#                           ~5#####################################BBBBBBB#B5!.                      
#                             ^?GBGB####################################BP?^                         
#                               JG5PBB##############################BBBJ:                            
#                              ~PPPGBBB####&&&&&&&&&&&&&&&&&&&##BBBBBBBJ                             
#                             .5PPGBBB##B~^~7JY5PGGBBBGGGP5Y?7GBB####BBG^                            
#                            .JP5PGBBB##!         ....        !######BBB5.                           
#                            !PPPGBBBB#?                       J&##&##BBB7                           
#                           .YGPGBBBB#5.                       .P&#&&##BBG^                          
#                           :GGGBBB#B#!                         ~B##&&##BB5.                         
#                           ^BBB####BB~                          ?&#&&&##B#J                         
#                           ^BB#######!                          .5&&&&&###B7:                       
#..............   .:^^~~!!!!J#######B#? ........................ .5&&&&&&@@@@#57^.    .              
#..........:...~JPB#&@@@@@@@@@@@@&&&&##5^.::::::::::::::::.......J@@@@@@@@@@@@@@@#P?~................
#::::::::::^^:J&@B#@@@@@@@@@@@@@@@@@@@@@#!^~~~~~~~~~~~~~~~^^^^^^^P@@@@@@@@@@@@@@@@@@@G!::::::::::::..
#^^^^^^^^~~~~!&@&YG@@@@@@@@@@@@@@@@@@@@@&Y?J?JJJJJJJJJJJJJJJJJJJ?P@@@@@&BP5G#@@@@@@@@@#~~~~^^^^^^^^::
#^^^^^^~~~~~!!5#&&&@@@@@@@@@@@@&&&###BBGP555YYYYYYYYYYYYYYYY5555PPG##&&@@@@@@@@@@@@@@&G7!!!!~~~~~~^^^
#::::^^^^^^^^^^~~!!!!7777777777!!!!!!!!!!!!!!77777777777777777777777777????????????777!~~~~~~~~^^^^^:
#..:::::::::::::::::::::::::::::^:^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^::::::