Free tools
Robots.txt Examples
Robots.txt file content for teepublic.com.
Robot.txt file for: teepublic.com
# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file # # To ban all spiders from the entire site uncomment the next two lines: # User-Agent: * # Disallow: / User-agent: * Allow: /kids-long-sleeve-t-shirt/*?feed_sku* Allow: /tote/*?countrycode* Disallow: /*/poll_avatar Disallow: /*/poll_banner Disallow: /*==$ Disallow: /*?*canvas_subclass= Disallow: /*?*color= Disallow: /*?*gender= Disallow: /*?*product_id= Disallow: /*?*query= Disallow: /*?*scroll_id= Disallow: /*?*sort= Disallow: /404$ Disallow: /422$ Disallow: /500$ Disallow: /503$ Disallow: /cart Disallow: /cdn-cgi/* Disallow: /checkout Disallow: /designs/*/edit Disallow: /following Disallow: /gift-ideas Disallow: /gifting-module/* Disallow: /header Disallow: /kids-long-sleeve-t-shirt/* Disallow: /kids-long-sleeve-t-shirts Disallow: /kids-long-sleeve-t-shirts/* Disallow: /laptop-case Disallow: /laptop-case/* Disallow: /my-store Disallow: /new_campaign Disallow: /notebook Disallow: /notebook/* Disallow: /stores/*/kids-long-sleeve-t-shirts Disallow: /stores/*/laptop-case Disallow: /stores/*/notebook Disallow: /stores/*/totes Disallow: /tote/* Disallow: /totes Disallow: /totes/* Disallow: /stores/*/baby-bodysuits Disallow: /baby-bodysuit/* Disallow: /baby-bodysuits Disallow: /baby-bodysuits/* Disallow: /user/*/kids-long-sleeve-t-shirts Disallow: /user/*/laptop-case Disallow: /user/*/notebook Disallow: /user/*/totes Disallow: /user/*/baby-bodysuits Disallow: /users/password Disallow: /*US_CENSUS_NAME* Disallow: /designs/*/canvas/*/also_available_as Disallow: /designs/*/canvas/*/also-available-products Disallow: /designs/*/canvas/*/product_images Disallow: /designs/*/canvas/*/similar_products Disallow: /lazyload_cart_component_content Disallow: /lazyload_cart_tray_content Disallow: /lazyload_shop_tray_best_sellers_content Disallow: /shop_designs_button Disallow: /tray_trending_tags Disallow: /ab-test/* Disallow: /youtube Disallow: /tracking/* # Disallow international locales Disallow: /de/ Disallow: /en-au/ Disallow: /en-gb/ Disallow: /es-mx/ Disallow: /fr/ Disallow: /it/ Disallow: /pl/ # Disallow for "bad traffic" adult terms Disallow: /*/*sexy-bf* Disallow: /*/*bf-sexy* Disallow: /*/*sexybf* Disallow: /*/*indian-bf* Disallow: /*/*sexi-bf* Disallow: /*/*baby-alien-porn* Disallow: /*/*xxx14* Disallow: /*/*bangladeshi-bf* Disallow: /*/*xxx12* Disallow: /*/*shemale-tube* Disallow: /*/*sexy-bf* Disallow: /*/*bf-video* Disallow: /*/*xxx15* Disallow: /*/*bangla-bf* Disallow: /*/*bengali-bf* Disallow: /*/*xxx13* Disallow: /*/*bangladeshi-bf-bangladeshi-bf* Disallow: /*/*baby-alien-porno* Disallow: /*/*xxx8* Disallow: /*/*bangla-bf-video* Disallow: /*/*bf-sexy* Disallow: /*/*cute-porn* Disallow: /*/*xxx10* Disallow: /*/*porn-baby-alien* Disallow: /*/*hot-fuck* Disallow: /*/*tamil-aunty* Disallow: /*/*bf-bangladeshi* Disallow: /*/*bf-bangla* Disallow: /*/*big-boobs-porn* User-agent: Pinterest/0.2 (+https://www.pinterest.com/bot.html) Crawl-delay: 1 User-agent: Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html) Crawl-delay: 1 User-agent: Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html) Crawl-delay: 1 User-agent: facebookexternalhit Disallow: /sizechart