Free tools

Robots.txt Examples

Robots.txt file content for alibaba.com.

Robot.txt file for: alibaba.com

      User-agent: *
Allow: /lp/*?
Allow: /*s=p
Allow: /*?_escaped_fragment_
Allow: /*src=sem_ggl
Allow: *?src=sem_ggl
Allow: /catalog/*?
Allow: /product-detail/
Allow: /showroom/
Allow: *?spm=
Allow: *?mark=google_shopping
Disallow: /bin/
Disallow: /apps/
Disallow: /buy/
Disallow: /memberhome/
Disallow: /minisite/
Disallow: /corporations/
Disallow: /buyers/
Disallow: /trade/
Disallow: /product/
Disallow: /catalogs/corporations/
Disallow: /rss/*/*.rss
Disallow: /activities/
Disallow: /productshowimg/
Disallow: /member/*/contactinfo.html
Disallow: /member/*/company_profile.html
Disallow: /member/*/productlist.html
Disallow: /searchweb/*.html
Disallow: /magellan/*.html
Disallow: /newsuppliers/
Disallow: /wholesalecheckout/
Disallow: /buyofferdetail/
Disallow: /trends/
Disallow: /industry-promotion/
Disallow: /city/
Disallow: /countrysearch/*/*-importer.html
Disallow: /countrysearch/*/*/importers/*.html
Disallow: /search/buyeroffers/*.html 
Disallow: /search/buyeroffers/*/*.html 
Disallow: /buyeroffers/*/*.html
Disallow: /buyeroffers/*/*/*.html
Disallow: /detail/ajax/
Disallow: /ajax/seoRequest/
Disallow: /event/
Disallow: /cache/
Disallow: /catalogs/products/
Disallow: /products/
Disallow: /g/
Disallow: /https:/www.alibaba.com/
Disallow:/compass/
Disallow:/product-detail/m.alibaba.com/trade/
Disallow: /messages/

User-agent: bingbot
Allow: /*s=p
Allow: /*?_escaped_fragment_
Allow: *?src=sem_ggl
Allow: /catalog/*?
Allow: *?mark=google_shopping
Disallow: /catalogs/products/
Disallow: /products/
Disallow: /bin/
Disallow: /apps/
Disallow: /buy/
Disallow: /memberhome/
Disallow: /minisite/
Disallow: /corporations/
Disallow: /buyers/
Disallow: /trade/
Disallow: /product/
Disallow: /catalogs/corporations/
Disallow: /rss/*/*.rss
Disallow: /activities/
Disallow: /productshowimg/
Disallow: /member/*/contactinfo.html
Disallow: /member/*/company_profile.html
Disallow: /member/*/productlist.html
Disallow: /searchweb/*.html
Disallow: /magellan/*.html
Disallow: /newsuppliers/
Disallow: /wholesalecheckout/
Disallow: /buyofferdetail/
Disallow: /trends/
Disallow: /industry-promotion/
Disallow: /city/
Disallow: /countrysearch/*/*-importer.html
Disallow: /countrysearch/*/*/importers/*.html
Disallow: /search/buyeroffers/*.html 
Disallow: /search/buyeroffers/*/*.html 
Disallow: /buyeroffers/*/*.html
Disallow: /buyeroffers/*/*/*.html
Disallow: /detail/ajax/
Disallow: /ajax/seoRequest/
Disallow: /event/
Disallow: /cache/
Disallow: /catalogs/products/
Disallow: /products/
Disallow: /g/
Disallow: /https:/www.alibaba.com/
Disallow:/compass/
Disallow: /product-detail/alisite-mobile.alibaba.com/
Disallow:/product-detail/m.alibaba.com/trade/

User-agent: Googlebot
Allow: /*s=p
Allow: /*?_escaped_fragment_
Allow: *?src=sem_ggl
Allow: /catalog/*?
Allow: /product-detail/
Allow: /showroom/
Allow: *?spm=
Allow: *?mark=google_shopping
Allow: /lp/*?
Disallow: /bin/
Disallow: /apps/
Disallow: /buy/
Disallow: /memberhome/
Disallow: /minisite/
Disallow: /corporations/
Disallow: /buyers/
Disallow: /trade/
Disallow: /product/
Disallow: /catalogs/corporations/
Disallow: /rss/*/*.rss
Disallow: /activities/
Disallow: /productshowimg/
Disallow: /member/*/contactinfo.html
Disallow: /member/*/company_profile.html
Disallow: /member/*/productlist.html
Disallow: /searchweb/*.html
Disallow: /magellan/*.html
Disallow: /newsuppliers/
Disallow: /wholesalecheckout/
Disallow: /buyofferdetail/
Disallow: /trends/
Disallow: /industry-promotion/
Disallow: /city/
Disallow: /countrysearch/*/*-importer.html
Disallow: /countrysearch/*/*/importers/*.html
Disallow: /search/buyeroffers/*.html 
Disallow: /search/buyeroffers/*/*.html 
Disallow: /buyeroffers/*/*.html
Disallow: /buyeroffers/*/*/*.html
Disallow: /detail/ajax/
Disallow: /ajax/seoRequest/
Disallow: /event/
Disallow: /cache/
Disallow: /catalogs/products/
Disallow: /products/
Disallow: /g/
Disallow: /https:/www.alibaba.com/
Disallow:/compass/
Disallow: /product-detail/alisite-mobile.alibaba.com/

User-agent: Googlebot-image
Allow: /lp/*?
Allow: /*s=p
Allow: /*?_escaped_fragment_
Allow: /*src=sem_ggl
Allow: *?src=sem_ggl
Allow: /catalog/*?
Allow: /product-detail/
Allow: /showroom/
Allow: *?spm=
Allow: *?mark=google_shopping
Disallow: /bin/
Disallow: /apps/
Disallow: /buy/
Disallow: /memberhome/
Disallow: /minisite/
Disallow: /corporations/
Disallow: /buyers/
Disallow: /trade/
Disallow: /product/
Disallow: /catalogs/corporations/
Disallow: /rss/*/*.rss
Disallow: /activities/
Disallow: /productshowimg/
Disallow: /member/*/contactinfo.html
Disallow: /member/*/company_profile.html
Disallow: /member/*/productlist.html
Disallow: /searchweb/*.html
Disallow: /magellan/*.html
Disallow: /newsuppliers/
Disallow: /wholesalecheckout/
Disallow: /buyofferdetail/
Disallow: /trends/
Disallow: /industry-promotion/
Disallow: /city/
Disallow: /countrysearch/*/*-importer.html
Disallow: /countrysearch/*/*/importers/*.html
Disallow: /search/buyeroffers/*.html 
Disallow: /search/buyeroffers/*/*.html 
Disallow: /buyeroffers/*/*.html
Disallow: /buyeroffers/*/*/*.html
Disallow: /detail/ajax/
Disallow: /ajax/seoRequest/
Disallow: /event/
Disallow: /cache/
Disallow: /catalogs/products/
Disallow: /products/
Disallow: /g/
Disallow: /https:/www.alibaba.com/
Disallow: /product-detail/alisite-mobile.alibaba.com/
Disallow:/compass/
Disallow:/product-detail/m.alibaba.com/trade/