Free tools

Robots.txt Examples

Robots.txt file content for thriftbooks.com.

Robot.txt file for: thriftbooks.com

      Sitemap: https://www.thriftbooks.com/static/sitemap/sitemap.xml
Sitemap: https://www.thriftbooks.com/static/images-sitemap/sitemap.xml

User-agent: *

Disallow: /app_themes/
Disallow: /bin/
Disallow: /admin/
Disallow: /account/
Disallow: /AjaxPages/
Disallow: /*.axd$
Disallow: /*.axd
Disallow: /ScriptResource.axd
Disallow: /WebResource.axd
Disallow: /BundleImages/
Disallow: /cheap196zpgf723nsd845books/
Disallow: /coverimages/
Disallow: /emailImages/
Disallow: /Masterpages/
Disallow: /PopCalendar2005/
Disallow: /Replacementbooks/
Disallow: /services/
Disallow: /SuppliersLogo/
Disallow: /Usercontrols/
Disallow: /donotfollow/
Disallow: /.aspx$
Disallow: /.ashx$
Disallow: /review/

User-agent: Yandex
Disallow:
Crawl-delay: 10

User-agent: MSNBot
Disallow:
Crawl-delay: 5

User-agent: bingbot
Disallow:
Crawl-delay: 5

User-agent: Adidxbot
Disallow:
Crawl-delay: 5

User-agent: BingPreview
Disallow:
Crawl-delay: 5


User-agent: Googlebot 
Disallow: /app_themes/
Disallow: /bin/
Disallow: /account/
Disallow: /admin/
Disallow: /*.axd$
Disallow: /*.axd
Disallow: /ScriptResource.axd
Disallow: /WebResource.axd
Disallow: /BundleImages/
Disallow: /cheap196zpgf723nsd845books/
Disallow: /coverimages/
Disallow: /emailImages/
Disallow: /Masterpages/
Disallow: /PopCalendar2005/
Disallow: /Replacementbooks/
Disallow: /services/
Disallow: /SuppliersLogo/
Disallow: /Usercontrols/
Disallow: /.aspx$
Disallow: /.ashx$
Disallow: /review/