Free tools

Robots.txt Examples

Robots.txt file content for hawaii.edu.

Robot.txt file for: hawaii.edu

      # robots.txt for http://www.hawaii.edu/

User-agent: *
Disallow: /tip2008
Disallow: /dir/pdf
Disallow: /institutionalresearch
Disallow: /sls/sls
Disallow: /bjs
Disallow: /bls
Disallow: /bwiki
Disallow: /casl2
Disallow: /cip
Disallow: /dw
Disallow: /ecafe
Disallow: /epmt
Disallow: /filedrop
Disallow: /institutionalresearch
Disallow: /irodr
Disallow: /its/ws/holiday
Disallow: /itsadmin
Disallow: /jira
Disallow: /leave
Disallow: /odsdd
Disallow: /payroll/paystub
Disallow: /pfstest
Disallow: /policy/docs/temp
Disallow: /prof
Disallow: /profileService
Disallow: /taps
Disallow: /testaging
Disallow: /transferdatabase
Disallow: /trvl
Disallow: /workflow
Crawl-delay: 3

User-agent: Baiduspider
Disallow: /bjs
Disallow: /bls
Disallow: /bwiki
Disallow: /casl2
Disallow: /cip
Disallow: /dw
Disallow: /ecafe
Disallow: /epmt
Disallow: /filedrop
Disallow: /institutionalresearch
Disallow: /irodr
Disallow: /its/ws/holiday
Disallow: /itsadmin
Disallow: /jira
Disallow: /leave
Disallow: /odsdd
Disallow: /payroll/paystub
Disallow: /pfstest
Disallow: /prof
Disallow: /profileService
Disallow: /taps
Disallow: /testaging
Disallow: /transferdatabase
Disallow: /trvl
Disallow: /workflow

User-agent: Baiduspider-image
Disallow: /bjs
Disallow: /bls
Disallow: /bwiki
Disallow: /casl2
Disallow: /cip
Disallow: /dw
Disallow: /ecafe
Disallow: /epmt
Disallow: /filedrop
Disallow: /institutionalresearch
Disallow: /irodr
Disallow: /its/ws/holiday
Disallow: /itsadmin
Disallow: /jira
Disallow: /leave
Disallow: /odsdd
Disallow: /payroll/paystub
Disallow: /pfstest
Disallow: /prof
Disallow: /profileService
Disallow: /taps
Disallow: /testaging
Disallow: /transferdatabase
Disallow: /trvl
Disallow: /workflow