Free tools

Robots.txt Examples

Robots.txt file content for academia.edu.

Robot.txt file for: academia.edu

      # If you run a search engine and would like to index Academia.edu, please email support@academia.edu.

User-agent: Googlebot
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: Mediapartners-Google
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: msnbot
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: bingbot
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: Slurp
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: DuckDuckbot
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: baiduspider
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: seznambot
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: naverbot
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: teoma
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: Yandex
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: ScoutJet
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: GurujiBot
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: Exabot
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: Solbot
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: FAST
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: VoilaBot
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: daumoa
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: ArchitextSpider
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: Twitterbot
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: SocSciBot
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: ia_archiver
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: coccoc
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: Browsershots
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: SiteArc
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: Cliqzbot
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: proximic
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: OnCrawlBot
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: Botify
Disallow: /ads.txt
Disallow: /attachments/*/download_file
Disallow: /Jobs/Search
Disallow: /people/search
Disallow: /ac_search
Disallow: /users/search
Disallow: /ac_universities
Disallow: /ac_university_departments
Disallow: /document_viewer/build_vu_embeds/

Disallow: /v0/*

Allow: /v0/users/*/following
Allow: /v0/users/*/coauthors

Allow: /

User-agent: *
Disallow: /


Sitemap: https://sitemap.academia.edu/works_index.xml