Robots.txt for semanticscholar.org

        # We are a non-profit research institute.  If you would like to collaborate with us,
# please contact us at: ai2-info@allenai.org
# Or check out our public API http://api.semanticscholar.org/
User-agent: *
Disallow: /search
Disallow: /error
Disallow: /me
Disallow: /api
Disallow: /author/*/claim
Disallow: /author/*?
Disallow: /paper/*?
Disallow: /reader/
Allow: /paper/*?p2df

Sitemap: https://www.semanticscholar.org/sitemap_author_index.xml
Sitemap: https://www.semanticscholar.org/sitemap_paper_index.xml
Sitemap: https://www.semanticscholar.org/sitemap_topic_index.xml