Free tools

Robots.txt Examples

Robots.txt file content for instagram.com.

Robot.txt file for: instagram.com

      # Notice: Collection of data on Instagram through automated means is
# prohibited unless you have express written permission from Instagram
# and may only be conducted for the limited purpose contained in said
# permission.
# All authorized user-agents listed on this page must comply with Meta’s
# Automated Data Collection Terms available at:
# https://www.facebook.com/legal/automated_data_collection_terms
User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: uptimerobot
Disallow: /

User-agent: viberbot
Disallow: /

User-agent: YaK
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: Applebot
Disallow: /accounts/login/*?next=
Disallow: /ajax/
Disallow: /client_error/
Disallow: /direct/
Disallow: /logging/
Disallow: /publicapi/
Disallow: /qp/batch_fetch_web/
Disallow: /query/

User-agent: Bingbot
Disallow: /*/c/
Disallow: /*/comments/
Disallow: /*/liked_by/
Disallow: /accounts/login/*?next=
Disallow: /ajax/
Disallow: /client_error/
Disallow: /direct/
Disallow: /logging/
Disallow: /publicapi/
Disallow: /qp/batch_fetch_web/
Disallow: /query/

User-agent: Discordbot
Disallow: /accounts/login/*?next=
Disallow: /ajax/
Disallow: /client_error/
Disallow: /direct/
Disallow: /logging/
Disallow: /publicapi/
Disallow: /qp/batch_fetch_web/
Disallow: /query/

User-agent: DuckDuckBot
Disallow: /accounts/login/*?next=
Disallow: /ajax/
Disallow: /client_error/
Disallow: /direct/
Disallow: /logging/
Disallow: /publicapi/
Disallow: /qp/batch_fetch_web/
Disallow: /query/

User-agent: facebookexternalhit
Disallow: /accounts/login/*?next=
Disallow: /ajax/
Disallow: /client_error/
Disallow: /direct/
Disallow: /logging/
Disallow: /publicapi/
Disallow: /qp/batch_fetch_web/
Disallow: /query/

User-agent: Googlebot
Disallow: /*/c/
Disallow: /*/comments/
Disallow: /*/liked_by/
Disallow: /accounts/login/*?next=
Disallow: /ajax/
Disallow: /client_error/
Disallow: /direct/
Disallow: /logging/
Disallow: /publicapi/
Disallow: /qp/batch_fetch_web/
Disallow: /query/

User-agent: Googlebot-Image
Disallow: /accounts/login/*?next=
Disallow: /ajax/
Disallow: /client_error/
Disallow: /direct/
Disallow: /logging/
Disallow: /publicapi/
Disallow: /qp/batch_fetch_web/
Disallow: /query/

User-agent: Googlebot-Video
Disallow: /accounts/login/*?next=
Disallow: /ajax/
Disallow: /client_error/
Disallow: /direct/
Disallow: /logging/
Disallow: /publicapi/
Disallow: /qp/batch_fetch_web/
Disallow: /query/

User-agent: Googlebot-News
Disallow: /accounts/login/*?next=
Disallow: /ajax/
Disallow: /client_error/
Disallow: /direct/
Disallow: /logging/
Disallow: /publicapi/
Disallow: /qp/batch_fetch_web/
Disallow: /query/

User-agent: Google-InspectionTool
Disallow: /accounts/login/*?next=
Disallow: /ajax/
Disallow: /client_error/
Disallow: /direct/
Disallow: /logging/
Disallow: /publicapi/
Disallow: /qp/batch_fetch_web/
Disallow: /query/

User-agent: LinkedInBot
Disallow: /accounts/login/*?next=
Disallow: /ajax/
Disallow: /client_error/
Disallow: /direct/
Disallow: /logging/
Disallow: /publicapi/
Disallow: /qp/batch_fetch_web/
Disallow: /query/

User-agent: msnbot
Disallow: /accounts/login/*?next=
Disallow: /ajax/
Disallow: /client_error/
Disallow: /direct/
Disallow: /logging/
Disallow: /publicapi/
Disallow: /qp/batch_fetch_web/
Disallow: /query/

User-agent: Naverbot
Disallow: /*/c/
Disallow: /*/comments/
Disallow: /*/liked_by/
Disallow: /accounts/login/*?next=
Disallow: /ajax/
Disallow: /client_error/
Disallow: /direct/
Disallow: /logging/
Disallow: /publicapi/
Disallow: /qp/batch_fetch_web/
Disallow: /query/

User-agent: Pinterestbot
Disallow: /accounts/login/*?next=
Disallow: /ajax/
Disallow: /client_error/
Disallow: /direct/
Disallow: /logging/
Disallow: /publicapi/
Disallow: /qp/batch_fetch_web/
Disallow: /query/

User-agent: Screaming Frog SEO Spider
Disallow: /accounts/login/*?next=
Disallow: /ajax/
Disallow: /client_error/
Disallow: /direct/
Disallow: /logging/
Disallow: /publicapi/
Disallow: /qp/batch_fetch_web/
Disallow: /query/

User-agent: seznambot
Disallow: /accounts/login/*?next=
Disallow: /ajax/
Disallow: /client_error/
Disallow: /direct/
Disallow: /logging/
Disallow: /publicapi/
Disallow: /qp/batch_fetch_web/
Disallow: /query/

User-agent: Slurp
Disallow: /accounts/login/*?next=
Disallow: /ajax/
Disallow: /client_error/
Disallow: /direct/
Disallow: /logging/
Disallow: /publicapi/
Disallow: /qp/batch_fetch_web/
Disallow: /query/

User-agent: teoma
Disallow: /accounts/login/*?next=
Disallow: /ajax/
Disallow: /client_error/
Disallow: /direct/
Disallow: /logging/
Disallow: /publicapi/
Disallow: /qp/batch_fetch_web/
Disallow: /query/

User-agent: TelegramBot
Disallow: /accounts/login/*?next=
Disallow: /ajax/
Disallow: /client_error/
Disallow: /direct/
Disallow: /logging/
Disallow: /publicapi/
Disallow: /qp/batch_fetch_web/
Disallow: /query/

User-agent: Twitterbot
Disallow: /accounts/login/*?next=
Disallow: /ajax/
Disallow: /client_error/
Disallow: /direct/
Disallow: /logging/
Disallow: /publicapi/
Disallow: /qp/batch_fetch_web/
Disallow: /query/

User-agent: Yeti
Disallow: /accounts/login/*?next=
Disallow: /ajax/
Disallow: /client_error/
Disallow: /direct/
Disallow: /logging/
Disallow: /publicapi/
Disallow: /qp/batch_fetch_web/
Disallow: /query/

User-agent: *
Disallow: /