Free tools

Robots.txt Examples

Robots.txt file content for dzen.ru.

Robot.txt file for: dzen.ru

      User-agent: Yandex
Allow: /
Allow: /.well-known/assetlinks.json
Disallow: /away
Disallow: /top
Disallow: /search
Disallow: /money
Disallow: /onboarding
Disallow: /news/top
Disallow: /news/smi
Disallow: /news/search
Disallow: /t/
Disallow: /user/
Disallow: /embed/
Disallow: /*/url
Disallow: /video/games/tags/
Disallow: /*sso_failed=
Disallow: /*erid=
Disallow: /*.html$

Clean-param: &_csrf
Clean-param: &action
Clean-param: &amp
Clean-param: &ampplace
Clean-param: &amupr_id
Clean-param: &appsearch_header
Clean-param: &autoplay
Clean-param: &backoffice
Clean-param: &branding
Clean-param: &briefEditorPublicationId
Clean-param: &bx_sender_conversion_id
Clean-param: &clid
Clean-param: &collection
Clean-param: &comment-request
Clean-param: &comments_data
Clean-param: &content
Clean-param: &contour
Clean-param: &country_code
Clean-param: &device_id
Clean-param: &encoded_pulse_user_id
Clean-param: &encoded_pulse_user_info
Clean-param: &epik
Clean-param: &experiment
Clean-param: &f
Clean-param: &family
Clean-param: &fan
Clean-param: &favid
Clean-param: &feed_exp
Clean-param: &feed_filter_source
Clean-param: &firstchars
Clean-param: &flight_id
Clean-param: &from
Clean-param: &from_block
Clean-param: &from_cloud
Clean-param: &from_floors_promo
Clean-param: &from_page
Clean-param: &from_parent_id
Clean-param: &from_parent_type
Clean-param: &from_site
Clean-param: &ga_client_id
Clean-param: &hide_interest_header
Clean-param: &id
Clean-param: &integration
Clean-param: &interview_id
Clean-param: &invite
Clean-param: &item_id
Clean-param: &lang
Clean-param: &link
Clean-param: &loop
Clean-param: &lr
Clean-param: &modal
Clean-param: &mt_click_id
Clean-param: &mt_link_id
Clean-param: &mute
Clean-param: &no_ad
Clean-param: &noredir
Clean-param: &nw
Clean-param: &p
Clean-param: &page
Clean-param: &parent_rid
Clean-param: &persistent_id
Clean-param: &persistent_id
Clean-param: &place
Clean-param: &play_on_visible
Clean-param: &pulse_user_id
Clean-param: &query
Clean-param: &ref
Clean-param: &referrer
Clean-param: &referrer_clid
Clean-param: &referrer_place
Clean-param: &repostId
Clean-param: &repostId
Clean-param: &rid
Clean-param: &rubric
Clean-param: &secdata
Clean-param: &service
Clean-param: &share_to
Clean-param: &sid
Clean-param: &sortby
Clean-param: &source
Clean-param: &sso_failed
Clean-param: &stid
Clean-param: &story
Clean-param: &t
Clean-param: &text
Clean-param: &to
Clean-param: &token
Clean-param: &train
Clean-param: &tst
Clean-param: &tt
Clean-param: &tv
Clean-param: &utm_campaign
Clean-param: &utm_content
Clean-param: &utm_floors
Clean-param: &utm_medium
Clean-param: &utm_referer
Clean-param: &utm_referrer
Clean-param: &utm_source
Clean-param: &utm_term
Clean-param: &utr
Clean-param: &uuid
Clean-param: &vertical
Clean-param: &via
Clean-param: &vid
Clean-param: &video_id
Clean-param: &wan
Clean-param: &web_uid
Clean-param: &web_yandexuid
Clean-param: &wizard
Clean-param: &yandexClientId

User-agent: Googlebot
Allow: /
Allow: /.well-known/assetlinks.json
Disallow: /away
Disallow: /top
Disallow: /search
Disallow: /money
Disallow: /onboarding
Disallow: /news/top
Disallow: /news/smi
Disallow: /news/search
Disallow: /t/
Disallow: /user/
Disallow: /embed/
Disallow: /*/url
Disallow: /video/games/tags/
Disallow: /*_csrf
Disallow: /*action
Disallow: /*amp
Disallow: /*ampplace
Disallow: /*amupr_id
Disallow: /*appsearch_header=
Disallow: /*autoplay=
Disallow: /*backoffice=
Disallow: /*branding=
Disallow: /*briefEditorPublicationId=
Disallow: /*bx_sender_conversion_id=
Disallow: /*cachebust=
Disallow: /*clid=
Disallow: /*collection=
Disallow: /*country_code=
Disallow: /*erid=
Disallow: /*experiment=
Disallow: /*feed_exp=
Disallow: /*from_cloud=
Disallow: /*from_page=
Disallow: /*invite=
Disallow: /*loop=
Disallow: /*pulse_user_id=
Disallow: /*sso_failed=
Disallow: /*t=
Disallow: /*to=
Disallow: /*token=
Disallow: /*utm_referrer=
Disallow: /*utm_source=

User-agent: Mail.ru
Allow: /
Allow: /.well-known/assetlinks.json
Disallow: /top
Disallow: /search
Disallow: /money
Disallow: /onboarding
Disallow: /news/top
Disallow: /news/smi
Disallow: /news/search
Disallow: /t/
Disallow: /user/
Disallow: /embed/
Disallow: /*/url
Disallow: /video/games/tags/
Disallow: /*sso_failed=
Disallow: /*to=
Disallow: /*erid=
Disallow: /*.html$

User-agent: Bingbot
Allow: /
Allow: /.well-known/assetlinks.json
Disallow: /top
Disallow: /search
Disallow: /money
Disallow: /onboarding
Disallow: /news/top
Disallow: /news/smi
Disallow: /news/search
Disallow: /t/
Disallow: /user/
Disallow: /embed/
Disallow: /*/url
Disallow: /video/games/tags/
Disallow: /*sso_failed=
Disallow: /*to=
Disallow: /*erid=
Disallow: /*.html$

User-agent: Applebot
Allow: /
Allow: /.well-known/assetlinks.json
Disallow: /away
Disallow: /top
Disallow: /search
Disallow: /money
Disallow: /onboarding
Disallow: /news/top
Disallow: /news/smi
Disallow: /news/search
Disallow: /t/
Disallow: /user/
Disallow: /embed/
Disallow: /*/url
Disallow: /video/games/tags/
Disallow: /*?sso_failed=
Disallow: /*to=
Disallow: /*erid=
Disallow: /*.html$

User-agent: baiduspider
Allow: /
Allow: /.well-known/assetlinks.json
Disallow: /away
Disallow: /top
Disallow: /search
Disallow: /money
Disallow: /onboarding
Disallow: /news/top
Disallow: /news/smi
Disallow: /news/search
Disallow: /t/
Disallow: /user/
Disallow: /embed/
Disallow: /*/url
Disallow: /video/games/tags/
Disallow: /*?sso_failed=
Disallow: /*to=
Disallow: /*erid=
Disallow: /*.html$

User-agent: msnbot
Allow: /
Allow: /.well-known/assetlinks.json
Disallow: /away
Disallow: /top
Disallow: /search
Disallow: /money
Disallow: /onboarding
Disallow: /news/top
Disallow: /news/smi
Disallow: /news/search
Disallow: /t/
Disallow: /user/
Disallow: /embed/
Disallow: /*/url
Disallow: /video/games/tags/
Disallow: /*sso_failed=
Disallow: /*to=
Disallow: /*erid=
Disallow: /*.html$

User-agent: GPTBot
Disallow: /

User-Agent: *
Allow: /
Allow: /.well-known/assetlinks.json
Disallow: /away
Disallow: /top
Disallow: /search
Disallow: /money
Disallow: /onboarding
Disallow: /news/top
Disallow: /news/smi
Disallow: /news/search
Disallow: /t/
Disallow: /user/
Disallow: /*/url
Disallow: /video/games/tags/
Disallow: /*sso_failed=
Disallow: /*to=
Disallow: /*erid=
Disallow: /*.html$

Sitemap: https://dzen.ru/sitemaps/daily/sitemap.xml
Sitemap: https://dzen.ru/sitemaps/daily/sitemap.xml.gz
Sitemap: https://dzen.ru/sitemaps/channels/sitemap.xml
Sitemap: https://dzen.ru/sitemaps/channels/sitemap.xml.gz
Sitemap: https://dzen.ru/sitemaps/video/sitemap.xml
Sitemap: https://dzen.ru/sitemaps/video/sitemap.xml.gz
Sitemap: https://dzen.ru/sitemaps/brief/sitemap.xml
Sitemap: https://dzen.ru/sitemaps/brief/sitemap.xml.gz
Sitemap: https://dzen.ru/sitemaps/native/sitemap.xml
Sitemap: https://dzen.ru/sitemaps/native/sitemap.xml.gz
Sitemap: https://dzen.ru/sitemaps/suite/sitemap.xml
Sitemap: https://dzen.ru/sitemaps/suite/sitemap.xml.gz
Sitemap: https://dzen.ru/sitemaps/video-games/sitemap.xml
Sitemap: https://dzen.ru/sitemaps/video-games/sitemap.xml.gz
Sitemap: https://dzen.ru/sitemaps/image/sitemap.xml
Sitemap: https://dzen.ru/sitemaps/image/sitemap.xml.gz
Sitemap: https://dzen.ru/sitemaps/tv/sitemap.xml
Sitemap: https://dzen.ru/sitemaps/tv/sitemap.xml.gz
Sitemap: https://dzen.ru/sitemaps/slug-factory/sitemap.xml
Sitemap: https://dzen.ru/sitemaps/slug-factory/sitemap.xml.gz
Sitemap: https://dzen.ru/sitemaps/news-archive/sitemap.xml
Sitemap: https://dzen.ru/sitemaps/news-archive/sitemap.xml.gz
Sitemap: https://dzen.ru/sitemaps/news-region/sitemap.xml
Sitemap: https://dzen.ru/sitemaps/news-region/sitemap.xml.gz
Sitemap: https://dzen.ru/sitemaps/news-rubric/sitemap.xml
Sitemap: https://dzen.ru/sitemaps/news-rubric/sitemap.xml.gz