Free tools

Robots.txt Examples

Robots.txt file content for allevents.in.

Robot.txt file for: allevents.in

      User-agent: AhrefsBot
Crawl-Delay: 10

User-agent: Yandex
Crawl-Delay: 30

User-agent: ICC-Crawler
Crawl-Delay: 30

User-agent: GoogleOther
Crawl-delay: 1

User-agent:*
disallow: /*/2013-0
disallow: /*/2013-1
allow: /*/*/2013-0
allow: /*/*/2013-1

disallow: /*/2014-0
disallow: /*/2014-1
allow: /*/*/2014-0
allow: /*/*/2014-1

disallow: /*/2015-0
disallow: /*/2015-1
allow: /*/*/2015-0
allow: /*/*/2015-1

disallow: /*/2016-0
disallow: /*/2016-1
allow: /*/*/2016-0
allow: /*/*/2016-1

disallow: /*/2017-0
disallow: /*/2017-1
allow: /*/*/2017-0
allow: /*/*/2017-1

disallow: /*/2018-0
disallow: /*/2018-1
allow: /*/*/2018-0
allow: /*/*/2018-1

disallow: /*/2019-0
disallow: /*/2019-1
allow: /*/*/2019-0
allow: /*/*/2019-1

disallow: /*/2020-0
disallow: /*/2020-1
allow: /*/*/2020-0
allow: /*/*/2020-1

disallow: /*/2021-0
disallow: /*/2021-1
allow: /*/*/2021-0
allow: /*/*/2021-1

disallow: /*/2022-0
disallow: /*/2022-1
allow: /*/*/2022-0
allow: /*/*/2022-1

disallow: /*/2023-0
disallow: /*/2023-1
allow: /*/*/2023-0
allow: /*/*/2023-1

disallow: /manage/*

Disallow: *?ref=eventpage$
disallow: /mobile/cityhome.php
Disallow: *?ref=rmenujs$
Disallow: *?ref=dmenujs$
Disallow: *?ref=ep_tags$
Disallow: *?ref=eventpage_v1$

disallow: /profile/messages/index.php
disallow: /go.php
disallow: /api/hotels_template.php