Konigle

November 26, 2024

Robots.txt for city-data.com

        User-agent: Mediapartners-Google
Disallow:

User-agent: Mediapartners-Google*
Disallow:


User-agent: *
Disallow: /forum/search.php*
Disallow: /forum/reputation.php*
Disallow: /forum/members
Disallow: /forum/newreply.php*
Disallow: /forum/subscribedforums.php*
Disallow: /cityname.php*
Disallow: /cityname2.php*
Disallow: /news/
Disallow: /forum2/
Disallow: /forumc/
Disallow: /forumbeta/
Disallow: /cityw/
Disallow: /roost_ads/
Disallow: /cd_ads/
Disallow: /forum/fort-myers-cape-coral-area/515615-beware-dr-robert-di-fronzo.html
Disallow: /bs/
Disallow: /business-profiles/map.php*
Disallow: /so/*.js
Disallow: /soz/*.js
Disallow: /knowledge/
Disallow: /mapIt.html*
Disallow: /restaurants/afd.class.php*
Disallow: /city/js/getBoxes.php*
Disallow: /county/js/getBoxes.php*
Disallow: /zips/js/getBoxes.php*
Disallow: /neighborhood/js/getBoxes.php*
Disallow: /mtc.class.php*
Disallow: /restaurants/menus/
Disallow: /mod_comments/*
Disallow: /fc/*

User-agent: Google-Extended
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: ImagesiftBot
User-agent: CCBot
User-agent: omgili
User-agent: Diffbot
User-agent: PerplexityBot
User-agent: anthropic-ai
User-agent: FacebookBot
User-agent: PiplBot
User-agent: GPTBot
User-agent: Amazonbot
User-agent: AwarioRssBot
User-agent: AwarioSmartBot
User-agent: cohere-ai
User-agent: DataForSeoBot
User-agent: Webzio-Extended
User-agent: FriendlyCrawler
User-agent: 0AI-SearchBot
User-agent: ImagesiftBot
User-agent: omgili
User-agent: omgilibot
User-agent: PerplexityBot
User-agent: Quora-Bot
User-agent: TurnitinBot
User-agent: Meta-ExternalAgent
Disallow: /

User-agent: 008
Disallow: /

User-agent: sitebot
Disallow: /

User-agent: businessdbbot
Disallow: /

User-agent: magpie-crawle
Disallow: /

User-agent: Bender
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: JikeSpider
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: proximic
Disallow: /

User-agent: ADmantX
Disallow: /

User-agent: Mediapartners-Google
Allow: /knowledge

User-agent: Baiduspider
User-agent: 360Spider
User-agent: Yisouspider
User-agent: PetalBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: Sogou inst spider
Disallow: /

Keep Reading