Free tools

Robots.txt Examples

Robots.txt file content for pagesjaunes.fr.

Robot.txt file for: pagesjaunes.fr

      User-agent: *
Disallow: /temporaire
Disallow: /annuaireinverse/*
Disallow: /pagesblanches/recherche
Disallow: /pagesblanches/recherche/*
Disallow: /particuliers/*
Disallow: /cartographie*?bloc_id=*
Disallow: /carte?*
Disallow: /carte/*
Disallow: /pros/avis-*
Disallow: /pros/detail*
Disallow: /pros/*/detail_*
Disallow: /pros/visionneuse/*
Disallow: /pros/cqfd
Disallow: /pros/apps/reservation
Disallow: /pros/B-
Disallow: /pros/*?transac=
Disallow: /pros/prolive_widget/*
Disallow: /pros/click-and-collect/*
Disallow: /pro/
Disallow: /pro/annonces-auto-moto/
Disallow: /pro/annonces-auto/
Disallow: /pro/annonces-immo/
Disallow: /pro/annonces-moto/
Disallow: /pro/reservation/
Disallow: /activites/*/*.html
Disallow: /annuaire/chercherlespros
Disallow: /annuaire/pub?*
Disallow: /annuaire/pub$
Disallow: /annuaire/pub/starsky*
Disallow: /annuaire/*/&dateRdv=
Disallow: /annuaire/click-and-collect/*
Disallow: /click-and-collect/*/catalogue
Disallow: /ville/*/bons-plans/*-*
Disallow: /ville/*/bons-plans/t_*
Disallow: /ville/*/dossiers/
Disallow: /ville/*/sorties/*
Disallow: /applemapaction
Disallow: /partenaire*
Disallow: /utilisateur/*
Disallow: /contribution/*
Disallow: /historique
Disallow: /pub$
Disallow: /ajax/*
Disallow: /*/ajax/*
Disallow: /?FRM_LOCALITE=
Disallow: /admin/
Disallow: /astuces
Disallow: /edito
Disallow: /ilo
Disallow: /moncompte
Disallow: /refer*
Disallow: /reservation-hotel*
Disallow: /suggestion/
Disallow: /*crypt=*
Disallow: /*form_contact*
Disallow: /*type=*
Disallow: /*idContext*
Disallow: /*filtreContribution*
Disallow: /*faire=*
Disallow: /*IdAnnonce=*
Disallow: /*=kapstages*
Disallow: /*region=*
Disallow: /sorties/*
Disallow: /*?iframe=true
Disallow: /*?closePush=true
Disallow: /?sid=
Disallow: */fragment/*
Disallow: /?auto/
Disallow: /?immo/
Disallow: */RealMedia/ads/Creatives/pub.js
Disallow: /plusdeservices/generate-captcha
Disallow: */dynaTraceMonitor?*
Disallow: /dtagent_*.js
Disallow: /pros/vcard?*
Disallow: */monochamp/toggle
Disallow: /beta
Disallow: /onboarding/*
Disallow: /kameleoon/*
Disallow: /stat/blocVu
Disallow: */transac/*
Disallow: *no_sequence=*
Disallow: /*?quoiqui=
Disallow: /*?code_localite=
Disallow: /*?code_rubrique=
Disallow: /*?dpa=
Disallow: /*?xtor=
Disallow: /sw-registration.js
Disallow: /javascript/ads.js?*
Disallow: /pmp/blocVu
Disallow: /pmp/pageal
Disallow: /pros/depot_avis_interne
Disallow: /pros/csrf-generation/*
Disallow: /pros/csrf-generation-tokens
Disallow: /assets/js/kameleoon.js
Disallow: /sdk-piano/*
Disallow: /empty.gif

#Vintage
Disallow: /scg/guideLieu.do

#Specifically Block user agent
User-agent: MJ12bot
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: dotbot
User-agent: GPTBot
User-agent: ChatGPT-User
User-Agent: Google-Extended
User-agent: CCBot
User-agent: Amazonbot
User-agent: OAI-SearchBot
Disallow: /

#Block AdsBot  
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
Disallow: /
Disallow: /pros/detail
Disallow: /annuaire/authentication/
Disallow: /pros/ajax/
Disallow: /annuaire/ajax/
Allow: /pros/
Allow: /pagesblanches/r/
Allow: /annuaire/
Allow: /siret/