Free tools

Robots.txt Examples

Robots.txt file content for wordreference.com.

Robot.txt file for: wordreference.com

      User-agent: Mediapartners-Google
User-agent: Googlebot-Mobile
User-agent: Googlebot
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
Allow: /ads.txt
Allow: /app-ads.txt
# temp for bots
Disallow: /enit/*.com
Disallow: /englishusage/
Disallow: /EnglishUsage/
Disallow: /*src
Disallow: /2012/collins/
Disallow: /dictionary/getcollins/
Disallow: /*/collins/
Disallow: /*/reverse/
Disallow: /2012/Goog
Disallow: /dictionary/googTranslate
Disallow: /translators/
Disallow: /english/suggestions.aspx
# these pages have NOINDEX...
Disallow:  /definizione/geraschelt
Disallow:  /enru/The%20Trachiniae%20[Sophocles]
Disallow:  /es/en/translation.asp?spen=CHAPITA
Disallow:  /enfr/Strandmeistereinheit
Disallow:  /entr/Die%20Geh%C3%A4lter%20wurden%20gek%C3%BCrzt
Disallow:  /iten/lastricher%C3%A0
Disallow:  /es/translation.asp?tranword=%5BRindenmulch%5D&dict=enes
Disallow:  /definition/barred%20ghoul
Disallow:  /sinonimos/innerpolitischselten
Disallow:  /enru/relative%20Adressierung
Disallow:  /pten/deslanchar
Disallow:  /esfr/Empella
Disallow:  /es/translation.asp?tranword=%5BReadinghtmliframe=true%5D
Disallow:  /enpt/treat%20sb
Disallow:  /sinonimos/[Muscle&Fitness]
Disallow:  /enzh/black%20capped%20screech%20owl%20[Megascops%20atricapillus]
Disallow:  /enpt/in%20Gefahr%20begab
Disallow:  /es/en/translation.asp?spen=incende
Disallow:  /ende/If%20I%20Knew%20You%20Were%20Comin/%5C'%20I/as%20if
Disallow:  /enfr/FUT+Hack
Disallow:  /esfr/Unternehmensanteil
Disallow:  /engr/Reluctivity
Disallow:  /fren/volumeters
Disallow:  /es/translation.asp?tranword=kivitatti
Disallow:  /enfr/in/?start=2500
Disallow:  /esfr/fixedneutered
Disallow:  /enpl/nass%20spritzen
Disallow:  /enit/prim%E4re+Wachstum
Disallow:  /enfr/He/%5C's%20good%20at%20French
Disallow:  /thesaurus/aloveh
Disallow:  /iten/knarrender
Disallow:  /enit/biophysicist%20[female]
Disallow:  /definition/Rosamund
Disallow:  /es/translation.asp?tranword=Mercedes+Schiff&dict=enes
Disallow:  /enja/Duscht
Disallow:  /fres/grandseigneur
Disallow:  /es/translation.asp?tranword=poetiskt&dict=enes
Disallow:  /enpl/to%20give%20somebody%20hell
Disallow:  /enes/expectant%20heir
Disallow:  /enar/to%20recognize%20a%20responsibility
Disallow:  /encz/%D1%80%D1%83%D0%BF%D0%B8%D1%98%D0%B0
Disallow:  /es/en/translation.asp?spen=en%20todo%20lo%20descubierto?start=100&start=200
Disallow:  /enit/hyperinsulin%E4mischer
Disallow:  /definicion/entlastendes
Disallow:  /iten/Rotationskolbenpumpe
Disallow:  /enzh/Ablaut%20[histor%20Sprachwissenschaft]
Disallow:  /definition/wipedout
Disallow:  /definition/A.%20O.
Disallow:  /engr/freeport


User-agent: Teoma
User-agent: Slurp
User-agent: Yeti
User-agent: Yandex
User-agent: NaverBot
User-agent: bingbot
User-agent: Applebot
User-agent: Baiduspider
User-agent: Sogou web spider
Crawl-delay: 1
Disallow: /2012/collins/
Disallow: /dictionary/getcollins/
Disallow: /2012/Goog
Disallow: /dictionary/googTranslate
Disallow: /translators/
Disallow: /*/collins/
Disallow: /*/reverse/
Disallow: /czde/
Disallow: /czes/
Disallow: /czfr/
Disallow: /czgr/
Disallow: /czit/
Disallow: /czja/
Disallow: /czko/
Disallow: /cznl/
Disallow: /czpl/
Disallow: /czpt/
Disallow: /czro/
Disallow: /czru/
Disallow: /czsv/
Disallow: /cztr/
Disallow: /czzh/
Disallow: /decz/
Disallow: /defr/
Disallow: /degr/
Disallow: /deit/
Disallow: /deja/
Disallow: /deko/
Disallow: /denl/
Disallow: /depl/
Disallow: /dept/
Disallow: /dero/
Disallow: /deru/
Disallow: /desv/
Disallow: /detr/
Disallow: /dezh/
Disallow: /escz/
Disallow: /esgr/
Disallow: /esja/
Disallow: /esko/
Disallow: /esnl/
Disallow: /espl/
Disallow: /esro/
Disallow: /esru/
Disallow: /essv/
Disallow: /estr/
Disallow: /eszh/
Disallow: /frcz/
Disallow: /frde/
Disallow: /frgr/
Disallow: /frit/
Disallow: /frja/
Disallow: /frko/
Disallow: /frnl/
Disallow: /frpl/
Disallow: /frpt/
Disallow: /frro/
Disallow: /frru/
Disallow: /frsv/
Disallow: /frtr/
Disallow: /frzh/
Disallow: /grcz/
Disallow: /grde/
Disallow: /gres/
Disallow: /grfr/
Disallow: /grit/
Disallow: /grja/
Disallow: /grko/
Disallow: /grnl/
Disallow: /grpl/
Disallow: /grpt/
Disallow: /grro/
Disallow: /grru/
Disallow: /grsv/
Disallow: /grtr/
Disallow: /grzh/
Disallow: /itcz/
Disallow: /itde/
Disallow: /itfr/
Disallow: /itgr/
Disallow: /itja/
Disallow: /itko/
Disallow: /itnl/
Disallow: /itpl/
Disallow: /itpt/
Disallow: /itro/
Disallow: /itru/
Disallow: /itsv/
Disallow: /ittr/
Disallow: /itzh/
Disallow: /jacz/
Disallow: /jade/
Disallow: /jaes/
Disallow: /jafr/
Disallow: /jagr/
Disallow: /jait/
Disallow: /jako/
Disallow: /janl/
Disallow: /japl/
Disallow: /japt/
Disallow: /jaro/
Disallow: /jaru/
Disallow: /jasv/
Disallow: /jatr/
Disallow: /jazh/
Disallow: /kocz/
Disallow: /kode/
Disallow: /koes/
Disallow: /kofr/
Disallow: /kogr/
Disallow: /koit/
Disallow: /koja/
Disallow: /konl/
Disallow: /kopl/
Disallow: /kopt/
Disallow: /koro/
Disallow: /koru/
Disallow: /kosv/
Disallow: /kotr/
Disallow: /kozh/
Disallow: /nlcz/
Disallow: /nlde/
Disallow: /nles/
Disallow: /nlfr/
Disallow: /nlgr/
Disallow: /nlit/
Disallow: /nlja/
Disallow: /nlko/
Disallow: /nlpl/
Disallow: /nlpt/
Disallow: /nlro/
Disallow: /nlru/
Disallow: /nlsv/
Disallow: /nltr/
Disallow: /nlzh/
Disallow: /plcz/
Disallow: /plde/
Disallow: /ples/
Disallow: /plfr/
Disallow: /plgr/
Disallow: /plit/
Disallow: /plja/
Disallow: /plko/
Disallow: /plnl/
Disallow: /plpt/
Disallow: /plro/
Disallow: /plru/
Disallow: /plsv/
Disallow: /pltr/
Disallow: /plzh/
Disallow: /ptcz/
Disallow: /ptde/
Disallow: /ptfr/
Disallow: /ptgr/
Disallow: /ptit/
Disallow: /ptja/
Disallow: /ptko/
Disallow: /ptnl/
Disallow: /ptpl/
Disallow: /ptro/
Disallow: /ptru/
Disallow: /ptsv/
Disallow: /pttr/
Disallow: /ptzh/
Disallow: /rocz/
Disallow: /rode/
Disallow: /roes/
Disallow: /rofr/
Disallow: /rogr/
Disallow: /roit/
Disallow: /roja/
Disallow: /roko/
Disallow: /ronl/
Disallow: /ropl/
Disallow: /ropt/
Disallow: /roru/
Disallow: /rosv/
Disallow: /rotr/
Disallow: /rozh/
Disallow: /rucz/
Disallow: /rude/
Disallow: /rues/
Disallow: /rufr/
Disallow: /rugr/
Disallow: /ruit/
Disallow: /ruja/
Disallow: /ruko/
Disallow: /runl/
Disallow: /rupl/
Disallow: /rupt/
Disallow: /ruro/
Disallow: /rusv/
Disallow: /rutr/
Disallow: /ruzh/
Disallow: /svcz/
Disallow: /svde/
Disallow: /sves/
Disallow: /svfr/
Disallow: /svgr/
Disallow: /svit/
Disallow: /svja/
Disallow: /svko/
Disallow: /svnl/
Disallow: /svpl/
Disallow: /svpt/
Disallow: /svro/
Disallow: /svru/
Disallow: /svtr/
Disallow: /svzh/
Disallow: /trcz/
Disallow: /trde/
Disallow: /tres/
Disallow: /trfr/
Disallow: /trgr/
Disallow: /trit/
Disallow: /trja/
Disallow: /trko/
Disallow: /trnl/
Disallow: /trpl/
Disallow: /trpt/
Disallow: /trro/
Disallow: /trru/
Disallow: /trsv/
Disallow: /trzh/
Disallow: /zhcz/
Disallow: /zhde/
Disallow: /zhes/
Disallow: /zhfr/
Disallow: /zhgr/
Disallow: /zhit/
Disallow: /zhja/
Disallow: /zhko/
Disallow: /zhnl/
Disallow: /zhpl/
Disallow: /zhpt/
Disallow: /zhro/
Disallow: /zhru/
Disallow: /zhsv/
Disallow: /zhtr/

# Tripleliftminutemedia ad partner
User-agent: Opebot-v (https://www.1plusx.com)
Allow: /
Disallow:


# now disallowing all other bots, No AI allowed, for permission, write us: https://forum.wordreference.com/misc/contact
User-agent: Google-Extended
User-agent: *
Allow: /ads.txt
Allow: /app-ads.txt
Disallow: /